|
| 1 | +#!/usr/bin/env bash |
| 2 | +# Resolve a pushed app digest to CodePipeline -> CodeDeploy -> every ECS target's |
| 3 | +# AllowTraffic event. An unchanged tag uses since-epoch=0 to verify the latest |
| 4 | +# pipeline execution instead of assuming the tagged image is already serving. |
| 5 | +# Usage: wait-for-ecs-cutover.sh <pipeline-name> <image-digest> <since-epoch> |
| 6 | +set -euo pipefail |
| 7 | + |
| 8 | +PIPELINE="${1:?pipeline name required}" |
| 9 | +DIGEST="${2:?image digest required}" |
| 10 | +SINCE_EPOCH="${3:?since-epoch required}" |
| 11 | +POLL_INTERVAL="${POLL_INTERVAL:-15}" |
| 12 | +OVERALL_TIMEOUT="${OVERALL_TIMEOUT:-4200}" |
| 13 | +if ! [[ "$PIPELINE" =~ ^[A-Za-z0-9.@_-]+$ && "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ && "$SINCE_EPOCH" =~ ^[0-9]+$ && "$POLL_INTERVAL" =~ ^[1-9][0-9]*$ && "$OVERALL_TIMEOUT" =~ ^[1-9][0-9]*$ ]]; then |
| 14 | + echo 'ERROR: invalid pipeline, digest, epoch, or polling budget' >&2 |
| 15 | + exit 1 |
| 16 | +fi |
| 17 | +export AWS_PAGER='' |
| 18 | +export AWS_RETRY_MODE=standard |
| 19 | +export AWS_MAX_ATTEMPTS=3 |
| 20 | + |
| 21 | +deadline=$(( $(date +%s) + OVERALL_TIMEOUT )) |
| 22 | +log() { echo "[wait-for-ecs-cutover] $*"; } |
| 23 | +check_deadline() { |
| 24 | + if [ "$(date +%s)" -ge "$deadline" ]; then |
| 25 | + log "ERROR: timed out after ${OVERALL_TIMEOUT}s waiting for $1" |
| 26 | + exit 1 |
| 27 | + fi |
| 28 | +} |
| 29 | +aws_read() { |
| 30 | + aws --cli-connect-timeout 10 --cli-read-timeout 30 "$@" |
| 31 | +} |
| 32 | + |
| 33 | +find_execution() { |
| 34 | + local executions |
| 35 | + executions=$(aws_read codepipeline list-pipeline-executions \ |
| 36 | + --pipeline-name "$PIPELINE" --max-items 30 \ |
| 37 | + --query 'pipelineExecutionSummaries' --output json) |
| 38 | + printf '%s\n' "$executions" | SINCE="$SINCE_EPOCH" DIGEST="$DIGEST" python3 -c ' |
| 39 | +import datetime, json, os, sys |
| 40 | +since = int(os.environ["SINCE"]) |
| 41 | +def epoch(execution): |
| 42 | + value = execution["startTime"] |
| 43 | + if isinstance(value, (int, float)): |
| 44 | + return value |
| 45 | + return datetime.datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() |
| 46 | +def matches(execution): |
| 47 | + return any(r["actionName"] == "ECR_Source" and r.get("revisionId") == os.environ["DIGEST"] for r in execution.get("sourceRevisions", [])) |
| 48 | +executions = sorted(json.load(sys.stdin), key=epoch, reverse=True) |
| 49 | +if since == 0: |
| 50 | + if not executions or not matches(executions[0]): |
| 51 | + raise SystemExit("ERROR: unchanged app tag does not match the latest pipeline execution; cutover is unverified") |
| 52 | + selected = executions[0] |
| 53 | +else: |
| 54 | + selected = executions[0] if executions and epoch(executions[0]) >= since else None |
| 55 | + if selected and not matches(selected): |
| 56 | + raise SystemExit("ERROR: latest pipeline execution does not match this app digest; deployment was superseded or its source is unverified") |
| 57 | +print(selected["pipelineExecutionId"] if selected else "") |
| 58 | +' |
| 59 | +} |
| 60 | + |
| 61 | +EXECUTION_ID='' |
| 62 | +while [ -z "$EXECUTION_ID" ]; do |
| 63 | + check_deadline 'the matching pipeline execution' |
| 64 | + EXECUTION_ID=$(find_execution) |
| 65 | + if [ -z "$EXECUTION_ID" ]; then |
| 66 | + log 'No matching execution since this push; waiting' |
| 67 | + sleep "$POLL_INTERVAL" |
| 68 | + fi |
| 69 | +done |
| 70 | +log "Matched pipeline execution: $EXECUTION_ID" |
| 71 | + |
| 72 | +DEPLOYMENT_ID='' |
| 73 | +while [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = 'None' ]; do |
| 74 | + check_deadline 'the CodeDeploy deployment (the Deploy stage may be queued)' |
| 75 | + status=$(aws_read codepipeline get-pipeline-execution \ |
| 76 | + --pipeline-name "$PIPELINE" --pipeline-execution-id "$EXECUTION_ID" \ |
| 77 | + --query 'pipelineExecution.status' --output text) |
| 78 | + case "$status" in |
| 79 | + Failed|Stopped|Stopping|Superseded|Cancelled) |
| 80 | + log "ERROR: pipeline execution ended in $status; not promoting"; exit 1 ;; |
| 81 | + InProgress|Succeeded) ;; |
| 82 | + *) log "ERROR: unexpected pipeline status: $status"; exit 1 ;; |
| 83 | + esac |
| 84 | + # Action history does not publish the external deployment ID until cleanup |
| 85 | + # finishes. Live state exposes it while traffic is shifting. Correlate both |
| 86 | + # the stage execution and action attempt so old state cannot satisfy this run. |
| 87 | + deploy_state=$(aws_read codepipeline get-pipeline-state --name "$PIPELINE" \ |
| 88 | + --query "stageStates[?stageName=='Deploy'] | [0]" --output json) |
| 89 | + deploy_actions=$(aws_read codepipeline list-action-executions \ |
| 90 | + --pipeline-name "$PIPELINE" --filter pipelineExecutionId="$EXECUTION_ID" \ |
| 91 | + --query "actionExecutionDetails[?stageName=='Deploy']" --output json) |
| 92 | + DEPLOYMENT_ID=$(printf '%s\n' "$deploy_actions" | DEPLOY_STATE="$deploy_state" EXECUTION_ID="$EXECUTION_ID" python3 -c ' |
| 93 | +import json, os, re, sys |
| 94 | +state = json.loads(os.environ["DEPLOY_STATE"]) |
| 95 | +actions = json.load(sys.stdin) |
| 96 | +if not state or state.get("latestExecution", {}).get("pipelineExecutionId") != os.environ["EXECUTION_ID"] or not actions: |
| 97 | + print("") |
| 98 | + sys.exit(0) |
| 99 | +if len({a["actionName"] for a in actions}) != 1: |
| 100 | + raise SystemExit("ERROR: expected one Deploy action in the app pipeline") |
| 101 | +latest = max(actions, key=lambda a: a["startTime"]) |
| 102 | +matches = [a["latestExecution"] for a in state.get("actionStates", []) |
| 103 | + if a["actionName"] == latest["actionName"] |
| 104 | + and a.get("latestExecution", {}).get("actionExecutionId") == latest["actionExecutionId"]] |
| 105 | +if len(matches) > 1: |
| 106 | + raise SystemExit("ERROR: ambiguous live Deploy action") |
| 107 | +if not matches: |
| 108 | + print("") |
| 109 | + sys.exit(0) |
| 110 | +if latest["status"] not in ("InProgress", "Succeeded"): |
| 111 | + raise SystemExit("ERROR: Deploy action ended in " + latest["status"]) |
| 112 | +deployment_id = matches[0].get("externalExecutionId", "") |
| 113 | +if deployment_id and not re.fullmatch(r"d-[A-Za-z0-9]+", deployment_id): |
| 114 | + raise SystemExit("ERROR: invalid CodeDeploy deployment ID in pipeline state") |
| 115 | +print(deployment_id) |
| 116 | +') |
| 117 | + if [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = 'None' ]; then |
| 118 | + if [ "$status" = 'Succeeded' ]; then |
| 119 | + log 'ERROR: successful pipeline has no CodeDeploy deployment'; exit 1 |
| 120 | + fi |
| 121 | + sleep "$POLL_INTERVAL" |
| 122 | + fi |
| 123 | +done |
| 124 | +log "CodeDeploy deployment: $DEPLOYMENT_ID" |
| 125 | + |
| 126 | +while true; do |
| 127 | + check_deadline 'AllowTraffic on every ECS target' |
| 128 | + status=$(aws_read deploy get-deployment --deployment-id "$DEPLOYMENT_ID" \ |
| 129 | + --query 'deploymentInfo.status' --output text) |
| 130 | + case "$status" in |
| 131 | + Failed|Stopped) log "ERROR: deployment ended in $status; not promoting"; exit 1 ;; |
| 132 | + Created|Queued|InProgress|Baking|Ready|Succeeded) ;; |
| 133 | + *) log "ERROR: unexpected deployment status: $status"; exit 1 ;; |
| 134 | + esac |
| 135 | + target_ids=$(aws_read deploy list-deployment-targets --deployment-id "$DEPLOYMENT_ID" \ |
| 136 | + --query 'targetIds' --output text) |
| 137 | + if [ -n "$target_ids" ] && [ "$target_ids" != 'None' ]; then |
| 138 | + all_ok=1 |
| 139 | + for target in $target_ids; do |
| 140 | + cutover=$(aws_read deploy get-deployment-target --deployment-id "$DEPLOYMENT_ID" --target-id "$target" \ |
| 141 | + --query "deploymentTarget.ecsTarget.lifecycleEvents[?lifecycleEventName=='AllowTraffic'].status | [0]" \ |
| 142 | + --output text) |
| 143 | + case "$cutover" in |
| 144 | + Succeeded) ;; |
| 145 | + Failed|Skipped|Unknown) log "ERROR: target $target cutover status $cutover"; exit 1 ;; |
| 146 | + Pending|InProgress|None|'') all_ok=0 ;; |
| 147 | + *) log "ERROR: unexpected cutover status: $cutover"; exit 1 ;; |
| 148 | + esac |
| 149 | + done |
| 150 | + if [ "$all_ok" = 1 ]; then |
| 151 | + LATEST_EXECUTION_ID=$(find_execution) |
| 152 | + if [ "$LATEST_EXECUTION_ID" != "$EXECUTION_ID" ]; then |
| 153 | + log 'ERROR: a newer pipeline execution appeared during cutover; not promoting' |
| 154 | + exit 1 |
| 155 | + fi |
| 156 | + log 'Traffic cutover complete on every ECS target' |
| 157 | + exit 0 |
| 158 | + fi |
| 159 | + fi |
| 160 | + log 'Traffic cutover is not complete; waiting' |
| 161 | + sleep "$POLL_INTERVAL" |
| 162 | +done |
0 commit comments