.github/scripts/autopilot_claude_code_boot_probe.sh134 of 134 statements covered (100.00%).
coveredmissednever traced by Bash (not counted)A line ending in … continues the statement above it and shares its fate.
| 1 | 14 | #!/usr/bin/env bash |
| 2 | # ============================================================================= | |
| 3 | # autopilot_claude_code_boot_probe.sh — boot the real `gco autopilot` Claude Code session | |
| 4 | # ============================================================================= | |
| 5 | # | |
| 6 | # Drives `gco autopilot` end-to-end the way a first-time user does, and | |
| 7 | # verifies the session boots to the last point reachable without real AWS | |
| 8 | # credentials. Used by integration:autopilot:claude-code-boot | |
| 9 | # (integration-tests.yml). The Codex twin lives in | |
| 10 | # autopilot_codex_boot_probe.sh; the phases are parallel on purpose so the | |
| 11 | # two probes stay comparable engine to engine. | |
| 12 | # | |
| 13 | # What runs for real (nothing about autopilot is mocked): | |
| 14 | # | |
| 15 | # 1. `gco autopilot --print-config` resolves the session plan from this | |
| 16 | # checkout (in-tree gco MCP server + the curated companion registry). | |
| 17 | # 2. Every server entry in the generated config is pre-warmed by running | |
| 18 | # its exact launch recipe (uvx/npx resolve, install, boot, exit on | |
| 19 | # stdin EOF). Warm caches keep the integrated boot inside Claude | |
| 20 | # Code's fixed 30s per-server MCP connection timeout on cold runners. | |
| 21 | # 3. `gco autopilot -y -- --version` exercises autopilot's own install | |
| 22 | # path: detect the missing binary, npm-install the pinned release, | |
| 23 | # re-detect it, write the session MCP config, and exec claude with | |
| 24 | # `--mcp-config <generated> --strict-mcp-config`. The passthrough | |
| 25 | # `--version` makes that exec exit 0 deterministically. | |
| 26 | # 4. `gco autopilot -- --debug -p "..."` boots the full interactive | |
| 27 | # stack in print mode: claude connects every MCP server in the | |
| 28 | # generated config and dispatches to Amazon Bedrock with the shipped | |
| 29 | # default model. With the fail-closed fake credentials exported | |
| 30 | # below, AWS answers 403 — proving a signed request left the wire. | |
| 31 | # The probe asserts, from Claude Code's own debug log: | |
| 32 | # | |
| 33 | # - MCP server "<name>": Successfully connected (for EVERY server) | |
| 34 | # - dispatching to bedrock model=<configured default> | |
| 35 | # - API error (attempt N/M): 403 | |
| 36 | # | |
| 37 | # and then terminates the session. The 403 is the success condition: | |
| 38 | # it is the exact credential boundary, the only part of the launch a | |
| 39 | # credential-less CI runner cannot cross. | |
| 40 | # | |
| 41 | # Marker stability: the three debug-log markers above were captured from | |
| 42 | # the pinned Claude Code release (cli/autopilot.py CLAUDE_CODE_VERSION). | |
| 43 | # A pin bump can rephrase them; the failure output names the missing | |
| 44 | # marker so the bump PR can refresh this probe alongside the pin. | |
| 45 | # | |
| 46 | # Requirements: gco (this checkout, installed), node+npm (pinned via | |
| 47 | # .github/scripts/use-pinned-npm.sh), uv/uvx, python3, GNU coreutils | |
| 48 | # `timeout`. The `claude` binary must NOT be preinstalled — the probe | |
| 49 | # exists to prove autopilot's own install path works. | |
| 50 | # | |
| 51 | # ============================================================================= | |
| 52 | ||
| 53 | 14 | set -euo pipefail |
| 54 | ||
| 55 | 56 | REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" |
| 56 | 14 | cd "$REPO_ROOT" |
| 57 | ||
| 58 | 14 | WORK_DIR="${RUNNER_TEMP:-$(mktemp -d)}/autopilot-claude-code-boot-probe" |
| 59 | 14 | mkdir -p "$WORK_DIR" |
| 60 | ||
| 61 | # Autopilot writes the session MCP config here instead of ~/.gco/autopilot. | |
| 62 | 28 | export GCO_AUTOPILOT_CONFIG_DIR="${WORK_DIR}/config" |
| 63 | ||
| 64 | # Where claude keeps its per-session debug logs (always written; --debug | |
| 65 | # additionally mirrors them to stderr). | |
| 66 | 14 | CLAUDE_DEBUG_DIR="${HOME}/.claude/debug" |
| 67 | ||
| 68 | 14 | SESSION_LOG="${WORK_DIR}/session.log" |
| 69 | 14 | PREWARM_DIR="${WORK_DIR}/prewarm" |
| 70 | 14 | mkdir -p "$PREWARM_DIR" |
| 71 | ||
| 72 | # How long the integrated session may take to show every boot marker. | |
| 73 | 14 | BOOT_TIMEOUT_SECONDS="${BOOT_TIMEOUT_SECONDS:-300}" |
| 74 | ||
| 75 | # The one EXIT trap: stop a still-running session and gather evidence | |
| 76 | # (claude's debug logs live under a hidden directory the artifact upload | |
| 77 | # would otherwise skip) into WORK_DIR for the always-uploaded artifact. | |
| 78 | 14 | SESSION_PID="" |
| 79 | collect_and_cleanup() { | |
| 80 | 14 | if [ -n "$SESSION_PID" ]; then |
| 81 | 9 | kill "$SESSION_PID" 2>/dev/null || true |
| 82 | 6 | wait "$SESSION_PID" 2>/dev/null || true |
| 83 | fi | |
| 84 | 14 | if compgen -G "${CLAUDE_DEBUG_DIR}/*.txt" >/dev/null 2>&1; then |
| 85 | 4 | mkdir -p "${WORK_DIR}/claude-debug" |
| 86 | 4 | cp "${CLAUDE_DEBUG_DIR}"/*.txt "${WORK_DIR}/claude-debug/" 2>/dev/null || true |
| 87 | fi | |
| 88 | } | |
| 89 | 14 | trap collect_and_cleanup EXIT |
| 90 | ||
| 91 | fail() { | |
| 92 | 9 | echo "✗ $1" >&2 |
| 93 | 9 | exit 1 |
| 94 | } | |
| 95 | ||
| 96 | pass() { | |
| 97 | 81 | echo "✓ $1" |
| 98 | } | |
| 99 | ||
| 100 | # ── Fail-closed credential environment ────────────────────────────────────── | |
| 101 | # The probe must never reach Bedrock with usable credentials, even if the | |
| 102 | # surrounding job one day exports some. A syntactically valid but fabricated | |
| 103 | # static key pair wins the SDK provider chain ahead of every file/role | |
| 104 | # source, and the file/IMDS sources are disabled outright. The key id is | |
| 105 | # assembled at runtime so repository secret scanners (gitleaks, trufflehog) | |
| 106 | # never see a contiguous AKIA-shaped literal in the tree. | |
| 107 | 28 | AWS_ACCESS_KEY_ID="$(printf 'AKIA%s' '00000000000fake0')" |
| 108 | 28 | AWS_SECRET_ACCESS_KEY="$(printf '%040d' 0)" |
| 109 | 14 | export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY |
| 110 | 28 | export AWS_SHARED_CREDENTIALS_FILE=/dev/null |
| 111 | 28 | export AWS_CONFIG_FILE=/dev/null |
| 112 | 28 | export AWS_EC2_METADATA_DISABLED=true |
| 113 | 14 | unset AWS_SESSION_TOKEN AWS_PROFILE AWS_ROLE_ARN AWS_WEB_IDENTITY_TOKEN_FILE 2>/dev/null || true |
| 114 | ||
| 115 | # ── Preflight ──────────────────────────────────────────────────────────────── | |
| 116 | ||
| 117 | 68 | for tool in gco npm uvx python3 timeout; do |
| 118 | 69 | command -v "$tool" >/dev/null || fail "required tool missing: $tool" |
| 119 | done | |
| 120 | ||
| 121 | 13 | if command -v claude >/dev/null; then |
| 122 | 2 | fail "claude is already installed at $(command -v claude) — this probe must exercise autopilot's own install path" |
| 123 | fi | |
| 124 | ||
| 125 | # Facts come from the shared autopilot CI contract — the same single | |
| 126 | # source unit:cli:autopilot and the dev-container step assert against. | |
| 127 | 12 | CONTRACT=".github/scripts/autopilot_ci_contract.py" |
| 128 | 24 | CLAUDE_PIN="$(python3 "$CONTRACT" pin)" |
| 129 | 24 | EXPECTED_MODEL="$(python3 "$CONTRACT" default-model)" |
| 130 | 12 | pass "preflight OK (pin ${CLAUDE_PIN}, default model ${EXPECTED_MODEL})" |
| 131 | ||
| 132 | # ── Phase 1: resolve the session plan from this checkout ──────────────────── | |
| 133 | ||
| 134 | 12 | GENERATED_CONFIG="${WORK_DIR}/print-config.json" |
| 135 | 12 | gco autopilot --print-config > "$GENERATED_CONFIG" |
| 136 | ||
| 137 | # Full structural validation from the shared contract (exact expected | |
| 138 | # server set, entry shapes, pruned-package bans), then load the expected | |
| 139 | # names for the per-server pre-warm and handshake assertions below. | |
| 140 | 11 | python3 "$CONTRACT" verify-config "$GENERATED_CONFIG" \ |
| 141 | 1 | || fail "generated config failed the shared autopilot CI contract" |
| 142 | 20 | mapfile -t SERVER_NAMES < <(python3 "$CONTRACT" expected-servers) |
| 143 | 10 | [ "${#SERVER_NAMES[@]}" -ge 2 ] || fail "contract lists ${#SERVER_NAMES[@]} servers; expected the gco server plus companions" |
| 144 | 10 | pass "session plan resolves: ${#SERVER_NAMES[@]} MCP servers (${SERVER_NAMES[*]})" |
| 145 | ||
| 146 | # ── Phase 2: pre-warm every server's exact launch recipe ──────────────────── | |
| 147 | # Each companion is launched exactly as the generated config specifies and | |
| 148 | # handed EOF on stdin, which a stdio MCP server treats as client | |
| 149 | # disconnect. This resolves and installs every uvx/npx package (an | |
| 150 | # independent per-package install check with a pinpointed log on failure) | |
| 151 | # and warms the caches so the integrated boot below is not racing package | |
| 152 | # managers against claude's 30s per-server connection timeout. | |
| 153 | ||
| 154 | 20 | mapfile -t PREWARM_CMDS < <(python3 - "$GENERATED_CONFIG" <<'PY' |
| 155 | import json, shlex, sys | |
| 156 | with open(sys.argv[1]) as handle: | |
| 157 | config = json.load(handle) | |
| 158 | for name, entry in sorted(config["mcpServers"].items()): | |
| 159 | env_prefix = " ".join( | |
| 160 | f"{key}={shlex.quote(str(value))}" for key, value in entry.get("env", {}).items() | |
| 161 | ) | |
| 162 | command = " ".join(shlex.quote(str(part)) for part in [entry["command"], *entry["args"]]) | |
| 163 | print(f"{name}\t{env_prefix} {command}".replace("\t ", "\t", 1)) | |
| 164 | PY | |
| 165 | ) | |
| 166 | 20 | |
| 167 | 10 | PREWARM_FAILURES=0 |
| 168 | 40 | for line in "${PREWARM_CMDS[@]}"; do |
| 169 | 40 | name="${line%%$'\t'*}" |
| 170 | 40 | launch="${line#*$'\t'}" |
| 171 | 40 | rc=0 |
| 172 | 44 | timeout 240 bash -c "$launch" </dev/null >"${PREWARM_DIR}/${name}.log" 2>&1 || rc=$? |
| 173 | 40 | case "$rc" in |
| 174 | 20 | 0) |
| 175 | 36 | pass "pre-warm ${name}: launched and exited on stdin EOF" ;; |
| 176 | 20 | 124) |
| 177 | 20 | # Ran the full 240s before timeout killed it: the package |
| 178 | 20 | # resolved, installed, and booted (cache warmed) — it just |
| 179 | 20 | # doesn't exit on EOF. The handshake assertion happens later |
| 180 | 20 | # under claude, where connection management is claude's job. |
| 181 | 1 | pass "pre-warm ${name}: launched and ran until the warm-up timeout" ;; |
| 182 | 20 | 125 | 126 | 127) |
| 183 | 20 | # timeout itself failed / command not executable / not found: |
| 184 | 20 | # the launch recipe is broken. |
| 185 | 2 | echo "── ${PREWARM_DIR}/${name}.log ──" |
| 186 | 2 | cat "${PREWARM_DIR}/${name}.log" || true |
| 187 | 2 | echo "✗ pre-warm ${name}: launch recipe failed (exit ${rc}): ${launch}" >&2 |
| 188 | 2 | PREWARM_FAILURES=$((PREWARM_FAILURES + 1)) ;; |
| 189 | 20 | *) |
| 190 | 20 | # Any other non-zero exit on EOF is server-specific and fine; |
| 191 | 20 | # the process launched, which is all warming needs. |
| 192 | 1 | pass "pre-warm ${name}: launched and exited on stdin EOF (rc ${rc})" ;; |
| 193 | 20 | esac |
| 194 | 20 | done |
| 195 | 11 | [ "$PREWARM_FAILURES" -eq 0 ] || fail "${PREWARM_FAILURES} companion launch recipe(s) failed to start at all" |
| 196 | 20 | |
| 197 | 20 | # ── Phase 3: autopilot's own install path, exec verified by --version ─────── |
| 198 | 20 | # claude is absent, so `-y` makes autopilot npm-install the exact pin, |
| 199 | 20 | # re-detect the binary, write the session config, and exec it with the |
| 200 | 20 | # generated `--mcp-config`/`--strict-mcp-config` argv. `--version` in the |
| 201 | 20 | # passthrough position makes that real exec terminate deterministically. |
| 202 | 20 | |
| 203 | 27 | VERSION_OUTPUT="$(gco autopilot -y -- --version 2>&1 | tee "${WORK_DIR}/version-probe.log")" |
| 204 | 18 | echo "$VERSION_OUTPUT" | grep -qF "$CLAUDE_PIN" \ |
| 205 | 1 | || fail "autopilot exec'd claude, but its --version output does not carry the pin ${CLAUDE_PIN}: ${VERSION_OUTPUT}" |
| 206 | 9 | command -v claude >/dev/null || fail "autopilot reported an install but claude is not on PATH" |
| 207 | 7 | pass "autopilot installed the pin and exec'd claude ${CLAUDE_PIN} with the generated config argv" |
| 208 | 20 | |
| 209 | 7 | WRITTEN_CONFIG="${GCO_AUTOPILOT_CONFIG_DIR}/mcp.json" |
| 210 | 8 | [ -f "$WRITTEN_CONFIG" ] || fail "autopilot did not write the session config to ${WRITTEN_CONFIG}" |
| 211 | 20 | python3 - "$GENERATED_CONFIG" "$WRITTEN_CONFIG" <<'PY' |
| 212 | import json, sys | |
| 213 | with open(sys.argv[1]) as handle: | |
| 214 | planned = set(json.load(handle)["mcpServers"]) | |
| 215 | with open(sys.argv[2]) as handle: | |
| 216 | written = set(json.load(handle)["mcpServers"]) | |
| 217 | assert planned == written, f"planned {sorted(planned)} != written {sorted(written)}" | |
| 218 | PY | |
| 219 | 5 | pass "written session config matches the printed plan (${WRITTEN_CONFIG})" |
| 220 | ||
| 221 | # ── Phase 4: full session boot, stopped at the credential boundary ────────── | |
| 222 | ||
| 223 | 5 | echo "booting the full session (budget ${BOOT_TIMEOUT_SECONDS}s): gco autopilot -- --debug -p ..." |
| 224 | # `gco autopilot` execvpe()s claude, so SESSION_PID *is* the claude process | |
| 225 | # (reaped by the EXIT trap above). | |
| 226 | 5 | gco autopilot -- --debug -p "Reply with the single word OK." \ |
| 227 | </dev/null >"$SESSION_LOG" 2>&1 & | |
| 228 | 5 | SESSION_PID=$! |
| 229 | ||
| 230 | # Collect the required markers from claude's own debug logs. HOME is | |
| 231 | # job-fresh, so every log under CLAUDE_DEBUG_DIR belongs to this probe. | |
| 232 | missing_markers() { | |
| 233 | 155 | local logs="$1" |
| 234 | 155 | local missing="" |
| 235 | 155 | local name |
| 236 | 620 | for name in "${SERVER_NAMES[@]}"; do |
| 237 | 620 | grep -qF "MCP server \"${name}\": Successfully connected" <<<"$logs" \ |
| 238 | 604 | || missing+="mcp-connect:${name} " |
| 239 | 604 | done |
| 240 | 155 | grep -qF "dispatching to bedrock model=${EXPECTED_MODEL}" <<<"$logs" \ |
| 241 | 151 | || missing+="bedrock-dispatch:${EXPECTED_MODEL} " |
| 242 | 155 | grep -qE 'API error \(attempt [0-9]+/[0-9]+\): 403' <<<"$logs" \ |
| 243 | 152 | || missing+="credential-boundary-403 " |
| 244 | 155 | echo "$missing" |
| 245 | } | |
| 246 | ||
| 247 | 5 | DEADLINE=$((SECONDS + BOOT_TIMEOUT_SECONDS)) |
| 248 | 5 | MISSING="initial" |
| 249 | 152 | while [ "$SECONDS" -lt "$DEADLINE" ]; do |
| 250 | 453 | LOGS="$(cat "${CLAUDE_DEBUG_DIR}"/*.txt 2>/dev/null || true)" |
| 251 | 302 | MISSING="$(missing_markers "$LOGS")" |
| 252 | 151 | [ -z "$MISSING" ] && break |
| 253 | 151 | if ! kill -0 "$SESSION_PID" 2>/dev/null; then |
| 254 | # claude exited before every marker appeared (e.g. it gave up its | |
| 255 | # API retries) — take one final look at the logs it left behind. | |
| 256 | 8 | LOGS="$(cat "${CLAUDE_DEBUG_DIR}"/*.txt 2>/dev/null || true)" |
| 257 | 8 | MISSING="$(missing_markers "$LOGS")" |
| 258 | 4 | break |
| 259 | fi | |
| 260 | 147 | sleep 5 |
| 261 | done | |
| 262 | ||
| 263 | 5 | if [ -n "$MISSING" ]; then |
| 264 | 2 | echo "── session stdout/stderr (${SESSION_LOG}) ──" |
| 265 | 2 | tail -50 "$SESSION_LOG" || true |
| 266 | 2 | echo "── claude debug logs (${CLAUDE_DEBUG_DIR}) ──" |
| 267 | 3 | tail -100 "${CLAUDE_DEBUG_DIR}"/*.txt 2>/dev/null || echo "(no debug logs found)" |
| 268 | 2 | fail "session did not reach these boot markers within ${BOOT_TIMEOUT_SECONDS}s: ${MISSING}" |
| 269 | fi | |
| 270 | ||
| 271 | 3 | pass "all ${#SERVER_NAMES[@]} MCP servers completed the handshake under claude" |
| 272 | 3 | pass "claude dispatched to Bedrock with the shipped default model (${EXPECTED_MODEL})" |
| 273 | 3 | pass "AWS rejected the fabricated credentials with 403 — the exact credential boundary" |
| 274 | ||
| 275 | # One-line-per-server evidence for the job summary. | |
| 276 | 3 | echo "" |
| 277 | 3 | echo "MCP connection report:" |
| 278 | 3 | grep -hoE 'MCP server "[^"]+": Successfully connected \(transport: [a-z]+\) in [0-9]+ms' \ |
| 279 | 6 | "${CLAUDE_DEBUG_DIR}"/*.txt 2>/dev/null | sort -u | sed 's/^/ /' || true |
| 280 | ||
| 281 | 3 | echo "" |
| 282 | 3 | echo "autopilot boot probe: PASS" |