← all scripts

.github/scripts/autopilot_codex_boot_probe.sh

126 of 126 statements covered (100.00%).

coveredmissednever traced by Bash (not counted)A line ending in … continues the statement above it and shares its fate.

113#!/usr/bin/env bash
2# =============================================================================
3# autopilot_codex_boot_probe.sh — boot the real `gco autopilot` Codex session
4# =============================================================================
5#
6# Drives `gco autopilot --engine codex` end-to-end the way a first-time user
7# does, and verifies the session boots to the last point reachable without
8# real AWS credentials. Used by integration:autopilot:codex-boot
9# (integration-tests.yml). The Claude Code twin lives in
10# autopilot_claude_code_boot_probe.sh; the phases are parallel on purpose so
11# the two probes stay comparable engine to engine.
12#
13# What runs for real (nothing about autopilot is mocked):
14#
15# 1. `gco autopilot --engine codex --print-config` resolves the session
16# plan from this checkout (the in-tree gco MCP server plus the curated
17# companion registry, rendered as Codex TOML).
18# 2. Every [mcp_servers.*] entry in the generated TOML is pre-warmed by
19# running its exact launch recipe (uvx/npx resolve, install, boot, exit
20# on stdin EOF). Warm caches keep the integrated boot inside Codex's
21# configured per-server startup timeout on cold runners.
22# 3. `gco autopilot --engine codex -y -- --version` exercises autopilot's
23# own install path: detect the missing binary, npm-install the pinned
24# release, re-detect it, write the isolated CODEX_HOME config, and exec
25# codex with the session-precedence overrides. The passthrough
26# `--version` makes that exec exit 0 deterministically.
27# 4. `gco autopilot --engine codex -- exec "..."` boots the full
28# non-interactive stack: codex loads the generated config, launches the
29# MCP servers, and dispatches to Amazon Bedrock Runtime's OpenAI-
30# compatible endpoint with the shipped default model. With the
31# fail-closed fake credentials exported below, SigV4 validation answers
32# 401 — proving a signed request left the wire. The probe asserts,
33# from codex's own RUST_LOG=info stderr:
34#
35# - mcp_server_count/mcp_servers="..." carrying EVERY planned server
36# - Service initialized as client (MCP handshakes under codex)
37# - model=<configured default> (the Bedrock dispatch)
38# - https://bedrock-runtime.<region>.amazonaws.com/openai/v1/responses
39# - Turn error: unexpected status 401 Unauthorized ... security token
40#
41# Codex retries the sampling request a bounded number of times and then
42# exits nonzero on its own, so the probe simply waits for it — no
43# background session management is needed.
44#
45# Engine delta vs the Claude probe, asserted honestly: Claude Code blocks on
46# every MCP connection before the first prompt, so its probe requires a
47# per-server "Successfully connected" line. Codex races MCP initialization
48# against the first turn, and this session ends at the credential boundary
49# after ~1 minute — slower servers may still be mid-launch when it exits.
50# Per-server *boot* proof therefore lives in the pre-warm phase (each launch
51# recipe must start), config fidelity is proven by the mcp_servers list the
52# session logs, and the MCP subsystem is proven live by requiring at least
53# one completed in-session initialize handshake.
54#
55# Marker stability: the debug markers above were captured from the pinned
56# Codex release (cli/autopilot.py CODEX_VERSION). A pin bump can rephrase
57# them; the failure output names the missing marker so the bump PR can
58# refresh this probe alongside the pin.
59#
60# Requirements: gco (this checkout, installed), node+npm (pinned via
61# .github/scripts/use-pinned-npm.sh), uv/uvx, python3, GNU coreutils
62# `timeout`. The `codex` binary must NOT be preinstalled — the probe exists
63# to prove autopilot's own install path works.
64#
65# =============================================================================
66
6713set -euo pipefail
68
6952REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
7013cd "$REPO_ROOT"
71
7213WORK_DIR="${RUNNER_TEMP:-$(mktemp -d)}/autopilot-codex-boot-probe"
7313mkdir -p "$WORK_DIR"
74
75# Autopilot writes the session config (and the isolated CODEX_HOME beneath
76# it) here instead of ~/.gco/autopilot.
7726export GCO_AUTOPILOT_CONFIG_DIR="${WORK_DIR}/config"
78
7913SESSION_LOG="${WORK_DIR}/session.log"
8013PREWARM_DIR="${WORK_DIR}/prewarm"
8113mkdir -p "$PREWARM_DIR"
82
83# How long the integrated session may take to reach the credential boundary.
84# Codex's own bounded retries finish in roughly a minute; the budget covers
85# slow MCP launches on a cold runner without masking a hang.
8613BOOT_TIMEOUT_SECONDS="${BOOT_TIMEOUT_SECONDS:-420}"
87
88# The one EXIT trap: gather evidence for the always-uploaded artifact. The
89# generated TOML and codex's own log directory both live under WORK_DIR
90# already (GCO_AUTOPILOT_CONFIG_DIR), so only the session log needs copying.
91collect_and_cleanup() {
9226 cp -f "$SESSION_LOG" "${WORK_DIR}/session.log" 2>/dev/null || true
93}
9413trap collect_and_cleanup EXIT
95
96fail() {
9710 echo "✗ $1" >&2
9810 exit 1
99}
100
101pass() {
10283 echo "✓ $1"
103}
104
105# ── Fail-closed credential environment ──────────────────────────────────────
106# The probe must never reach Bedrock with usable credentials, even if the
107# surrounding job one day exports some. A syntactically valid but fabricated
108# static key pair wins the SDK provider chain ahead of every file/role
109# source, and the file/IMDS sources are disabled outright. The key id is
110# assembled at runtime so repository secret scanners (gitleaks, trufflehog)
111# never see a contiguous AKIA-shaped literal in the tree.
11226AWS_ACCESS_KEY_ID="$(printf 'AKIA%s' '00000000000fake0')"
11326AWS_SECRET_ACCESS_KEY="$(printf '%040d' 0)"
11413export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
11526export AWS_SHARED_CREDENTIALS_FILE=/dev/null
11626export AWS_CONFIG_FILE=/dev/null
11726export AWS_EC2_METADATA_DISABLED=true
11813unset AWS_SESSION_TOKEN AWS_PROFILE AWS_ROLE_ARN AWS_WEB_IDENTITY_TOKEN_FILE 2>/dev/null || true
119
120# ── Preflight ────────────────────────────────────────────────────────────────
121
12262for tool in gco npm uvx python3 timeout; do
12363 command -v "$tool" >/dev/null || fail "required tool missing: $tool"
124done
125
12612if command -v codex >/dev/null; then
1272 fail "codex is already installed at $(command -v codex) — this probe must exercise autopilot's own install path"
128fi
129
130# Facts come from the shared autopilot CI contract — the same single source
131# unit:cli:autopilot, the dev-container step, and the Claude probe assert
132# against.
13311CONTRACT=".github/scripts/autopilot_ci_contract.py"
13422CODEX_PIN="$(python3 "$CONTRACT" pin --engine codex)"
13522EXPECTED_MODEL="$(python3 "$CONTRACT" default-model --engine codex)"
13611pass "preflight OK (pin ${CODEX_PIN}, default model ${EXPECTED_MODEL})"
137
138# ── Phase 1: resolve the session plan from this checkout ────────────────────
139
14011GENERATED_CONFIG="${WORK_DIR}/print-config.toml"
14111gco autopilot --engine codex --print-config > "$GENERATED_CONFIG"
142
143# Full structural validation from the shared contract (model/provider/wire
144# API lines, exact expected server set, entry shapes), then load the
145# expected names for the pre-warm and session assertions below.
14611python3 "$CONTRACT" verify-codex-config "$GENERATED_CONFIG" \
1471 || fail "generated Codex config failed the shared autopilot CI contract"
14820mapfile -t SERVER_NAMES < <(python3 "$CONTRACT" expected-servers)
14910[ "${#SERVER_NAMES[@]}" -ge 2 ] || fail "contract lists ${#SERVER_NAMES[@]} servers; expected the gco server plus companions"
15010pass "session plan resolves: ${#SERVER_NAMES[@]} MCP servers (${SERVER_NAMES[*]})"
151
152# ── Phase 2: pre-warm every server's exact launch recipe ────────────────────
153# Each companion is launched exactly as the generated TOML specifies and
154# handed EOF on stdin, which a stdio MCP server treats as client disconnect.
155# This resolves and installs every uvx/npx package (an independent
156# per-package install check with a pinpointed log on failure) and warms the
157# caches so the integrated boot below is not racing package managers against
158# Codex's per-server startup timeout.
159
16020mapfile -t PREWARM_CMDS < <(python3 - "$GENERATED_CONFIG" <<'PY'
161import shlex, sys, tomllib
162with open(sys.argv[1], "rb") as handle:
163 config = tomllib.load(handle)
164for name, entry in sorted(config["mcp_servers"].items()):
165 env_prefix = " ".join(
166 f"{key}={shlex.quote(str(value))}" for key, value in entry.get("env", {}).items()
167 )
168 command = " ".join(shlex.quote(str(part)) for part in [entry["command"], *entry.get("args", [])])
169 print(f"{name}\t{env_prefix} {command}".replace("\t ", "\t", 1))
170PY
171)
17220
17310PREWARM_FAILURES=0
17440for line in "${PREWARM_CMDS[@]}"; do
17540 name="${line%%$'\t'*}"
17640 launch="${line#*$'\t'}"
17740 rc=0
17843 timeout 240 bash -c "$launch" </dev/null >"${PREWARM_DIR}/${name}.log" 2>&1 || rc=$?
17940 case "$rc" in
18020 0)
18137 pass "pre-warm ${name}: launched and exited on stdin EOF" ;;
18220 124)
18320 # Ran the full 240s before timeout killed it: the package
18420 # resolved, installed, and booted (cache warmed) — it just
18520 # doesn't exit on EOF. The in-session behavior happens later
18620 # under codex, where connection management is codex's job.
1871 pass "pre-warm ${name}: launched and ran until the warm-up timeout" ;;
18820 125 | 126 | 127)
18920 # timeout itself failed / command not executable / not found:
19020 # the launch recipe is broken.
1911 echo "── ${PREWARM_DIR}/${name}.log ──"
1921 cat "${PREWARM_DIR}/${name}.log" || true
1931 echo "✗ pre-warm ${name}: launch recipe failed (exit ${rc}): ${launch}" >&2
1941 PREWARM_FAILURES=$((PREWARM_FAILURES + 1)) ;;
19520 *)
19620 # Any other non-zero exit on EOF is server-specific and fine;
19720 # the process launched, which is all warming needs.
1981 pass "pre-warm ${name}: launched and exited on stdin EOF (rc ${rc})" ;;
19920 esac
20020done
20111[ "$PREWARM_FAILURES" -eq 0 ] || fail "${PREWARM_FAILURES} companion launch recipe(s) failed to start at all"
20220
20320# ── Phase 3: autopilot's own install path, exec verified by --version ───────
20420# codex is absent, so `-y` makes autopilot npm-install the exact pin,
20520# re-detect the binary, write the isolated CODEX_HOME config, and exec it
20620# with the session-precedence argv. `--version` in the passthrough position
20720# makes that real exec terminate deterministically.
20820
20927VERSION_OUTPUT="$(gco autopilot --engine codex -y -- --version 2>&1 | tee "${WORK_DIR}/version-probe.log")"
21018echo "$VERSION_OUTPUT" | grep -qF "$CODEX_PIN" \
2111 || fail "autopilot exec'd codex, but its --version output does not carry the pin ${CODEX_PIN}: ${VERSION_OUTPUT}"
2129command -v codex >/dev/null || fail "autopilot reported an install but codex is not on PATH"
2137pass "autopilot installed the pin and exec'd codex ${CODEX_PIN} with the session-precedence argv"
21420
2157WRITTEN_CONFIG="${GCO_AUTOPILOT_CONFIG_DIR}/codex/config.toml"
2168[ -f "$WRITTEN_CONFIG" ] || fail "autopilot did not write the isolated Codex config to ${WRITTEN_CONFIG}"
21720python3 - "$GENERATED_CONFIG" "$WRITTEN_CONFIG" <<'PY'
218import sys, tomllib
219def servers(path):
220 with open(path, "rb") as handle:
221 return set(tomllib.load(handle)["mcp_servers"])
222planned, written = servers(sys.argv[1]), servers(sys.argv[2])
223assert planned == written, f"planned {sorted(planned)} != written {sorted(written)}"
224PY
2255pass "written CODEX_HOME config matches the printed plan (${WRITTEN_CONFIG})"
226
227# ── Phase 4: full session boot, stopped at the credential boundary ──────────
228# RUST_LOG=info surfaces codex's tracing on stderr: the session-configured
229# event (mcp_server_count / mcp_servers list), each MCP initialize handshake,
230# the Bedrock Runtime dispatch spans, and the terminal 401. `codex exec`
231# retries the rejected sampling request a bounded number of times and then
232# exits nonzero by itself, so this run is awaited in the foreground; the
233# expected exit is nonzero and asserted as such.
234
2355echo "booting the full session (budget ${BOOT_TIMEOUT_SECONDS}s): gco autopilot --engine codex -- exec ..."
2365SESSION_RC=0
23710RUST_LOG=info timeout "$BOOT_TIMEOUT_SECONDS" \
238 gco autopilot --engine codex -- exec "Reply with the single word OK." \
2394 </dev/null >"$SESSION_LOG" 2>&1 || SESSION_RC=$?
240
2415if [ "$SESSION_RC" -eq 124 ]; then
2421 echo "── session stdout/stderr (${SESSION_LOG}) ──"
2431 tail -50 "$SESSION_LOG" || true
2441 fail "session still running after ${BOOT_TIMEOUT_SECONDS}s — it never reached codex's own bounded-retry exit"
245fi
2464[ "$SESSION_RC" -ne 0 ] \
2471 || fail "session exited 0 with fabricated credentials — the credential boundary was never enforced"
2483pass "session ran to codex's own bounded-retry exit (rc ${SESSION_RC})"
249
2506LOGS="$(cat "$SESSION_LOG")"
251
2523MISSING=""
25312for name in "${SERVER_NAMES[@]}"; do
25412 grep -qE "mcp_servers=\"[^\"]*\b${name}\b" <<<"$LOGS" \
2553 || MISSING+="session-config:${name} "
2563done
2573grep -qF "Service initialized as client" <<<"$LOGS" \
2581 || MISSING+="mcp-initialize-handshake "
2593grep -qF "model=${EXPECTED_MODEL}" <<<"$LOGS" \
2601 || MISSING+="bedrock-dispatch:${EXPECTED_MODEL} "
2613grep -qE 'https://bedrock-runtime\.[a-z0-9-]+\.amazonaws\.com/openai/v1/responses' <<<"$LOGS" \
2621 || MISSING+="bedrock-runtime-endpoint "
2633grep -qE 'Turn error: unexpected status 40[13] .*security token' <<<"$LOGS" \
2641 || MISSING+="credential-boundary-401 "
2651
2663if [ -n "$MISSING" ]; then
2671 echo "── session stdout/stderr (${SESSION_LOG}) ──"
2681 tail -80 "$SESSION_LOG" || true
2691 fail "session did not produce these boot markers: ${MISSING}"
270fi
271
2722pass "codex loaded the generated plan: all ${#SERVER_NAMES[@]} servers in the session's mcp_servers list"
2732pass "MCP subsystem live under codex (initialize handshake observed)"
2742pass "codex dispatched to Bedrock Runtime with the shipped default model (${EXPECTED_MODEL})"
2752pass "AWS rejected the fabricated credentials — the exact credential boundary"
276
277# In-session handshake evidence for the job summary (informational: codex
278# races MCP init against the turn, so the set observed before the bounded
279# 401 exit varies run to run — the required minimum is asserted above).
2802echo ""
2812echo "MCP initialize report (servers' self-reported names):"
2822grep -oE 'Implementation \{ name: "[^"]+"' "$SESSION_LOG" \
2834 | sed 's/Implementation { name: / /' | sort -u || true
284
2852echo ""
2862echo "autopilot codex boot probe: PASS"