Coverage for cli / commands / release_cmd.py: 100.00%
109 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-09-14 22:07 +0000
1"""Release lifecycle commands.
3``gco release validate`` wraps the live release validation harness
4(``scripts/live_release_validation``) so an operator runs one command
5instead of exporting six environment variables and assembling a module
6invocation by hand. The wrapper derives everything derivable — commit SHA,
7branch, run id, report directory — and reserves flags for the things a
8human must consciously assert:
10* which account the run may touch (``--expected-account``); and
11* that they understand it deploys and destroys paid infrastructure
12 (``--i-understand-this-deploys-and-destroys-infrastructure``, plus
13 ``--confirm-kms-key-deletion`` whenever the deploy action is selected).
15There are deliberately NO interactive prompts: presence of the flags is the
16consent, which makes the command automatable while keeping accidental
17invocation implausible. The harness itself re-verifies every identity claim
18(account, SHA, branch, clean worktree) before acting, so this wrapper adds
19convenience on top of those guarantees rather than replacing them.
21``--emulator-endpoint`` runs the identical harness against a local AWS
22emulator (Floci) for CI rehearsal; the harness proves the endpoint is an
23emulator before touching anything (see
24``scripts/live_release_validation/emulator.py`` and docs/FLOCI_TESTING.md).
25"""
27from __future__ import annotations
29import os
30import re
31import subprocess
32import sys
33from datetime import UTC, datetime
34from pathlib import Path
35from typing import NoReturn
37import click
39from .._image_reference import immutable_sha256_digest
41_ACCOUNT_RE = re.compile(r"\d{12}")
43#: The consent flag's exact name, referenced from error messages and docs.
44CONSENT_FLAG = "--i-understand-this-deploys-and-destroys-infrastructure"
47def _fail(message: str) -> NoReturn:
48 raise click.ClickException(message)
51def _run_git(repo_root: Path | None, *args: str) -> str:
52 result = subprocess.run(
53 ["git", *args],
54 cwd=repo_root,
55 capture_output=True,
56 text=True,
57 check=False,
58 )
59 if result.returncode != 0:
60 _fail(f"git {' '.join(args)} failed: {result.stderr.strip()}")
61 return result.stdout.strip()
64def _repo_root() -> Path:
65 root = Path(_run_git(None, "rev-parse", "--show-toplevel"))
66 if not (root / "cdk.json").is_file() or not (root / "scripts").is_dir():
67 _fail(
68 f"{root} is not a GCO checkout (cdk.json or scripts/ missing); "
69 "run from inside the repository"
70 )
71 return root
74@click.group()
75def release() -> None:
76 """Release validation lifecycle."""
79@release.command("validate")
80@click.option(
81 "--expected-account",
82 required=True,
83 metavar="ACCOUNT_ID",
84 help="Exact 12-digit AWS account id this run is allowed to touch.",
85)
86@click.option(
87 CONSENT_FLAG,
88 "authorized",
89 is_flag=True,
90 default=False,
91 help=(
92 "Required consent: the run deploys real, paid infrastructure into the "
93 "expected account and destroys it afterwards. No prompt will ask again."
94 ),
95)
96@click.option(
97 "--confirm-kms-key-deletion",
98 is_flag=True,
99 default=False,
100 help=(
101 "Authorize scheduling this run's retained EKS KMS keys for their 7-day "
102 "deletion window during cleanup. Required whenever the deploy action runs."
103 ),
104)
105@click.option(
106 "--actions",
107 default="all",
108 show_default=True,
109 metavar="NAME[,NAME...]",
110 help="Harness actions to run; dependencies are added automatically.",
111)
112@click.option("--inference-region", default=None, help="Region for the inference matrix.")
113@click.option("--inference-vllm-image", default=None, help="Immutable vLLM @sha256 image.")
114@click.option("--inference-vllm-model-id", default=None, help="Exact vLLM model identifier.")
115@click.option(
116 "--inference-vllm-model-revision",
117 default=None,
118 help="Full immutable 40-hex vLLM model commit.",
119)
120@click.option("--inference-tgi-image", default=None, help="Immutable TGI @sha256 image.")
121@click.option("--inference-tgi-model-id", default=None, help="Exact TGI model identifier.")
122@click.option(
123 "--inference-tgi-model-revision",
124 default=None,
125 help="Full immutable 40-hex TGI model commit.",
126)
127@click.option("--inference-gpu-count", type=click.IntRange(min=0), default=0, show_default=True)
128@click.option(
129 "--optional-schedulers",
130 "optional_schedulers",
131 default=None,
132 metavar="NAME[,NAME...]",
133 help=(
134 "Force-enable off-by-default schedulers (yunikorn, slurm, or all) for "
135 "this run's deploy so the schedulers action proves them too."
136 ),
137)
138@click.option(
139 "--profile",
140 type=click.Choice(["configured", "single-region", "multi-region"]),
141 default="configured",
142 show_default=True,
143 help="Topology profile to validate against cdk.json (never rewritten).",
144)
145@click.option("--run-id", default=None, help="Stable run id (default: UTC timestamp + SHA).")
146@click.option(
147 "--report-dir",
148 default=None,
149 type=click.Path(path_type=Path),
150 help="Report directory (default: ~/gco-live-release-validation-reports/<run-id>).",
151)
152@click.option(
153 "--resume",
154 is_flag=True,
155 default=False,
156 help="Resume an interrupted run; requires the original --run-id and --report-dir.",
157)
158@click.option(
159 "--protected-stack",
160 multiple=True,
161 metavar="NAME",
162 help="Additional non-project CloudFormation stack to preserve exactly (repeatable).",
163)
164@click.option(
165 "--emulator-endpoint",
166 default=None,
167 metavar="URL",
168 help=(
169 "Run the identical harness against a local AWS emulator (Floci) instead of "
170 "real AWS. The harness verifies the endpoint is an emulator before acting."
171 ),
172)
173def release_validate(
174 expected_account: str,
175 authorized: bool,
176 confirm_kms_key_deletion: bool,
177 actions: str,
178 inference_region: str | None,
179 inference_vllm_image: str | None,
180 inference_vllm_model_id: str | None,
181 inference_vllm_model_revision: str | None,
182 inference_tgi_image: str | None,
183 inference_tgi_model_id: str | None,
184 inference_tgi_model_revision: str | None,
185 inference_gpu_count: int,
186 optional_schedulers: str | None,
187 profile: str,
188 run_id: str | None,
189 report_dir: Path | None,
190 resume: bool,
191 protected_stack: tuple[str, ...],
192 emulator_endpoint: str | None,
193) -> None:
194 """Run live release validation end to end without prompts.
196 Derives the expected commit SHA and branch from the current checkout,
197 generates a run id and a private report directory outside the worktree,
198 and executes ``python -m scripts.live_release_validation``. Exits with
199 the harness's exit code; reports land in the report directory.
200 """
201 if not _ACCOUNT_RE.fullmatch(expected_account):
202 _fail("--expected-account must be an exact 12-digit AWS account id")
203 if not authorized:
204 _fail(
205 "Refusing to run without explicit consent. Add "
206 f"{CONSENT_FLAG} to acknowledge that this deploys and destroys real "
207 "infrastructure in account " + expected_account + "."
208 )
209 selected = {name.strip() for name in actions.split(",") if name.strip()}
210 if not selected:
211 _fail("--actions must name at least one action")
212 inference_selected = bool(selected & {"all", "inference"})
213 # Every action other than preflight/baseline transitively depends on
214 # deploy, and the harness expands dependencies automatically — so any
215 # such selection deploys real infrastructure and creates retained EKS
216 # KMS keys, not just a literal `deploy`/`all`.
217 deploy_selected = bool(selected & {"all", "deploy"}) or bool(
218 selected - {"preflight", "baseline"}
219 )
220 if deploy_selected and not confirm_kms_key_deletion:
221 _fail(
222 "The selected actions imply the deploy action, which creates retained "
223 "EKS KMS keys; add --confirm-kms-key-deletion to authorize scheduling "
224 "exactly this run's keys for deletion during cleanup."
225 )
226 if resume and (run_id is None or report_dir is None):
227 _fail(
228 "--resume replays an exact checkpoint identity: pass the original "
229 "--run-id and --report-dir from the interrupted run."
230 )
231 if inference_selected:
232 required_inference = {
233 "--inference-region": inference_region,
234 "--inference-vllm-image": inference_vllm_image,
235 "--inference-vllm-model-id": inference_vllm_model_id,
236 "--inference-vllm-model-revision": inference_vllm_model_revision,
237 "--inference-tgi-image": inference_tgi_image,
238 "--inference-tgi-model-id": inference_tgi_model_id,
239 "--inference-tgi-model-revision": inference_tgi_model_revision,
240 }
241 missing = [name for name, value in required_inference.items() if not value]
242 if missing:
243 _fail("The inference action requires " + ", ".join(missing) + ".")
244 image_digests: list[str] = []
245 for option, image in (
246 ("--inference-vllm-image", inference_vllm_image),
247 ("--inference-tgi-image", inference_tgi_image),
248 ):
249 digest = immutable_sha256_digest(image)
250 if digest is None:
251 _fail(f"{option} must be an immutable lowercase @sha256: reference")
252 image_digests.append(digest)
253 if len(set(image_digests)) != 2:
254 _fail("vLLM and TGI inference images must have distinct immutable digests")
255 for option, revision in (
256 ("--inference-vllm-model-revision", inference_vllm_model_revision),
257 ("--inference-tgi-model-revision", inference_tgi_model_revision),
258 ):
259 if revision is None or not re.fullmatch(r"[0-9a-f]{40}", revision):
260 _fail(f"{option} must be a full lowercase 40-hex commit")
262 repo_root = _repo_root()
263 expected_sha = _run_git(repo_root, "rev-parse", "HEAD")
264 expected_branch = _run_git(repo_root, "symbolic-ref", "--short", "HEAD")
265 resolved_run_id = run_id or (
266 datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + expected_sha[:12]
267 )
268 resolved_report_dir = report_dir or (
269 Path.home() / "gco-live-release-validation-reports" / resolved_run_id
270 )
272 env = dict(os.environ)
273 if emulator_endpoint:
274 normalized = emulator_endpoint.rstrip("/")
275 # The harness verifies these before acting; setting both here keeps a
276 # single flag sufficient and makes a split-endpoint run impossible.
277 env["GCO_LIVE_VALIDATION_EMULATOR"] = normalized
278 env["AWS_ENDPOINT_URL"] = normalized
280 command = [
281 sys.executable,
282 "-m",
283 "scripts.live_release_validation",
284 "--repo-root",
285 str(repo_root),
286 "--expected-account",
287 expected_account,
288 "--expected-sha",
289 expected_sha,
290 "--expected-branch",
291 expected_branch,
292 "--profile",
293 profile,
294 "--actions",
295 ",".join(sorted(selected)),
296 "--run-id",
297 resolved_run_id,
298 "--report-dir",
299 str(resolved_report_dir),
300 "--checkpoint",
301 str(resolved_report_dir / "checkpoint.json"),
302 ]
303 if inference_selected:
304 command.extend(
305 [
306 "--inference-region",
307 str(inference_region),
308 "--inference-vllm-image",
309 str(inference_vllm_image),
310 "--inference-vllm-model-id",
311 str(inference_vllm_model_id),
312 "--inference-vllm-model-revision",
313 str(inference_vllm_model_revision),
314 "--inference-tgi-image",
315 str(inference_tgi_image),
316 "--inference-tgi-model-id",
317 str(inference_tgi_model_id),
318 "--inference-tgi-model-revision",
319 str(inference_tgi_model_revision),
320 "--inference-gpu-count",
321 str(inference_gpu_count),
322 "--confirm-inference-deployment",
323 ]
324 )
325 if confirm_kms_key_deletion:
326 command.append("--confirm-kms-key-deletion")
327 if optional_schedulers:
328 command.extend(["--optional-schedulers", optional_schedulers])
329 if resume:
330 command.append("--resume")
331 for name in protected_stack:
332 command.extend(["--protected-stack", name])
334 click.echo(f"run-id: {resolved_run_id}")
335 click.echo(f"sha: {expected_sha}")
336 click.echo(f"branch: {expected_branch}")
337 click.echo(f"account: {expected_account}")
338 click.echo(f"actions: {','.join(sorted(selected))}")
339 if optional_schedulers:
340 click.echo(f"schedulers: {optional_schedulers} (force-enabled for this run)")
341 click.echo(f"report-dir: {resolved_report_dir}")
342 if emulator_endpoint:
343 click.echo(f"emulator: {emulator_endpoint} (verified by the harness before use)")
345 # Stream harness output directly; operators watch progress live and the
346 # harness owns its own reporting/cleanup guarantees.
347 result = subprocess.run(command, cwd=repo_root, env=env, check=False)
348 sys.exit(result.returncode)