Coverage for cli/commands/release

1"""Release lifecycle commands.

3``gco release validate`` wraps the live release validation harness

4(``scripts/live_release_validation``) so an operator runs one command

5instead of exporting six environment variables and assembling a module

6invocation by hand. The wrapper derives everything derivable — commit SHA,

7branch, run id, report directory — and reserves flags for the things a

8human must consciously assert:

10* which account the run may touch (``--expected-account``); and

11* that they understand it deploys and destroys paid infrastructure

12 (``--i-understand-this-deploys-and-destroys-infrastructure``, plus

13 ``--confirm-kms-key-deletion`` whenever the deploy action is selected).

15There are deliberately NO interactive prompts: presence of the flags is the

16consent, which makes the command automatable while keeping accidental

17invocation implausible. The harness itself re-verifies every identity claim

18(account, SHA, branch, clean worktree) before acting, so this wrapper adds

19convenience on top of those guarantees rather than replacing them.

21``--emulator-endpoint`` runs the identical harness against a local AWS

22emulator (Floci) for CI rehearsal; the harness proves the endpoint is an

23emulator before touching anything (see

24``scripts/live_release_validation/emulator.py`` and docs/FLOCI_TESTING.md).

26``gco release validate-upgrade`` wraps the upgrade harness

27(``scripts/upgrade_validation``) the same way: it deploys the previous

28release with that release's own ``gco``, upgrades it to this checkout with

29``gco upgrade``, verifies the result, and destroys it (see

30docs/UPGRADE_VALIDATION.md).

31"""

33from __future__ import annotations

35import os

36import re

37import subprocess

38import sys

39from datetime import UTC, datetime

40from pathlib import Path

41from typing import NoReturn

43import click

45from gco.eks_capabilities_config import EKS_CAPABILITY_TYPES

47from .._image_reference import immutable_sha256_digest

49_ACCOUNT_RE = re.compile(r"\d{12}")

51#: The consent flag's exact name, referenced from error messages and docs.

52CONSENT_FLAG = "--i-understand-this-deploys-and-destroys-infrastructure"

55def _fail(message: str) -> NoReturn:

56 raise click.ClickException(message)

59def _run_git(repo_root: Path | None, *args: str) -> str:

60 result = subprocess.run(

61 ["git", *args],

62 cwd=repo_root,

63 capture_output=True,

64 text=True,

65 check=False,

66 )

67 if result.returncode != 0:

68 _fail(f"git {' '.join(args)} failed: {result.stderr.strip()}")

69 return result.stdout.strip()

72def _repo_root() -> Path:

73 root = Path(_run_git(None, "rev-parse", "--show-toplevel"))

74 if not (root / "cdk.json").is_file() or not (root / "scripts").is_dir():

75 _fail(

76 f"{root} is not a GCO checkout (cdk.json or scripts/ missing); "

77 "run from inside the repository"

78 )

79 return root

82@click.group()

83def release() -> None:

84 """Release validation lifecycle."""

87@release.command("validate")

88@click.option(

89 "--expected-account",

90 required=True,

91 metavar="ACCOUNT_ID",

92 help="Exact 12-digit AWS account id this run is allowed to touch.",

93)

94@click.option(

95 CONSENT_FLAG,

96 "authorized",

97 is_flag=True,

98 default=False,

99 help=(

100 "Required consent: the run deploys real, paid infrastructure into the "

101 "expected account and destroys it afterwards. No prompt will ask again."

102 ),

103)

104@click.option(

105 "--confirm-kms-key-deletion",

106 is_flag=True,

107 default=False,

108 help=(

109 "Authorize scheduling this run's retained EKS KMS keys for their 7-day "

110 "deletion window during cleanup. Required whenever the deploy action runs."

111 ),

112)

113@click.option(

114 "--actions",

115 default="all",

116 show_default=True,

117 metavar="NAME[,NAME...]",

118 help="Harness actions to run; dependencies are added automatically.",

119)

120@click.option("--inference-region", default=None, help="Region for the inference matrix.")

121@click.option("--inference-vllm-image", default=None, help="Immutable vLLM @sha256 image.")

122@click.option("--inference-vllm-model-id", default=None, help="Exact vLLM model identifier.")

123@click.option(

124 "--inference-vllm-model-revision",

125 default=None,

126 help="Full immutable 40-hex vLLM model commit.",

127)

128@click.option("--inference-sglang-image", default=None, help="Immutable SGLang @sha256 image.")

129@click.option("--inference-sglang-model-id", default=None, help="Exact SGLang model identifier.")

130@click.option(

131 "--inference-sglang-model-revision",

132 default=None,

133 help="Full immutable 40-hex SGLang model commit.",

134)

135@click.option(

136 "--inference-triton-image",

137 default=None,

138 help=(

139 "Immutable Triton Inference Server @sha256 image. Its model is the "

140 "harness-generated identity repository, uploaded to the model registry "

141 "for the run and removed afterwards."

142 ),

143)

144@click.option("--inference-gpu-count", type=click.IntRange(min=0), default=0, show_default=True)

145@click.option(

146 "--optional-schedulers",

147 "optional_schedulers",

148 default=None,

149 metavar="NAME[,NAME...]",

150 help=(

151 "Force-enable off-by-default schedulers (yunikorn, slurm, or all) for "

152 "this run's deploy so the schedulers action proves them too."

153 ),

154)

155@click.option(

156 "--eks-capabilities",

157 "eks_capabilities",

158 default=None,

159 metavar="NAME[,NAME...]",

160 help=(

161 "Enable off-by-default EKS Capabilities (ack, kro, or all) for this run's "

162 "deploy so the eks-capabilities action proves each one attaches ACTIVE."

163 ),

164)

165@click.option(

166 "--profile",

167 type=click.Choice(["configured", "single-region", "multi-region"]),

168 default="configured",

169 show_default=True,

170 help="Topology profile to validate against cdk.json (never rewritten).",

171)

172@click.option("--run-id", default=None, help="Stable run id (default: UTC timestamp + SHA).")

173@click.option(

174 "--report-dir",

175 default=None,

176 type=click.Path(path_type=Path),

177 help="Report directory (default: ~/gco-live-release-validation-reports/<run-id>).",

178)

179@click.option(

180 "--resume",

181 is_flag=True,

182 default=False,

183 help="Resume an interrupted run; requires the original --run-id and --report-dir.",

184)

185@click.option(

186 "--protected-stack",

187 multiple=True,

188 metavar="NAME",

189 help="Additional non-project CloudFormation stack to preserve exactly (repeatable).",

190)

191@click.option(

192 "--emulator-endpoint",

193 default=None,

194 metavar="URL",

195 help=(

196 "Run the identical harness against a local AWS emulator (Floci) instead of "

197 "real AWS. The harness verifies the endpoint is an emulator before acting."

198 ),

199)

200def release_validate(

201 expected_account: str,

202 authorized: bool,

203 confirm_kms_key_deletion: bool,

204 actions: str,

205 inference_region: str | None,

206 inference_vllm_image: str | None,

207 inference_vllm_model_id: str | None,

208 inference_vllm_model_revision: str | None,

209 inference_sglang_image: str | None,

210 inference_sglang_model_id: str | None,

211 inference_sglang_model_revision: str | None,

212 inference_triton_image: str | None,

213 inference_gpu_count: int,

214 optional_schedulers: str | None,

215 eks_capabilities: str | None,

216 profile: str,

217 run_id: str | None,

218 report_dir: Path | None,

219 resume: bool,

220 protected_stack: tuple[str, ...],

221 emulator_endpoint: str | None,

222) -> None:

223 """Run live release validation end to end without prompts.

224

225 Derives the expected commit SHA and branch from the current checkout,

226 generates a run id and a private report directory outside the worktree,

227 and executes ``python -m scripts.live_release_validation``. Exits with

228 the harness's exit code; reports land in the report directory.

229 """

230 if not _ACCOUNT_RE.fullmatch(expected_account):

231 _fail("--expected-account must be an exact 12-digit AWS account id")

232 if not authorized:

233 _fail(

234 "Refusing to run without explicit consent. Add "

235 f"{CONSENT_FLAG} to acknowledge that this deploys and destroys real "

236 "infrastructure in account " + expected_account + "."

237 )

238 selected = {name.strip() for name in actions.split(",") if name.strip()}

239 if not selected:

240 _fail("--actions must name at least one action")

241 inference_selected = bool(selected & {"all", "inference"})

242 # Every action other than preflight/baseline transitively depends on

243 # deploy, and the harness expands dependencies automatically — so any

244 # such selection deploys real infrastructure and creates retained EKS

245 # KMS keys, not just a literal `deploy`/`all`.

246 deploy_selected = bool(selected & {"all", "deploy"}) or bool(

247 selected - {"preflight", "baseline"}

248 )

249 if deploy_selected and not confirm_kms_key_deletion:

250 _fail(

251 "The selected actions imply the deploy action, which creates retained "

252 "EKS KMS keys; add --confirm-kms-key-deletion to authorize scheduling "

253 "exactly this run's keys for deletion during cleanup."

254 )

255 if resume and (run_id is None or report_dir is None):

256 _fail(

257 "--resume replays an exact checkpoint identity: pass the original "

258 "--run-id and --report-dir from the interrupted run."

259 )

260 capability_names = {

261 name.strip() for name in (eks_capabilities or "").split(",") if name.strip()

262 }

263 if eks_capabilities is not None and not capability_names:

264 _fail("--eks-capabilities must name at least one capability")

265 unknown_capabilities = sorted(capability_names - set(EKS_CAPABILITY_TYPES) - {"all"})

266 if unknown_capabilities:

267 _fail(

268 "--eks-capabilities accepts "

269 + ", ".join((*EKS_CAPABILITY_TYPES, "all"))

270 + "; got: "

271 + ", ".join(unknown_capabilities)

272 )

273 if "all" in capability_names and len(capability_names) != 1:

274 _fail("--eks-capabilities 'all' cannot be combined with individual names")

275 if inference_selected:

276 required_inference = {

277 "--inference-region": inference_region,

278 "--inference-vllm-image": inference_vllm_image,

279 "--inference-vllm-model-id": inference_vllm_model_id,

280 "--inference-vllm-model-revision": inference_vllm_model_revision,

281 "--inference-sglang-image": inference_sglang_image,

282 "--inference-sglang-model-id": inference_sglang_model_id,

283 "--inference-sglang-model-revision": inference_sglang_model_revision,

284 "--inference-triton-image": inference_triton_image,

285 }

286 missing = [name for name, value in required_inference.items() if not value]

287 if missing:

288 _fail("The inference action requires " + ", ".join(missing) + ".")

289 image_digests: list[str] = []

290 for option, image in (

291 ("--inference-vllm-image", inference_vllm_image),

292 ("--inference-sglang-image", inference_sglang_image),

293 ("--inference-triton-image", inference_triton_image),

294 ):

295 digest = immutable_sha256_digest(image)

296 if digest is None:

297 _fail(f"{option} must be an immutable lowercase @sha256: reference")

298 image_digests.append(digest)

299 if len(set(image_digests)) != 3:

300 _fail("vLLM, SGLang and Triton inference images must have distinct immutable digests")

301 for option, revision in (

302 ("--inference-vllm-model-revision", inference_vllm_model_revision),

303 ("--inference-sglang-model-revision", inference_sglang_model_revision),

304 ):

305 if revision is None or not re.fullmatch(r"[0-9a-f]{40}", revision):

306 _fail(f"{option} must be a full lowercase 40-hex commit")

307

308 repo_root = _repo_root()

309 expected_sha = _run_git(repo_root, "rev-parse", "HEAD")

310 expected_branch = _run_git(repo_root, "symbolic-ref", "--short", "HEAD")

311 resolved_run_id = run_id or (

312 datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + expected_sha[:12]

313 )

314 resolved_report_dir = report_dir or (

315 Path.home() / "gco-live-release-validation-reports" / resolved_run_id

316 )

317

318 env = dict(os.environ)

319 if emulator_endpoint:

320 normalized = emulator_endpoint.rstrip("/")

321 # The harness verifies these before acting; setting both here keeps a

322 # single flag sufficient and makes a split-endpoint run impossible.

323 env["GCO_LIVE_VALIDATION_EMULATOR"] = normalized

324 env["AWS_ENDPOINT_URL"] = normalized

325

326 command = [

327 sys.executable,

328 "-m",

329 "scripts.live_release_validation",

330 "--repo-root",

331 str(repo_root),

332 "--expected-account",

333 expected_account,

334 "--expected-sha",

335 expected_sha,

336 "--expected-branch",

337 expected_branch,

338 "--profile",

339 profile,

340 "--actions",

341 ",".join(sorted(selected)),

342 "--run-id",

343 resolved_run_id,

344 "--report-dir",

345 str(resolved_report_dir),

346 "--checkpoint",

347 str(resolved_report_dir / "checkpoint.json"),

348 ]

349 if inference_selected:

350 command.extend(

351 [

352 "--inference-region",

353 str(inference_region),

354 "--inference-vllm-image",

355 str(inference_vllm_image),

356 "--inference-vllm-model-id",

357 str(inference_vllm_model_id),

358 "--inference-vllm-model-revision",

359 str(inference_vllm_model_revision),

360 "--inference-sglang-image",

361 str(inference_sglang_image),

362 "--inference-sglang-model-id",

363 str(inference_sglang_model_id),

364 "--inference-sglang-model-revision",

365 str(inference_sglang_model_revision),

366 "--inference-triton-image",

367 str(inference_triton_image),

368 "--inference-gpu-count",

369 str(inference_gpu_count),

370 "--confirm-inference-deployment",

371 ]

372 )

373 if confirm_kms_key_deletion:

374 command.append("--confirm-kms-key-deletion")

375 if optional_schedulers:

376 command.extend(["--optional-schedulers", optional_schedulers])

377 if eks_capabilities:

378 command.extend(["--eks-capabilities", eks_capabilities])

379 if resume:

380 command.append("--resume")

381 for name in protected_stack:

382 command.extend(["--protected-stack", name])

383

384 click.echo(f"run-id: {resolved_run_id}")

385 click.echo(f"sha: {expected_sha}")

386 click.echo(f"branch: {expected_branch}")

387 click.echo(f"account: {expected_account}")

388 click.echo(f"actions: {','.join(sorted(selected))}")

389 if optional_schedulers:

390 click.echo(f"schedulers: {optional_schedulers} (force-enabled for this run)")

391 if eks_capabilities:

392 click.echo(f"capabilities: {eks_capabilities} (enabled for this run)")

393 click.echo(f"report-dir: {resolved_report_dir}")

394 if emulator_endpoint:

395 click.echo(f"emulator: {emulator_endpoint} (verified by the harness before use)")

396

397 # Stream harness output directly; operators watch progress live and the

398 # harness owns its own reporting/cleanup guarantees.

399 result = subprocess.run(command, cwd=repo_root, env=env, check=False)

400 sys.exit(result.returncode)

401

402

403@release.command("validate-upgrade")

404@click.option(

405 "--expected-account",

406 required=True,

407 metavar="ACCOUNT_ID",

408 help="Exact 12-digit AWS account id this run may touch.",

409)

410@click.option(

411 CONSENT_FLAG,

412 "authorized",

413 is_flag=True,

414 default=False,

415 help=(

416 "Required consent: deploys the previous release into the expected account, "

417 "upgrades it to this checkout, and destroys it afterwards."

418 ),

419)

420@click.option(

421 "--confirm-kms-key-deletion",

422 is_flag=True,

423 default=False,

424 help=(

425 "Authorize scheduling this run's retained EKS KMS keys (both the base and the "

426 "upgraded generation) for their 7-day deletion window during cleanup."

427 ),

428)

429@click.option(

430 "--base-ref",

431 default=None,

432 metavar="vX.Y.Z",

433 help="Release to deploy first and upgrade from (default: the newest earlier release).",

434)

435@click.option(

436 "--actions",

437 default="all",

438 show_default=True,

439 metavar="NAME[,NAME...]",

440 help="Harness actions to run; dependencies are added automatically.",

441)

442@click.option("--run-id", default=None, help="Stable run id (default: UTC timestamp + SHA).")

443@click.option(

444 "--report-dir",

445 default=None,

446 type=click.Path(path_type=Path),

447 help="Report directory (default: ~/gco-upgrade-validation-reports/<run-id>).",

448)

449@click.option(

450 "--resume",

451 is_flag=True,

452 default=False,

453 help="Resume an interrupted run; requires the original --run-id and --report-dir.",

454)

455@click.option(

456 "--protected-stack",

457 multiple=True,

458 metavar="NAME",

459 help="Additional non-project CloudFormation stack to preserve exactly (repeatable).",

460)

461def release_validate_upgrade(

462 expected_account: str,

463 authorized: bool,

464 confirm_kms_key_deletion: bool,

465 base_ref: str | None,

466 actions: str,

467 run_id: str | None,

468 report_dir: Path | None,

469 resume: bool,

470 protected_stack: tuple[str, ...],

471) -> None:

472 """Validate `gco upgrade` from the previous release to this checkout, live.

473

474 Deploys the base release from a private clone with that release's own

475 gco, runs its `gco upgrade` to the checked-out commit, verifies the

476 upgraded deployment and a sentinel it must preserve, then destroys

477 everything and verifies the account is back to its baseline. Executes

478 ``python -m scripts.upgrade_validation`` and exits with its code. See

479 docs/UPGRADE_VALIDATION.md.

480 """

481 if not _ACCOUNT_RE.fullmatch(expected_account):

482 _fail("--expected-account must be an exact 12-digit AWS account id")

483 if not authorized:

484 _fail(

485 "Refusing to run without explicit consent. Add "

486 f"{CONSENT_FLAG} to acknowledge that this deploys and destroys real "

487 "infrastructure in account " + expected_account + "."

488 )

489 selected = {name.strip() for name in actions.split(",") if name.strip()}

490 if not selected:

491 _fail("--actions must name at least one action")

492 # Only preflight, baseline, and prepare stop short of the deploy action.

493 deploy_selected = bool(selected & {"all", "deploy"}) or bool(

494 selected - {"preflight", "baseline", "prepare"}

495 )

496 if deploy_selected and not confirm_kms_key_deletion:

497 _fail(

498 "The selected actions imply the deploy action, which creates retained "

499 "EKS KMS keys; add --confirm-kms-key-deletion to authorize scheduling "

500 "exactly this run's keys for deletion during cleanup."

501 )

502 if resume and (run_id is None or report_dir is None):

503 _fail(

504 "--resume replays an exact checkpoint identity: pass the original "

505 "--run-id and --report-dir from the interrupted run."

506 )

507

508 repo_root = _repo_root()

509 expected_sha = _run_git(repo_root, "rev-parse", "HEAD")

510 expected_branch = _run_git(repo_root, "symbolic-ref", "--short", "HEAD")

511 resolved_run_id = run_id or (

512 datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + expected_sha[:12]

513 )

514 resolved_report_dir = report_dir or (

515 Path.home() / "gco-upgrade-validation-reports" / resolved_run_id

516 )

517

518 command = [

519 sys.executable,

520 "-m",

521 "scripts.upgrade_validation",

522 "--repo-root",

523 str(repo_root),

524 "--expected-account",

525 expected_account,

526 "--expected-sha",

527 expected_sha,

528 "--expected-branch",

529 expected_branch,

530 "--actions",

531 ",".join(sorted(selected)),

532 "--run-id",

533 resolved_run_id,

534 "--report-dir",

535 str(resolved_report_dir),

536 "--checkpoint",

537 str(resolved_report_dir / "checkpoint.json"),

538 ]

539 if base_ref:

540 command.extend(["--base-ref", base_ref])

541 if confirm_kms_key_deletion:

542 command.append("--confirm-kms-key-deletion")

543 if resume:

544 command.append("--resume")

545 for name in protected_stack:

546 command.extend(["--protected-stack", name])

547

548 click.echo(f"run-id: {resolved_run_id}")

549 click.echo(f"sha: {expected_sha}")

550 click.echo(f"branch: {expected_branch}")

551 click.echo(f"account: {expected_account}")

552 click.echo(f"base: {base_ref or 'the newest earlier release'}")

553 click.echo(f"actions: {','.join(sorted(selected))}")

554 click.echo(f"report-dir: {resolved_report_dir}")

555

556 result = subprocess.run(command, cwd=repo_root, env=dict(os.environ), check=False)

557 sys.exit(result.returncode)

Coverage for cli / commands / release_cmd.py: 100.00%

170 statements