Coverage for cli / commands / examples_cmd.py: 100.00%
73 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-09-14 22:07 +0000
1"""Example-manifest validation commands.
3``gco examples validate`` wraps the example-job validation harness
4(``scripts/example_job_validation``) exactly the way ``gco release
5validate`` wraps the release harness: identity is derived from the
6checkout, and the flags that remain are the ones a human must consciously
7assert (target account, deploy/destroy consent, KMS deletion consent).
8``--static-only`` needs none of those — it runs entirely offline.
9"""
11from __future__ import annotations
13import os
14import re
15import subprocess
16import sys
17from datetime import UTC, datetime
18from pathlib import Path
19from typing import NoReturn
21import click
23from .release_cmd import CONSENT_FLAG, _repo_root, _run_git
25_ACCOUNT_RE = re.compile(r"\d{12}")
28def _fail(message: str) -> NoReturn:
29 raise click.ClickException(message)
32@click.group()
33def examples() -> None:
34 """Validate the shipped example manifests."""
37@examples.command("validate")
38@click.option(
39 "--expected-account",
40 default=None,
41 metavar="ACCOUNT_ID",
42 help="Exact 12-digit AWS account id this run may touch (required unless --static-only).",
43)
44@click.option(
45 CONSENT_FLAG,
46 "authorized",
47 is_flag=True,
48 default=False,
49 help=(
50 "Required consent for live runs: deploys real, paid infrastructure "
51 "into the expected account and destroys it afterwards."
52 ),
53)
54@click.option(
55 "--confirm-kms-key-deletion",
56 is_flag=True,
57 default=False,
58 help=(
59 "Authorize scheduling this run's retained EKS KMS keys for their 7-day "
60 "deletion window during cleanup. Required for live runs."
61 ),
62)
63@click.option(
64 "--examples",
65 "selected",
66 default=None,
67 metavar="NAME[,NAME...]",
68 help="Only validate these examples (file stems under examples/; default: all).",
69)
70@click.option(
71 "--skip-examples",
72 default=None,
73 metavar="NAME[,NAME...]",
74 help="Exclude these examples from the selection.",
75)
76@click.option(
77 "--static-only",
78 is_flag=True,
79 default=False,
80 help="Run only the offline checks (no AWS access, no consent flags needed).",
81)
82@click.option(
83 "--max-parallel",
84 type=click.IntRange(min=0),
85 default=0,
86 show_default="0 (all selected at once)",
87 metavar="N",
88 help="Maximum examples running concurrently in the examples action (1 = serial).",
89)
90@click.option(
91 "--actions",
92 default="all",
93 show_default=True,
94 metavar="NAME[,NAME...]",
95 help="Harness actions to run; dependencies are added automatically.",
96)
97@click.option("--run-id", default=None, help="Stable run id (default: UTC timestamp + SHA).")
98@click.option(
99 "--report-dir",
100 default=None,
101 type=click.Path(path_type=Path),
102 help="Report directory (default: ~/gco-example-job-validation-reports/<run-id>).",
103)
104@click.option(
105 "--resume",
106 is_flag=True,
107 default=False,
108 help="Resume an interrupted run; requires the original --run-id and --report-dir.",
109)
110@click.option(
111 "--protected-stack",
112 multiple=True,
113 metavar="NAME",
114 help="Additional non-project CloudFormation stack to preserve exactly (repeatable).",
115)
116def examples_validate(
117 expected_account: str | None,
118 authorized: bool,
119 confirm_kms_key_deletion: bool,
120 selected: str | None,
121 skip_examples: str | None,
122 static_only: bool,
123 max_parallel: int,
124 actions: str,
125 run_id: str | None,
126 report_dir: Path | None,
127 resume: bool,
128 protected_stack: tuple[str, ...],
129) -> None:
130 """Validate example manifests, live (deploy → run → destroy) or offline.
132 Live runs execute every selected example through its documented
133 submission path against freshly deployed infrastructure, then destroy
134 everything and write per-example reports. ``--static-only`` runs the
135 offline contract checks in seconds and is the minimum bar for ANY
136 change under ``examples/`` (CI enforces it too); behavior changes also
137 require a live run for the affected examples. See
138 docs/EXAMPLE_VALIDATION.md.
139 """
140 repo_root = _repo_root()
141 selection_args: list[str] = []
142 if selected:
143 selection_args.extend(["--examples", selected])
144 if skip_examples:
145 selection_args.extend(["--skip-examples", skip_examples])
147 if static_only:
148 command = [
149 sys.executable,
150 "-m",
151 "scripts.example_job_validation",
152 "--static-only",
153 *selection_args,
154 ]
155 result = subprocess.run(command, cwd=repo_root, check=False)
156 sys.exit(result.returncode)
158 if not expected_account or not _ACCOUNT_RE.fullmatch(expected_account):
159 _fail("--expected-account must be an exact 12-digit AWS account id (or use --static-only)")
160 if not authorized:
161 _fail(
162 "Refusing to run without explicit consent. Add "
163 f"{CONSENT_FLAG} to acknowledge that this deploys and destroys real "
164 f"infrastructure in account {expected_account}."
165 )
166 selected_actions = {name.strip() for name in actions.split(",") if name.strip()}
167 if not selected_actions:
168 _fail("--actions must name at least one action")
169 deploy_selected = bool(selected_actions & {"all", "deploy"}) or bool(
170 selected_actions - {"preflight", "baseline", "static"}
171 )
172 if deploy_selected and not confirm_kms_key_deletion:
173 _fail(
174 "The selected actions imply the deploy action, which creates retained "
175 "EKS KMS keys; add --confirm-kms-key-deletion to authorize scheduling "
176 "exactly this run's keys for deletion during cleanup."
177 )
178 if resume and (run_id is None or report_dir is None):
179 _fail(
180 "--resume replays an exact checkpoint identity: pass the original "
181 "--run-id and --report-dir from the interrupted run."
182 )
184 expected_sha = _run_git(repo_root, "rev-parse", "HEAD")
185 expected_branch = _run_git(repo_root, "symbolic-ref", "--short", "HEAD")
186 resolved_run_id = run_id or (
187 datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + expected_sha[:12]
188 )
189 resolved_report_dir = report_dir or (
190 Path.home() / "gco-example-job-validation-reports" / resolved_run_id
191 )
193 command = [
194 sys.executable,
195 "-m",
196 "scripts.example_job_validation",
197 "--repo-root",
198 str(repo_root),
199 "--expected-account",
200 expected_account,
201 "--expected-sha",
202 expected_sha,
203 "--expected-branch",
204 expected_branch,
205 "--actions",
206 ",".join(sorted(selected_actions)),
207 "--run-id",
208 resolved_run_id,
209 "--report-dir",
210 str(resolved_report_dir),
211 "--checkpoint",
212 str(resolved_report_dir / "checkpoint.json"),
213 *selection_args,
214 ]
215 if max_parallel:
216 command.extend(["--max-parallel", str(max_parallel)])
217 if confirm_kms_key_deletion:
218 command.append("--confirm-kms-key-deletion")
219 if resume:
220 command.append("--resume")
221 for name in protected_stack:
222 command.extend(["--protected-stack", name])
224 click.echo(f"run-id: {resolved_run_id}")
225 click.echo(f"sha: {expected_sha}")
226 click.echo(f"branch: {expected_branch}")
227 click.echo(f"account: {expected_account}")
228 click.echo(f"actions: {','.join(sorted(selected_actions))}")
229 click.echo(
230 f"examples: {selected or 'all'}" + (f" minus {skip_examples}" if skip_examples else "")
231 )
232 click.echo(f"report-dir: {resolved_report_dir}")
234 result = subprocess.run(command, cwd=repo_root, env=dict(os.environ), check=False)
235 sys.exit(result.returncode)