Coverage for scripts / live_release_validation / __main__.py: 100.00%
157 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-09-14 22:07 +0000
1"""Command-line entry point for live release validation."""
3from __future__ import annotations
5import argparse
6import json
7import os
8import re
9import shlex
10import sys
11import traceback
12from datetime import UTC, datetime
13from pathlib import Path
15from gco.inference_proxy_config import (
16 INFERENCE_PROXY_TLS_CPU_REQUEST_MILLICORES_DEFAULT,
17 INFERENCE_PROXY_TLS_CPU_TARGET_UTILIZATION_DEFAULT,
18)
20from .checks.schedulers import OPTIONAL_SCHEDULERS
21from .cli_args import path_from_root, repository_root, split_csv_names
22from .models import (
23 InferenceRuntimeSpec,
24 RunCheckpoint,
25 RunSettings,
26 ValidationReport,
27 ensure_private_run_directory,
28 utc_now,
29)
30from .registry import build_action_registry
31from .runner import LiveValidationRunner, require_local_execution
33# Backwards-compatible aliases for this module's historical private helpers.
34_repository_root = repository_root
35_split_actions = split_csv_names
36_path_from_root = path_from_root
39def _build_parser() -> argparse.ArgumentParser:
40 registry = build_action_registry()
41 parser = argparse.ArgumentParser(
42 prog="python -m scripts.live_release_validation",
43 description=(
44 "Deploy, validate, and always destroy an exact GCO commit while producing "
45 "local JSON and Markdown reports. Reports enumerate account-specific "
46 "identifiers; post only a sanitized summary publicly."
47 ),
48 )
49 parser.add_argument("--repo-root", help="GCO checkout (default: current Git root)")
50 parser.add_argument(
51 "--expected-account",
52 default=os.environ.get("GCO_LIVE_EXPECTED_ACCOUNT"),
53 help="Exact 12-digit AWS account ID (or GCO_LIVE_EXPECTED_ACCOUNT)",
54 )
55 parser.add_argument(
56 "--expected-sha",
57 default=os.environ.get("GCO_LIVE_EXPECTED_SHA"),
58 help="Exact 40-character Git commit (or GCO_LIVE_EXPECTED_SHA)",
59 )
60 parser.add_argument(
61 "--expected-branch",
62 default=os.environ.get("GCO_LIVE_EXPECTED_BRANCH"),
63 help="Exact local branch identity (or GCO_LIVE_EXPECTED_BRANCH)",
64 )
65 parser.add_argument(
66 "--profile",
67 choices=("configured", "single-region", "multi-region"),
68 default="configured",
69 help="Validate, but never rewrite, the topology in cdk.json",
70 )
71 parser.add_argument(
72 "--actions",
73 type=_split_actions,
74 default=("all",),
75 metavar="NAME[,NAME...]",
76 help="Selectable actions; dependencies are added automatically (default: all)",
77 )
78 parser.add_argument("--list-actions", action="store_true", help="List actions and exit")
79 parser.add_argument(
80 "--run-id",
81 help="Stable run/checkpoint identifier (default: UTC timestamp plus commit)",
82 )
83 parser.add_argument(
84 "--report-dir",
85 help="Report directory (default: .live-release-validation/<run-id>)",
86 )
87 parser.add_argument(
88 "--checkpoint",
89 help="Checkpoint JSON path (default: <report-dir>/checkpoint.json)",
90 )
91 parser.add_argument(
92 "--resume",
93 action="store_true",
94 help="Resume an exact identity-matched checkpoint",
95 )
96 parser.add_argument(
97 "--protected-stack",
98 action="append",
99 default=[],
100 metavar="NAME",
101 help="Additional non-project CloudFormation stack to preserve exactly",
102 )
103 parser.add_argument("--max-workers", type=int, default=4)
104 parser.add_argument("--job-timeout-seconds", type=int, default=1800)
105 parser.add_argument("--queue-timeout-seconds", type=int, default=900)
106 parser.add_argument("--poll-interval-seconds", type=int, default=10)
107 parser.add_argument("--destroy-attempts", type=int, default=3)
108 parser.add_argument("--destroy-retry-delay-seconds", type=int, default=30)
109 parser.add_argument(
110 "--min-free-disk-gib",
111 type=int,
112 default=20,
113 help=(
114 "Free disk space (GiB) preflight requires on the checkout, report directory, "
115 "and home volume before deploy builds container images; 0 disables the check"
116 ),
117 )
118 parser.add_argument(
119 "--confirm-kms-key-deletion",
120 action="store_true",
121 help=(
122 "Explicitly authorize scheduling only this run's exact retained EKS "
123 "KMS keys for deletion after stack teardown"
124 ),
125 )
126 parser.add_argument(
127 "--optional-schedulers",
128 type=_split_actions,
129 default=(),
130 metavar="NAME[,NAME...]",
131 help=(
132 "Force-enable off-by-default schedulers for this run's deploy so the "
133 "schedulers action can prove them (yunikorn, slurm, or all)"
134 ),
135 )
136 parser.add_argument(
137 "--inference-region",
138 help="Deployed Region used by the inference action",
139 )
140 for framework, default_port in (("vllm", 8000), ("tgi", 8080)):
141 parser.add_argument(
142 f"--inference-{framework}-image",
143 help=f"Immutable {framework} image reference containing @sha256:",
144 )
145 parser.add_argument(
146 f"--inference-{framework}-model-id",
147 help=f"Exact model identifier served by {framework}",
148 )
149 parser.add_argument(
150 f"--inference-{framework}-model-revision",
151 help=f"Full immutable 40-hex model commit served by {framework}",
152 )
153 parser.set_defaults(**{f"inference_{framework}_port": default_port})
154 parser.add_argument("--inference-gpu-count", type=int, default=0)
155 parser.add_argument(
156 "--confirm-inference-deployment",
157 action="store_true",
158 help=(
159 "Explicitly authorize the inference action to create and delete "
160 "four strictly sequential vLLM/TGI endpoint scenarios"
161 ),
162 )
163 parser.epilog = "Actions: " + ", ".join(registry)
164 return parser
167def _inference_selected(actions: tuple[str, ...]) -> bool:
168 """Return whether dependency expansion will execute the inference action."""
169 return "all" in actions or "inference" in actions
172def _validate_args(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None:
173 if not args.expected_account or not re.fullmatch(r"\d{12}", args.expected_account):
174 parser.error("--expected-account must be an exact 12-digit AWS account ID")
175 if not args.expected_sha or not re.fullmatch(r"[0-9a-fA-F]{40}", args.expected_sha):
176 parser.error("--expected-sha must be an exact 40-character commit SHA")
177 if not args.expected_branch or not args.expected_branch.strip():
178 parser.error("--expected-branch is required")
179 if args.run_id and not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,79}", args.run_id):
180 parser.error("--run-id must be 1-80 safe filename characters")
181 for name in args.protected_stack:
182 if not name or not re.fullmatch(r"[A-Za-z][-A-Za-z0-9]{0,127}", name):
183 parser.error(f"Invalid --protected-stack name: {name!r}")
184 for option in (
185 "max_workers",
186 "job_timeout_seconds",
187 "queue_timeout_seconds",
188 "poll_interval_seconds",
189 "destroy_attempts",
190 "destroy_retry_delay_seconds",
191 ):
192 if getattr(args, option) <= 0:
193 parser.error(f"--{option.replace('_', '-')} must be positive")
194 if args.min_free_disk_gib < 0:
195 parser.error("--min-free-disk-gib must be zero or positive")
196 valid_optional = set(OPTIONAL_SCHEDULERS)
197 unknown_schedulers = sorted(set(args.optional_schedulers) - valid_optional - {"all"})
198 if unknown_schedulers:
199 parser.error(
200 "--optional-schedulers accepts "
201 + ", ".join((*OPTIONAL_SCHEDULERS, "all"))
202 + f"; got: {', '.join(unknown_schedulers)}"
203 )
204 if "all" in args.optional_schedulers and len(args.optional_schedulers) != 1:
205 parser.error("--optional-schedulers 'all' cannot be combined with individual names")
206 if _inference_selected(args.actions):
207 required = (
208 "inference_region",
209 "inference_vllm_image",
210 "inference_vllm_model_id",
211 "inference_vllm_model_revision",
212 "inference_tgi_image",
213 "inference_tgi_model_id",
214 "inference_tgi_model_revision",
215 )
216 for option in required:
217 if not getattr(args, option):
218 parser.error(f"--{option.replace('_', '-')} is required when inference runs")
219 if not args.confirm_inference_deployment:
220 parser.error(
221 "--confirm-inference-deployment is required when the inference action runs"
222 )
223 if args.inference_gpu_count < 0:
224 parser.error("--inference-gpu-count must be non-negative")
227def _settings_from_args(
228 parser: argparse.ArgumentParser,
229 args: argparse.Namespace,
230) -> RunSettings:
231 _validate_args(parser, args)
232 root = _repository_root(args.repo_root)
233 run_id = args.run_id or (
234 datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + args.expected_sha[:12].lower()
235 )
236 report_dir = _path_from_root(
237 root,
238 args.report_dir,
239 Path(".live-release-validation") / run_id,
240 )
241 checkpoint = _path_from_root(
242 root,
243 args.checkpoint,
244 report_dir / "checkpoint.json",
245 )
246 protected = tuple(dict.fromkeys(("CDKToolkit", "GCOGitHubOIDCStack", *args.protected_stack)))
247 inference_enabled = _inference_selected(args.actions)
248 proxy_config: dict[str, object] = {
249 "tls_proxy_cpu_request_millicores": (INFERENCE_PROXY_TLS_CPU_REQUEST_MILLICORES_DEFAULT),
250 "tls_proxy_cpu_target_utilization_percentage": (
251 INFERENCE_PROXY_TLS_CPU_TARGET_UTILIZATION_DEFAULT
252 ),
253 }
254 if inference_enabled:
255 try:
256 cdk_config = json.loads((root / "cdk.json").read_text(encoding="utf-8"))
257 context = cdk_config.get("context") if isinstance(cdk_config, dict) else None
258 candidate = context.get("inference_proxy") if isinstance(context, dict) else None
259 if candidate is not None and not isinstance(candidate, dict):
260 parser.error("cdk.json context.inference_proxy must be an object or null")
261 if isinstance(candidate, dict):
262 proxy_config.update(candidate)
263 except (OSError, UnicodeError, json.JSONDecodeError) as error:
264 parser.error(f"could not read inference_proxy settings from cdk.json: {error}")
265 proxy_request = proxy_config["tls_proxy_cpu_request_millicores"]
266 proxy_target = proxy_config["tls_proxy_cpu_target_utilization_percentage"]
267 if inference_enabled and (type(proxy_request) is not int or type(proxy_target) is not int):
268 parser.error("cdk.json inference_proxy TLS CPU settings must be integers")
269 runtimes = (
270 (
271 InferenceRuntimeSpec(
272 framework="vllm",
273 image=args.inference_vllm_image or "",
274 model_id=args.inference_vllm_model_id or "",
275 model_revision=args.inference_vllm_model_revision or "",
276 port=8000,
277 ),
278 InferenceRuntimeSpec(
279 framework="tgi",
280 image=args.inference_tgi_image or "",
281 model_id=args.inference_tgi_model_id or "",
282 model_revision=args.inference_tgi_model_revision or "",
283 port=8080,
284 ),
285 )
286 if inference_enabled
287 else ()
288 )
289 return RunSettings(
290 run_id=run_id,
291 repo_root=root,
292 report_dir=report_dir,
293 checkpoint_path=checkpoint,
294 expected_account=args.expected_account,
295 expected_sha=args.expected_sha.lower(),
296 expected_branch=args.expected_branch.strip(),
297 profile=args.profile,
298 requested_actions=args.actions,
299 protected_stack_names=protected,
300 max_workers=args.max_workers,
301 job_timeout_seconds=args.job_timeout_seconds,
302 queue_timeout_seconds=args.queue_timeout_seconds,
303 poll_interval_seconds=args.poll_interval_seconds,
304 destroy_attempts=args.destroy_attempts,
305 destroy_retry_delay_seconds=args.destroy_retry_delay_seconds,
306 min_free_disk_gib=args.min_free_disk_gib,
307 confirm_kms_key_deletion=args.confirm_kms_key_deletion,
308 resume=args.resume,
309 optional_schedulers=(
310 OPTIONAL_SCHEDULERS
311 if "all" in args.optional_schedulers
312 else tuple(sorted(set(args.optional_schedulers)))
313 ),
314 inference_enabled=inference_enabled,
315 selected_region=args.inference_region or "",
316 inference_runtimes=runtimes,
317 proxy_tls_cpu_request=f"{proxy_request}m" if inference_enabled else "100m",
318 proxy_tls_cpu_target=proxy_target if isinstance(proxy_target, int) else 70,
319 gpu_count=args.inference_gpu_count,
320 consent=args.confirm_inference_deployment,
321 )
324def main() -> int:
325 """Parse arguments and execute the live validation runner."""
326 try:
327 require_local_execution()
328 except RuntimeError as exc:
329 print(f"Live validation could not start: {exc}", file=sys.stderr)
330 return 1
332 parser = _build_parser()
333 args = parser.parse_args()
334 if args.list_actions:
335 for definition in build_action_registry().values():
336 dependencies = ", ".join(definition.dependencies) or "none"
337 print(f"{definition.name:16} {definition.description} [depends: {dependencies}]")
338 return 0
340 settings: RunSettings | None = None
341 try:
342 settings = _settings_from_args(parser, args)
343 return LiveValidationRunner(settings).run()
344 except KeyboardInterrupt:
345 print("Live validation interrupted before the runner initialized", file=sys.stderr)
346 return 130
347 except BaseException as exc:
348 print(f"Live validation could not start: {type(exc).__name__}: {exc}", file=sys.stderr)
349 if settings is not None:
350 report = ValidationReport(
351 run_id=settings.run_id,
352 identity=settings.identity(),
353 selected_actions=list(settings.requested_actions),
354 started_at=utc_now(),
355 ended_at=utc_now(),
356 status="failed",
357 fatal_error="".join(traceback.format_exception(type(exc), exc, exc.__traceback__)),
358 )
359 if settings.checkpoint_path.is_file():
360 try:
361 checkpoint = RunCheckpoint.from_path(settings.checkpoint_path)
362 except OSError, ValueError:
363 checkpoint = None
364 if checkpoint is not None and checkpoint.deployment_attempted:
365 if checkpoint.identity == settings.identity():
366 recovery_argv = [
367 sys.executable,
368 "-m",
369 "scripts.live_release_validation",
370 *sys.argv[1:],
371 ]
372 if "--resume" not in recovery_argv:
373 recovery_argv.append("--resume")
374 report.cleanup = {
375 "needed": True,
376 "completed": False,
377 "blocked": (
378 "Runner construction failed after an identity-verified deployed "
379 "checkpoint was loaded; safe automatic destruction could not be "
380 "initialized."
381 ),
382 "recovery_command": shlex.join(recovery_argv),
383 }
384 else:
385 report.cleanup = {
386 "needed": True,
387 "completed": False,
388 "blocked": (
389 "A deployed checkpoint exists, but its identity does not match "
390 "this invocation. No cleanup authority was established; resume "
391 "with the original exact command and checkpoint identity."
392 ),
393 }
394 try:
395 ensure_private_run_directory(settings.report_dir, settings.checkpoint_path)
396 json_path, markdown_path = report.write(settings.report_dir)
397 except (OSError, ValueError) as report_exc:
398 print(
399 "Failure report was not written because the output directory is unsafe: "
400 f"{report_exc}",
401 file=sys.stderr,
402 )
403 else:
404 print(f"JSON report: {json_path}", file=sys.stderr)
405 print(f"Markdown report: {markdown_path}", file=sys.stderr)
406 return 1
409if __name__ == "__main__":
410 raise SystemExit(main())