Coverage for scripts / live_release_validation / __main__.py: 100.00%
172 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-10-28 16:19 +0000
1"""Command-line entry point for live release validation."""
3from __future__ import annotations
5import argparse
6import json
7import os
8import re
9import shlex
10import sys
11import traceback
12from datetime import UTC, datetime
13from pathlib import Path
15from gco.eks_capabilities_config import EKS_CAPABILITY_TYPES
16from gco.inference_proxy_config import (
17 INFERENCE_PROXY_TLS_CPU_REQUEST_MILLICORES_DEFAULT,
18 INFERENCE_PROXY_TLS_CPU_TARGET_UTILIZATION_DEFAULT,
19)
21from .checks.eks_capabilities import build_eks_capabilities_overrides, overrides_json
22from .checks.schedulers import OPTIONAL_SCHEDULERS
23from .checks.tracing import tracing_overrides_json
24from .cli_args import path_from_root, repository_root, split_csv_names
25from .inference_contract import TRITON_MODEL_NAME, triton_model_repository_revision
26from .models import (
27 InferenceRuntimeSpec,
28 RunCheckpoint,
29 RunSettings,
30 ValidationReport,
31 ensure_private_run_directory,
32 utc_now,
33)
34from .registry import build_action_registry
35from .runner import LiveValidationRunner, require_local_execution
37# Backwards-compatible aliases for this module's historical private helpers.
38_repository_root = repository_root
39_split_actions = split_csv_names
40_path_from_root = path_from_root
43def _build_parser() -> argparse.ArgumentParser:
44 registry = build_action_registry()
45 parser = argparse.ArgumentParser(
46 prog="python -m scripts.live_release_validation",
47 description=(
48 "Deploy, validate, and always destroy an exact GCO commit while producing "
49 "local JSON and Markdown reports. Reports enumerate account-specific "
50 "identifiers; post only a sanitized summary publicly."
51 ),
52 )
53 parser.add_argument("--repo-root", help="GCO checkout (default: current Git root)")
54 parser.add_argument(
55 "--expected-account",
56 default=os.environ.get("GCO_LIVE_EXPECTED_ACCOUNT"),
57 help="Exact 12-digit AWS account ID (or GCO_LIVE_EXPECTED_ACCOUNT)",
58 )
59 parser.add_argument(
60 "--expected-sha",
61 default=os.environ.get("GCO_LIVE_EXPECTED_SHA"),
62 help="Exact 40-character Git commit (or GCO_LIVE_EXPECTED_SHA)",
63 )
64 parser.add_argument(
65 "--expected-branch",
66 default=os.environ.get("GCO_LIVE_EXPECTED_BRANCH"),
67 help="Exact local branch identity (or GCO_LIVE_EXPECTED_BRANCH)",
68 )
69 parser.add_argument(
70 "--profile",
71 choices=("configured", "single-region", "multi-region"),
72 default="configured",
73 help="Validate, but never rewrite, the topology in cdk.json",
74 )
75 parser.add_argument(
76 "--actions",
77 type=_split_actions,
78 default=("all",),
79 metavar="NAME[,NAME...]",
80 help="Selectable actions; dependencies are added automatically (default: all)",
81 )
82 parser.add_argument("--list-actions", action="store_true", help="List actions and exit")
83 parser.add_argument(
84 "--run-id",
85 help="Stable run/checkpoint identifier (default: UTC timestamp plus commit)",
86 )
87 parser.add_argument(
88 "--report-dir",
89 help="Report directory (default: .live-release-validation/<run-id>)",
90 )
91 parser.add_argument(
92 "--checkpoint",
93 help="Checkpoint JSON path (default: <report-dir>/checkpoint.json)",
94 )
95 parser.add_argument(
96 "--resume",
97 action="store_true",
98 help="Resume an exact identity-matched checkpoint",
99 )
100 parser.add_argument(
101 "--protected-stack",
102 action="append",
103 default=[],
104 metavar="NAME",
105 help="Additional non-project CloudFormation stack to preserve exactly",
106 )
107 parser.add_argument("--max-workers", type=int, default=4)
108 parser.add_argument("--job-timeout-seconds", type=int, default=1800)
109 parser.add_argument("--queue-timeout-seconds", type=int, default=900)
110 parser.add_argument("--poll-interval-seconds", type=int, default=10)
111 parser.add_argument("--destroy-attempts", type=int, default=3)
112 parser.add_argument("--destroy-retry-delay-seconds", type=int, default=30)
113 parser.add_argument(
114 "--min-free-disk-gib",
115 type=int,
116 default=20,
117 help=(
118 "Free disk space (GiB) preflight requires on the checkout, report directory, "
119 "and home volume before deploy builds container images; 0 disables the check"
120 ),
121 )
122 parser.add_argument(
123 "--confirm-kms-key-deletion",
124 action="store_true",
125 help=(
126 "Explicitly authorize scheduling only this run's exact retained EKS "
127 "KMS keys for deletion after stack teardown"
128 ),
129 )
130 parser.add_argument(
131 "--optional-schedulers",
132 type=_split_actions,
133 default=(),
134 metavar="NAME[,NAME...]",
135 help=(
136 "Force-enable off-by-default schedulers for this run's deploy so the "
137 "schedulers action can prove them (yunikorn, slurm, or all)"
138 ),
139 )
140 parser.add_argument(
141 "--eks-capabilities",
142 type=_split_actions,
143 default=(),
144 metavar="NAME[,NAME...]",
145 help=(
146 "Enable off-by-default EKS Capabilities for this run's deploy so the "
147 "eks-capabilities action can prove each one attaches ACTIVE (ack, kro, or all)"
148 ),
149 )
150 parser.add_argument(
151 "--inference-region",
152 help="Deployed Region used by the inference action",
153 )
154 for framework, default_port in (("vllm", 8000), ("sglang", 30000)):
155 parser.add_argument(
156 f"--inference-{framework}-image",
157 help=f"Immutable {framework} image reference containing @sha256:",
158 )
159 parser.add_argument(
160 f"--inference-{framework}-model-id",
161 help=f"Exact model identifier served by {framework}",
162 )
163 parser.add_argument(
164 f"--inference-{framework}-model-revision",
165 help=f"Full immutable 40-hex model commit served by {framework}",
166 )
167 parser.set_defaults(**{f"inference_{framework}_port": default_port})
168 parser.add_argument(
169 "--inference-triton-image",
170 help=(
171 "Immutable Triton Inference Server image reference containing @sha256:; "
172 "its model is the harness-generated identity repository served from "
173 "the model registry"
174 ),
175 )
176 parser.add_argument("--inference-gpu-count", type=int, default=0)
177 parser.add_argument(
178 "--confirm-inference-deployment",
179 action="store_true",
180 help=(
181 "Explicitly authorize the inference action to create and delete "
182 "every strictly sequential vLLM/SGLang/Triton endpoint scenario, "
183 "including the Mooncake KV-cache store and its model-registry upload"
184 ),
185 )
186 parser.epilog = "Actions: " + ", ".join(registry)
187 return parser
190def _inference_selected(actions: tuple[str, ...]) -> bool:
191 """Return whether dependency expansion will execute the inference action."""
192 return "all" in actions or "inference" in actions
195def _validate_args(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None:
196 if not args.expected_account or not re.fullmatch(r"\d{12}", args.expected_account):
197 parser.error("--expected-account must be an exact 12-digit AWS account ID")
198 if not args.expected_sha or not re.fullmatch(r"[0-9a-fA-F]{40}", args.expected_sha):
199 parser.error("--expected-sha must be an exact 40-character commit SHA")
200 if not args.expected_branch or not args.expected_branch.strip():
201 parser.error("--expected-branch is required")
202 if args.run_id and not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,79}", args.run_id):
203 parser.error("--run-id must be 1-80 safe filename characters")
204 for name in args.protected_stack:
205 if not name or not re.fullmatch(r"[A-Za-z][-A-Za-z0-9]{0,127}", name):
206 parser.error(f"Invalid --protected-stack name: {name!r}")
207 for option in (
208 "max_workers",
209 "job_timeout_seconds",
210 "queue_timeout_seconds",
211 "poll_interval_seconds",
212 "destroy_attempts",
213 "destroy_retry_delay_seconds",
214 ):
215 if getattr(args, option) <= 0:
216 parser.error(f"--{option.replace('_', '-')} must be positive")
217 if args.min_free_disk_gib < 0:
218 parser.error("--min-free-disk-gib must be zero or positive")
219 valid_optional = set(OPTIONAL_SCHEDULERS)
220 unknown_schedulers = sorted(set(args.optional_schedulers) - valid_optional - {"all"})
221 if unknown_schedulers:
222 parser.error(
223 "--optional-schedulers accepts "
224 + ", ".join((*OPTIONAL_SCHEDULERS, "all"))
225 + f"; got: {', '.join(unknown_schedulers)}"
226 )
227 if "all" in args.optional_schedulers and len(args.optional_schedulers) != 1:
228 parser.error("--optional-schedulers 'all' cannot be combined with individual names")
229 unknown_capabilities = sorted(set(args.eks_capabilities) - set(EKS_CAPABILITY_TYPES) - {"all"})
230 if unknown_capabilities:
231 parser.error(
232 "--eks-capabilities accepts "
233 + ", ".join((*EKS_CAPABILITY_TYPES, "all"))
234 + f"; got: {', '.join(unknown_capabilities)}"
235 )
236 if "all" in args.eks_capabilities and len(args.eks_capabilities) != 1:
237 parser.error("--eks-capabilities 'all' cannot be combined with individual names")
238 if _inference_selected(args.actions):
239 required = (
240 "inference_region",
241 "inference_vllm_image",
242 "inference_vllm_model_id",
243 "inference_vllm_model_revision",
244 "inference_sglang_image",
245 "inference_sglang_model_id",
246 "inference_sglang_model_revision",
247 "inference_triton_image",
248 )
249 for option in required:
250 if not getattr(args, option):
251 parser.error(f"--{option.replace('_', '-')} is required when inference runs")
252 if not args.confirm_inference_deployment:
253 parser.error(
254 "--confirm-inference-deployment is required when the inference action runs"
255 )
256 if args.inference_gpu_count < 0:
257 parser.error("--inference-gpu-count must be non-negative")
260def _settings_from_args(
261 parser: argparse.ArgumentParser,
262 args: argparse.Namespace,
263) -> RunSettings:
264 _validate_args(parser, args)
265 root = _repository_root(args.repo_root)
266 run_id = args.run_id or (
267 datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + args.expected_sha[:12].lower()
268 )
269 report_dir = _path_from_root(
270 root,
271 args.report_dir,
272 Path(".live-release-validation") / run_id,
273 )
274 checkpoint = _path_from_root(
275 root,
276 args.checkpoint,
277 report_dir / "checkpoint.json",
278 )
279 protected = tuple(dict.fromkeys(("CDKToolkit", "GCOGitHubOIDCStack", *args.protected_stack)))
280 capability_types = (
281 EKS_CAPABILITY_TYPES
282 if "all" in args.eks_capabilities
283 else tuple(name for name in EKS_CAPABILITY_TYPES if name in args.eks_capabilities)
284 )
285 capabilities_overrides_json = ""
286 if capability_types:
287 capabilities_overrides_json = overrides_json(
288 build_eks_capabilities_overrides(types=capability_types)
289 )
290 inference_enabled = _inference_selected(args.actions)
291 proxy_config: dict[str, object] = {
292 "tls_proxy_cpu_request_millicores": (INFERENCE_PROXY_TLS_CPU_REQUEST_MILLICORES_DEFAULT),
293 "tls_proxy_cpu_target_utilization_percentage": (
294 INFERENCE_PROXY_TLS_CPU_TARGET_UTILIZATION_DEFAULT
295 ),
296 }
297 if inference_enabled:
298 try:
299 cdk_config = json.loads((root / "cdk.json").read_text(encoding="utf-8"))
300 context = cdk_config.get("context") if isinstance(cdk_config, dict) else None
301 candidate = context.get("inference_proxy") if isinstance(context, dict) else None
302 if candidate is not None and not isinstance(candidate, dict):
303 parser.error("cdk.json context.inference_proxy must be an object or null")
304 if isinstance(candidate, dict):
305 proxy_config.update(candidate)
306 except (OSError, UnicodeError, json.JSONDecodeError) as error:
307 parser.error(f"could not read inference_proxy settings from cdk.json: {error}")
308 proxy_request = proxy_config["tls_proxy_cpu_request_millicores"]
309 proxy_target = proxy_config["tls_proxy_cpu_target_utilization_percentage"]
310 if inference_enabled and (type(proxy_request) is not int or type(proxy_target) is not int):
311 parser.error("cdk.json inference_proxy TLS CPU settings must be integers")
312 runtimes = (
313 (
314 InferenceRuntimeSpec(
315 framework="vllm",
316 image=args.inference_vllm_image or "",
317 model_id=args.inference_vllm_model_id or "",
318 model_revision=args.inference_vllm_model_revision or "",
319 port=8000,
320 ),
321 InferenceRuntimeSpec(
322 framework="sglang",
323 image=args.inference_sglang_image or "",
324 model_id=args.inference_sglang_model_id or "",
325 model_revision=args.inference_sglang_model_revision or "",
326 port=30000,
327 ),
328 # Triton serves the repository the harness generates and uploads;
329 # its identity is that repository's content digest.
330 InferenceRuntimeSpec(
331 framework="triton",
332 image=args.inference_triton_image or "",
333 model_id=TRITON_MODEL_NAME,
334 model_revision=triton_model_repository_revision(),
335 port=8000,
336 ),
337 )
338 if inference_enabled
339 else ()
340 )
341 return RunSettings(
342 run_id=run_id,
343 repo_root=root,
344 report_dir=report_dir,
345 checkpoint_path=checkpoint,
346 expected_account=args.expected_account,
347 expected_sha=args.expected_sha.lower(),
348 expected_branch=args.expected_branch.strip(),
349 profile=args.profile,
350 requested_actions=args.actions,
351 protected_stack_names=protected,
352 max_workers=args.max_workers,
353 job_timeout_seconds=args.job_timeout_seconds,
354 queue_timeout_seconds=args.queue_timeout_seconds,
355 poll_interval_seconds=args.poll_interval_seconds,
356 destroy_attempts=args.destroy_attempts,
357 destroy_retry_delay_seconds=args.destroy_retry_delay_seconds,
358 min_free_disk_gib=args.min_free_disk_gib,
359 confirm_kms_key_deletion=args.confirm_kms_key_deletion,
360 resume=args.resume,
361 optional_schedulers=(
362 OPTIONAL_SCHEDULERS
363 if "all" in args.optional_schedulers
364 else tuple(sorted(set(args.optional_schedulers)))
365 ),
366 eks_capabilities_overrides_json=capabilities_overrides_json,
367 # Every CDK invocation of a release run samples every request, so the
368 # tracing action finds a span for each call it drives.
369 tracing_overrides_json=tracing_overrides_json(),
370 inference_enabled=inference_enabled,
371 selected_region=args.inference_region or "",
372 inference_runtimes=runtimes,
373 proxy_tls_cpu_request=f"{proxy_request}m" if inference_enabled else "100m",
374 proxy_tls_cpu_target=proxy_target if isinstance(proxy_target, int) else 70,
375 gpu_count=args.inference_gpu_count,
376 consent=args.confirm_inference_deployment,
377 )
380def main() -> int:
381 """Parse arguments and execute the live validation runner."""
382 try:
383 require_local_execution()
384 except RuntimeError as exc:
385 print(f"Live validation could not start: {exc}", file=sys.stderr)
386 return 1
388 parser = _build_parser()
389 args = parser.parse_args()
390 if args.list_actions:
391 for definition in build_action_registry().values():
392 dependencies = ", ".join(definition.dependencies) or "none"
393 print(f"{definition.name:16} {definition.description} [depends: {dependencies}]")
394 return 0
396 settings: RunSettings | None = None
397 try:
398 settings = _settings_from_args(parser, args)
399 return LiveValidationRunner(settings).run()
400 except KeyboardInterrupt:
401 print("Live validation interrupted before the runner initialized", file=sys.stderr)
402 return 130
403 except BaseException as exc:
404 print(f"Live validation could not start: {type(exc).__name__}: {exc}", file=sys.stderr)
405 if settings is not None:
406 report = ValidationReport(
407 run_id=settings.run_id,
408 identity=settings.identity(),
409 selected_actions=list(settings.requested_actions),
410 started_at=utc_now(),
411 ended_at=utc_now(),
412 status="failed",
413 fatal_error="".join(traceback.format_exception(type(exc), exc, exc.__traceback__)),
414 )
415 if settings.checkpoint_path.is_file():
416 try:
417 checkpoint = RunCheckpoint.from_path(settings.checkpoint_path)
418 except OSError, ValueError:
419 checkpoint = None
420 if checkpoint is not None and checkpoint.deployment_attempted:
421 if checkpoint.identity == settings.identity():
422 recovery_argv = [
423 sys.executable,
424 "-m",
425 "scripts.live_release_validation",
426 *sys.argv[1:],
427 ]
428 if "--resume" not in recovery_argv:
429 recovery_argv.append("--resume")
430 report.cleanup = {
431 "needed": True,
432 "completed": False,
433 "blocked": (
434 "Runner construction failed after an identity-verified deployed "
435 "checkpoint was loaded; safe automatic destruction could not be "
436 "initialized."
437 ),
438 "recovery_command": shlex.join(recovery_argv),
439 }
440 else:
441 report.cleanup = {
442 "needed": True,
443 "completed": False,
444 "blocked": (
445 "A deployed checkpoint exists, but its identity does not match "
446 "this invocation. No cleanup authority was established; resume "
447 "with the original exact command and checkpoint identity."
448 ),
449 }
450 try:
451 ensure_private_run_directory(settings.report_dir, settings.checkpoint_path)
452 json_path, markdown_path = report.write(settings.report_dir)
453 except (OSError, ValueError) as report_exc:
454 print(
455 "Failure report was not written because the output directory is unsafe: "
456 f"{report_exc}",
457 file=sys.stderr,
458 )
459 else:
460 print(f"JSON report: {json_path}", file=sys.stderr)
461 print(f"Markdown report: {markdown_path}", file=sys.stderr)
462 return 1
465if __name__ == "__main__":
466 raise SystemExit(main())