Coverage for scripts / example_job_validation / __main__.py: 100.00%

1"""CLI entry: ``python -m scripts.example_job_validation``. 

2 

3Mirrors ``scripts.live_release_validation.__main__`` (same identity flags, 

4consent posture, checkpoint/resume semantics) plus example selection and a 

5fully offline ``--static-only`` mode. 

6""" 

7 

8from __future__ import annotations 

9 

10import argparse 

11import contextlib 

12import os 

13import re 

14import sys 

15import traceback 

16from datetime import UTC, datetime 

17from pathlib import Path 

18 

19from scripts.live_release_validation.cli_args import ( 

20 path_from_root, 

21 repository_root, 

22 split_csv_names, 

23) 

24from scripts.live_release_validation.models import ( 

25 ValidationReport, 

26 utc_now, 

27) 

28from scripts.live_release_validation.runner import ( 

29 LiveValidationRunner, 

30 require_local_execution, 

31) 

32 

33from .models import ExampleRunSettings 

34from .registry import build_action_registry 

35from .specs import EXAMPLE_SPECS 

36from .static_checks import run_static_checks 

37 

38REPORT_TITLE = "GCO Example Job Validation" 

39REPORT_STEM = "example-job-validation" 

40 

41 

42def _split_names(value: str) -> tuple[str, ...]: 

43 names = tuple(dict.fromkeys(item.strip() for item in value.split(",") if item.strip())) 

44 if not names: 

45 raise argparse.ArgumentTypeError("expected at least one name") 

46 return names 

47 

48 

49def _build_parser() -> argparse.ArgumentParser: 

50 registry = build_action_registry() 

51 parser = argparse.ArgumentParser( 

52 prog="python -m scripts.example_job_validation", 

53 description=( 

54 "Deploy the configured GCO topology, run every selected example " 

55 "through its documented submission path, verify its success " 

56 "criteria, and always destroy what was deployed. Reports carry " 

57 "account-specific identifiers; post only sanitized summaries." 

58 ), 

59 ) 

60 parser.add_argument("--repo-root", help="GCO checkout (default: current Git root)") 

61 parser.add_argument( 

62 "--expected-account", 

63 default=os.environ.get("GCO_LIVE_EXPECTED_ACCOUNT"), 

64 help="Exact 12-digit AWS account ID (or GCO_LIVE_EXPECTED_ACCOUNT)", 

65 ) 

66 parser.add_argument( 

67 "--expected-sha", 

68 default=os.environ.get("GCO_LIVE_EXPECTED_SHA"), 

69 help="Exact 40-character Git commit (or GCO_LIVE_EXPECTED_SHA)", 

70 ) 

71 parser.add_argument( 

72 "--expected-branch", 

73 default=os.environ.get("GCO_LIVE_EXPECTED_BRANCH"), 

74 help="Exact local branch identity (or GCO_LIVE_EXPECTED_BRANCH)", 

75 ) 

76 parser.add_argument( 

77 "--actions", 

78 type=split_csv_names, 

79 default=("all",), 

80 metavar="NAME[,NAME...]", 

81 help="Selectable actions; dependencies are added automatically (default: all)", 

82 ) 

83 parser.add_argument("--list-actions", action="store_true", help="List actions and exit") 

84 parser.add_argument( 

85 "--examples", 

86 type=_split_names, 

87 default=(), 

88 metavar="NAME[,NAME...]", 

89 help="Only validate these examples (file stems; default: every example)", 

90 ) 

91 parser.add_argument( 

92 "--skip-examples", 

93 type=_split_names, 

94 default=(), 

95 metavar="NAME[,NAME...]", 

96 help="Exclude these examples from the selection", 

97 ) 

98 parser.add_argument( 

99 "--static-only", 

100 action="store_true", 

101 help="Run only the offline checks (no AWS access) and exit", 

102 ) 

103 parser.add_argument( 

104 "--max-parallel", 

105 type=int, 

106 default=0, 

107 metavar="N", 

108 help=( 

109 "Maximum examples running concurrently in the examples action " 

110 "(default 0 = all selected at once; 1 = serial)" 

111 ), 

112 ) 

113 parser.add_argument("--run-id", help="Stable run/checkpoint identifier") 

114 parser.add_argument( 

115 "--report-dir", help="Report directory (default: .example-job-validation/<run-id>)" 

116 ) 

117 parser.add_argument( 

118 "--checkpoint", help="Checkpoint JSON path (default: <report-dir>/checkpoint.json)" 

119 ) 

120 parser.add_argument( 

121 "--resume", action="store_true", help="Resume an exact identity-matched checkpoint" 

122 ) 

123 parser.add_argument( 

124 "--protected-stack", 

125 action="append", 

126 default=[], 

127 metavar="NAME", 

128 help="Additional non-project CloudFormation stack to preserve exactly", 

129 ) 

130 parser.add_argument( 

131 "--confirm-kms-key-deletion", 

132 action="store_true", 

133 help=( 

134 "Explicitly authorize scheduling only this run's exact retained EKS " 

135 "KMS keys for deletion after stack teardown" 

136 ), 

137 ) 

138 parser.epilog = ( 

139 "Actions: " + ", ".join(registry) + ". Examples: " + ", ".join(sorted(EXAMPLE_SPECS)) 

140 ) 

141 return parser 

142 

143 

144def _select_examples(parser: argparse.ArgumentParser, args: argparse.Namespace) -> tuple[str, ...]: 

145 selected = list(args.examples) if args.examples else sorted(EXAMPLE_SPECS) 

146 unknown = sorted({*args.examples, *args.skip_examples} - set(EXAMPLE_SPECS)) 

147 if unknown: 

148 parser.error(f"Unknown example name(s): {', '.join(unknown)}") 

149 return tuple(name for name in selected if name not in set(args.skip_examples)) 

150 

151 

152def _settings_from_args( 

153 parser: argparse.ArgumentParser, args: argparse.Namespace 

154) -> ExampleRunSettings: 

155 if not args.expected_account or not re.fullmatch(r"\d{12}", args.expected_account): 

156 parser.error("--expected-account must be an exact 12-digit AWS account ID") 

157 if not args.expected_sha or not re.fullmatch(r"[0-9a-fA-F]{40}", args.expected_sha): 

158 parser.error("--expected-sha must be an exact 40-character commit SHA") 

159 if not args.expected_branch or not args.expected_branch.strip(): 

160 parser.error("--expected-branch is required") 

161 if args.run_id and not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,79}", args.run_id): 

162 parser.error("--run-id must be 1-80 safe filename characters") 

163 if args.max_parallel < 0: 

164 parser.error("--max-parallel must be >= 0 (0 = all selected at once)") 

165 root = repository_root(args.repo_root) 

166 run_id = args.run_id or ( 

167 datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + args.expected_sha[:12].lower() 

168 ) 

169 report_dir = path_from_root(root, args.report_dir, Path(".example-job-validation") / run_id) 

170 checkpoint = path_from_root(root, args.checkpoint, report_dir / "checkpoint.json") 

171 protected = tuple(dict.fromkeys(("CDKToolkit", "GCOGitHubOIDCStack", *args.protected_stack))) 

172 return ExampleRunSettings( 

173 run_id=run_id, 

174 repo_root=root, 

175 report_dir=report_dir, 

176 checkpoint_path=checkpoint, 

177 expected_account=args.expected_account, 

178 expected_sha=args.expected_sha.lower(), 

179 expected_branch=args.expected_branch.strip(), 

180 profile="configured", 

181 requested_actions=args.actions, 

182 protected_stack_names=protected, 

183 confirm_kms_key_deletion=args.confirm_kms_key_deletion, 

184 resume=args.resume, 

185 selected_examples=_select_examples(parser, args), 

186 max_parallel_examples=args.max_parallel, 

187 ) 

188 

189 

190def _run_static_only(parser: argparse.ArgumentParser, args: argparse.Namespace) -> int: 

191 root = repository_root(args.repo_root) 

192 names = list(_select_examples(parser, args)) 

193 findings = run_static_checks(root, names) 

194 failed = [finding for finding in findings if not finding.passed] 

195 for finding in findings: 

196 marker = "ok " if finding.passed else "FAIL" 

197 detail = f" — {finding.detail}" if finding.detail else "" 

198 print(f"[{marker}] {finding.example}: {finding.check}{detail}") 

199 print(f"{len(findings)} checks, {len(failed)} failed") 

200 return 1 if failed else 0 

201 

202 

203def main() -> int: 

204 parser = _build_parser() 

205 args = parser.parse_args() 

206 if args.list_actions: 

207 for definition in build_action_registry().values(): 

208 dependencies = ", ".join(definition.dependencies) or "none" 

209 print(f"{definition.name:16} {definition.description} [depends: {dependencies}]") 

210 return 0 

211 if args.static_only: 

212 return _run_static_only(parser, args) 

213 try: 

214 require_local_execution() 

215 except RuntimeError as exc: 

216 print(f"Example validation could not start: {exc}", file=sys.stderr) 

217 return 1 

218 settings: ExampleRunSettings | None = None 

219 try: 

220 settings = _settings_from_args(parser, args) 

221 runner = LiveValidationRunner(settings, registry=build_action_registry()) 

222 runner.report.title = REPORT_TITLE 

223 runner.report.report_stem = REPORT_STEM 

224 return runner.run() 

225 except KeyboardInterrupt: 

226 print("Example validation interrupted before the runner initialized", file=sys.stderr) 

227 return 130 

228 except BaseException as exc: 

229 print(f"Example validation could not start: {type(exc).__name__}: {exc}", file=sys.stderr) 

230 if settings is not None: 

231 report = ValidationReport( 

232 run_id=settings.run_id, 

233 identity=settings.identity(), 

234 selected_actions=list(settings.requested_actions), 

235 started_at=utc_now(), 

236 ended_at=utc_now(), 

237 status="failed", 

238 fatal_error="".join(traceback.format_exception(type(exc), exc, exc.__traceback__)), 

239 title=REPORT_TITLE, 

240 report_stem=REPORT_STEM, 

241 ) 

242 with contextlib.suppress(OSError): 

243 report.write(settings.report_dir) 

244 return 1 

245 

246 

247if __name__ == "__main__": 

248 sys.exit(main())