Coverage for .github / scripts / validate_helm_charts.py: 100.00%
517 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-09-14 22:07 +0000
1"""Validate every Helm chart (name, version) pinned in charts.yaml.
3Confirms that each chart entry in ``lambda/helm-installer/charts.yaml`` is a
4real, installable Helm chart at the pinned version — i.e. the
5``(chart, version)`` combination is something Helm can actually pull and
6render for a cluster, not a typo or a tag that never shipped. The
7helm-installer Lambda trusts these pins blindly at deploy time, so a bad pin
8today only surfaces as a failed ``helm upgrade --install`` mid-deploy; this
9check moves that failure left into CI.
11Two layers, matching what each CI stage can afford:
13 Structural (offline, always): pure-Python checks on the parsed YAML — no
14 network, no ``helm`` binary. Every entry needs a ``chart``, a SemVer-ish
15 ``version`` and a ``repo_url``; an ``oci://`` ``repo_url`` must set
16 ``use_oci: true`` (and vice-versa); a classic HTTP(S) repo needs a
17 ``repo_name`` for ``helm repo add``. Catches the obvious mistakes before
18 spending a network round-trip.
20 Online (needs ``helm`` + network): for every chart, build the *same*
21 reference the installer Lambda builds (see ``handler.install_chart``),
22 then
24 * ``helm show chart <ref> --version <ver>`` — proves Helm can pull the
25 chart metadata at exactly the pinned version; and
26 * ``helm template <ref> --version <ver> --values <configured>`` —
27 proves the chart renders to Kubernetes manifests (installable) with
28 the values ``charts.yaml`` ships.
30 Two retry layers ride out intermittent registry failures instead of
31 forcing a manual job rerun. Inner: every network-touching helm call
32 (repo add/update, show chart, template) goes through ``_run_with_retry``
33 — a fixed number of attempts with exponential backoff, first success
34 wins. Outer: charts still failing after the first sweep get a bounded
35 number of re-passes (default one more, ~30s later) behind a fresh
36 ``helm repo add`` + index refresh, covering outages that outlast a
37 single command's retry window. A genuinely bad pin fails every attempt
38 of every pass and still surfaces.
40By default *every* chart in the file is validated, including entries with
41``enabled: false``: those are toggled on via ``cdk.json``, so their pinned
42``(name, version)`` must be valid too.
44Usage::
46 # Structural only (no helm needed):
47 python3 .github/scripts/validate_helm_charts.py --mode offline
49 # Full check (requires helm on PATH + network) — the CI gate:
50 python3 .github/scripts/validate_helm_charts.py --mode online
52 # Auto: structural always, online when helm happens to be installed:
53 python3 .github/scripts/validate_helm_charts.py
55 # Point at a non-default charts.yaml (used by the test suite):
56 python3 .github/scripts/validate_helm_charts.py --charts /path/to/charts.yaml
58 # Emit one chart's pinned reference / shipped values (consumed by the
59 # integration:kind:examples-smoke job so its `helm install` uses the
60 # exact pins and values the installer Lambda would — no copies in CI):
61 python3 .github/scripts/validate_helm_charts.py --emit-ref mlflow
62 python3 .github/scripts/validate_helm_charts.py --emit-values mlflow
64Exit codes::
66 0 all validated charts are well-formed (and, when online, resolvable
67 + renderable at their pinned versions)
68 1 one or more charts failed validation
69 2 unexpected I/O / argument error (charts.yaml missing or unparseable,
70 or --mode online requested without a helm binary)
72The module is importable from the test suite — call ``validate_structure()``,
73``build_refs()`` or ``validate_online()`` directly to exercise the logic
74against fixtures.
75"""
77from __future__ import annotations
79import argparse
80import contextlib
81import copy
82import os
83import re
84import shutil
85import subprocess # nosec B404 - used only to invoke the pinned `helm` binary with fixed argv
86import sys
87import tempfile
88import time
89from dataclasses import dataclass
90from pathlib import Path
91from typing import Any
93import yaml
95# charts.yaml lives next to the helm-installer Lambda handler. This script is
96# .github/scripts/validate_helm_charts.py, so the repo root is two parents up.
97_REPO_ROOT = Path(__file__).resolve().parents[2]
98_DEFAULT_CHARTS = _REPO_ROOT / "lambda" / "helm-installer" / "charts.yaml"
100# Lenient SemVer-ish matcher for the pinned chart ``version``. Helm requires
101# SemVer2 chart versions, so anything failing this is guaranteed to fail a real
102# ``helm ... --version`` resolve too — we just catch it offline first. Accepts
103# an optional leading "v" (cert-manager / aws-efa tag their charts that way)
104# and an optional pre-release / build-metadata suffix.
105_VERSION_RE = re.compile(r"^v?\d+(?:\.\d+){1,2}(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$")
108@dataclass(frozen=True)
109class ChartRef:
110 """Everything needed to resolve one ``charts.yaml`` entry with Helm."""
112 name: str
113 chart: str
114 version: str
115 repo_name: str
116 repo_url: str
117 use_oci: bool
118 namespace: str
119 enabled: bool
120 values: dict[str, Any]
122 def reference(self) -> str:
123 """Build the Helm chart reference exactly like ``handler.install_chart``.
125 OCI charts are addressed by their full ``oci://.../<chart>`` URL;
126 classic HTTP(S) repos are addressed as ``<repo_name>/<chart>`` after
127 the repo has been added with ``helm repo add``.
128 """
129 if self.use_oci:
130 return f"{self.repo_url}/{self.chart}"
131 return f"{self.repo_name}/{self.chart}"
134def load_charts(path: Path) -> dict[str, Any]:
135 """Parse ``charts.yaml`` and return the ``charts:`` mapping.
137 Raises ``FileNotFoundError`` when the file is absent and ``ValueError``
138 when the top-level ``charts:`` mapping is missing or malformed; ``main()``
139 turns both into exit code 2.
140 """
141 with open(path, encoding="utf-8") as fh:
142 data = yaml.safe_load(fh) or {}
143 charts = data.get("charts")
144 if not isinstance(charts, dict):
145 raise ValueError("missing or malformed top-level 'charts:' mapping")
146 return charts
149def validate_structure(charts: dict[str, Any], *, enabled_only: bool = False) -> list[str]:
150 """Return a list of structural problems (empty == all well-formed).
152 These are the offline, pure-Python checks: enough to guarantee we can even
153 build a Helm reference for each entry and that OCI/classic metadata is
154 self-consistent. Every returned string names the offending chart so the
155 failure is actionable from the CI log alone.
156 """
157 errors: list[str] = []
158 if not charts:
159 return ["charts.yaml contains no chart entries under 'charts:'"]
161 for name, cfg in charts.items():
162 if not isinstance(cfg, dict):
163 errors.append(f"{name}: entry is not a mapping")
164 continue
165 if enabled_only and not cfg.get("enabled", False):
166 continue
168 chart = cfg.get("chart")
169 version = cfg.get("version")
170 repo_url = cfg.get("repo_url")
171 repo_name = cfg.get("repo_name")
172 use_oci = bool(cfg.get("use_oci", False))
174 if not (isinstance(chart, str) and chart.strip()):
175 errors.append(f"{name}: missing or empty 'chart'")
177 if not (isinstance(version, str) and version.strip()):
178 errors.append(f"{name}: missing or empty 'version'")
179 elif not _VERSION_RE.match(version.strip()):
180 errors.append(f"{name}: version {version!r} is not a valid SemVer chart version")
182 if not (isinstance(repo_url, str) and repo_url.strip()):
183 errors.append(f"{name}: missing or empty 'repo_url'")
184 else:
185 is_oci_url = repo_url.startswith("oci://")
186 if is_oci_url and not use_oci:
187 errors.append(f"{name}: repo_url is oci:// but use_oci is not set to true")
188 if use_oci and not is_oci_url:
189 errors.append(
190 f"{name}: use_oci is true but repo_url {repo_url!r} is not an oci:// URL"
191 )
192 if not use_oci and not is_oci_url:
193 if not repo_url.startswith(("http://", "https://")):
194 errors.append(
195 f"{name}: classic repo_url {repo_url!r} must be http(s):// or oci://"
196 )
197 if not (isinstance(repo_name, str) and repo_name.strip()):
198 errors.append(f"{name}: non-OCI chart needs a 'repo_name' for 'helm repo add'")
200 return errors
203def build_refs(charts: dict[str, Any], *, enabled_only: bool = False) -> list[ChartRef]:
204 """Build a ``ChartRef`` for every entry well-formed enough to resolve.
206 Entries too malformed to build a reference (no chart / version / repo_url,
207 or a classic repo with no repo_name) are skipped here — ``validate_structure``
208 already reports them, so there is no point trying to hit Helm for them.
209 """
210 refs: list[ChartRef] = []
211 for name, cfg in charts.items():
212 if not isinstance(cfg, dict):
213 continue
214 if enabled_only and not cfg.get("enabled", False):
215 continue
217 chart = cfg.get("chart")
218 version = cfg.get("version")
219 repo_url = cfg.get("repo_url")
220 use_oci = bool(cfg.get("use_oci", False))
221 repo_name = cfg.get("repo_name") or ""
223 if not (isinstance(chart, str) and chart.strip()):
224 continue
225 if not (isinstance(version, str) and version.strip()):
226 continue
227 if not (isinstance(repo_url, str) and repo_url.strip()):
228 continue
229 if not use_oci and not str(repo_name).strip():
230 continue
232 values = cfg.get("values")
233 refs.append(
234 ChartRef(
235 name=str(name),
236 chart=chart.strip(),
237 version=version.strip(),
238 repo_name=str(repo_name).strip(),
239 repo_url=repo_url.strip(),
240 use_oci=use_oci,
241 namespace=str(cfg.get("namespace", "default")),
242 enabled=bool(cfg.get("enabled", False)),
243 values=values if isinstance(values, dict) else {},
244 )
245 )
246 return refs
249def _run(cmd: list[str], env: dict[str, str], *, timeout: int = 120) -> tuple[int, str, str]:
250 """Run a command with a fixed argv (never a shell), returning (rc, out, err).
252 A subprocess timeout maps to ``(-1, "", "timeout: ...")`` so callers get a
253 uniform failure contract instead of an exception — mirrors the same pattern
254 in the helm-installer Lambda's ``run_helm``.
255 """
256 try:
257 proc = (
258 subprocess.run( # nosemgrep: dangerous-subprocess-use-audit - fixed argv, no shell=True
259 cmd,
260 capture_output=True,
261 text=True,
262 env=env,
263 timeout=timeout,
264 check=False,
265 )
266 )
267 except subprocess.TimeoutExpired as exc:
268 return -1, "", f"timeout: command exceeded {exc.timeout}s"
269 return proc.returncode, proc.stdout, proc.stderr
272def _run_with_retry(
273 cmd: list[str],
274 env: dict[str, str],
275 *,
276 attempts: int = 4,
277 base_delay: float = 2.0,
278 max_delay: float = 20.0,
279 timeout: int = 120,
280 verbose: bool = False,
281 description: str = "",
282) -> tuple[int, str, str]:
283 """Run a network-touching helm command up to ``attempts`` times; first success wins.
285 The command is retried on any non-zero exit, regardless of why it failed:
286 the first attempt that succeeds (rc 0) is returned immediately, and if none
287 do, the last failure is returned. Between attempts we sleep an exponentially
288 growing delay (``base_delay`` doubling each round, capped at ``max_delay``)
289 to give a blipping registry a moment to recover.
291 This is the guard against intermittent registry failures (timeouts, resets,
292 5xx) that otherwise force a manual rerun of the ``integration:helm:charts-valid``
293 job. A genuinely bad pin fails on every attempt and still surfaces, just a
294 few seconds later.
295 """
296 result: tuple[int, str, str] = (1, "", "")
297 for attempt in range(1, attempts + 1):
298 result = _run(cmd, env, timeout=timeout)
299 if result[0] == 0:
300 return result
301 if attempt < attempts:
302 delay = min(base_delay * (2 ** (attempt - 1)), max_delay)
303 if verbose:
304 label = description or " ".join(cmd)
305 print(
306 f" attempt {attempt}/{attempts} failed, retrying in "
307 f"{delay:.1f}s — {label}: {_tail(result[2], 200)}"
308 )
309 time.sleep(delay)
310 return result
313def _helm_env(helm_home: Path) -> dict[str, str]:
314 """Return an environment that isolates Helm's cache/config/data in a temp dir.
316 Keeps the check hermetic: it never reads or clobbers a developer's real
317 ``helm repo`` list, and CI starts from a clean slate every run.
318 """
319 env = os.environ.copy()
320 env["HELM_CACHE_HOME"] = str(helm_home / "cache")
321 env["HELM_CONFIG_HOME"] = str(helm_home / "config")
322 env["HELM_DATA_HOME"] = str(helm_home / "data")
323 return env
326def _tail(text: str, limit: int = 400) -> str:
327 """Trim helm stderr to the last ``limit`` chars so reports stay readable."""
328 text = (text or "").strip()
329 if len(text) <= limit:
330 return text
331 return "..." + text[-limit:]
334def _chart_version_from_show(show_output: str) -> str | None:
335 """Extract the ``version`` field from ``helm show chart`` YAML output."""
336 try:
337 meta = yaml.safe_load(show_output)
338 except yaml.YAMLError:
339 return None
340 if isinstance(meta, dict):
341 version = meta.get("version")
342 if isinstance(version, str):
343 return version
344 return None
347def _versions_match(resolved: str, requested: str) -> bool:
348 """Compare versions ignoring a leading ``v`` on either side."""
349 return resolved.lstrip("v") == requested.lstrip("v")
352def _render_chart(
353 ref: ChartRef,
354 ref_str: str,
355 helm_binary: str,
356 env: dict[str, str],
357 *,
358 verbose: bool = False,
359) -> str | None:
360 """``helm template`` the chart with its shipped values; return an error or None.
362 Rendering with the exact ``values`` block from ``charts.yaml`` proves the
363 chart is installable *as GCO configures it*, not just with upstream
364 defaults. No cluster is contacted, but templating a remote ``--version``
365 ref still pulls the chart from its registry, so it goes through
366 ``_run_with_retry`` to ride out the same transient blips as the resolve
367 step.
368 """
369 args = [
370 helm_binary,
371 "template",
372 ref.name,
373 ref_str,
374 "--version",
375 ref.version,
376 "--namespace",
377 ref.namespace,
378 ]
380 values_path: str | None = None
381 if ref.values:
382 fd, values_path = tempfile.mkstemp(suffix=".yaml", prefix=f"{ref.name}-values-")
383 try:
384 # fdopen takes ownership of fd; the with-block closes it on any exit,
385 # so a failing dump must NOT close it again (that raised EBADF and
386 # masked the real error). Remove the half-written file instead.
387 with os.fdopen(fd, "w", encoding="utf-8") as fh:
388 yaml.safe_dump(ref.values, fh)
389 except Exception:
390 with contextlib.suppress(OSError):
391 os.remove(values_path)
392 raise
393 args.extend(["--values", values_path])
395 try:
396 rc, _out, err = _run_with_retry(
397 args,
398 env,
399 timeout=180,
400 verbose=verbose,
401 description=f"helm template {ref.name}",
402 )
403 finally:
404 if values_path:
405 with contextlib.suppress(OSError):
406 os.remove(values_path)
408 if rc != 0:
409 return f"chart failed to render (helm template): {_tail(err)}"
410 return None
413def _sync_classic_repos(
414 classic: dict[str, str],
415 helm_binary: str,
416 env: dict[str, str],
417 *,
418 verbose: bool = False,
419) -> list[str]:
420 """``helm repo add`` every classic repo and refresh the index once.
422 Returns error strings for repos that could not be added. The index
423 refresh is network-bound too — it goes through the same retry so a blip
424 there doesn't cascade into spurious "cannot resolve" errors downstream.
425 """
426 errors: list[str] = []
427 for repo_name, repo_url in classic.items():
428 rc, _out, err = _run_with_retry(
429 [helm_binary, "repo", "add", repo_name, repo_url, "--force-update"],
430 env,
431 verbose=verbose,
432 description=f"helm repo add {repo_name}",
433 )
434 if rc != 0:
435 errors.append(f"helm repo add {repo_name} ({repo_url}) failed: {_tail(err)}")
436 if classic:
437 _run_with_retry(
438 [helm_binary, "repo", "update"],
439 env,
440 timeout=180,
441 verbose=verbose,
442 description="helm repo update",
443 )
444 return errors
447def _validate_refs(
448 refs: list[ChartRef],
449 helm_binary: str,
450 env: dict[str, str],
451 *,
452 skip_template: bool = False,
453 verbose: bool = False,
454) -> dict[str, list[str]]:
455 """Resolve + render each chart; return ``{chart name: [errors]}`` for failures."""
456 failures: dict[str, list[str]] = {}
457 for ref in refs:
458 ref_str = ref.reference()
459 label = f"{ref.name} {ref.version} ({ref_str})"
460 chart_errors: list[str] = []
462 rc, out, err = _run_with_retry(
463 [helm_binary, "show", "chart", ref_str, "--version", ref.version],
464 env,
465 verbose=verbose,
466 description=f"helm show chart {ref.name}",
467 )
468 if rc != 0:
469 chart_errors.append(
470 f"{ref.name}: helm cannot resolve {ref_str} at version "
471 f"{ref.version!r}: {_tail(err)}"
472 )
473 if verbose:
474 print(f"FAIL resolve {label}")
475 failures[ref.name] = chart_errors
476 continue
478 resolved = _chart_version_from_show(out)
479 if resolved and not _versions_match(resolved, ref.version):
480 chart_errors.append(
481 f"{ref.name}: requested version {ref.version!r} but helm resolved {resolved!r}"
482 )
483 if verbose:
484 print(f"ok resolve {label}")
486 if not skip_template:
487 render_error = _render_chart(ref, ref_str, helm_binary, env, verbose=verbose)
488 if render_error:
489 chart_errors.append(f"{ref.name}: {render_error}")
490 if verbose:
491 print(f"FAIL render {label}")
492 elif verbose:
493 print(f"ok render {label}")
495 if chart_errors:
496 failures[ref.name] = chart_errors
497 return failures
500def validate_online(
501 refs: list[ChartRef],
502 *,
503 helm_binary: str = "helm",
504 skip_template: bool = False,
505 verbose: bool = False,
506 passes: int = 2,
507 repass_delay: float = 30.0,
508) -> list[str]:
509 """Resolve (and, unless skipped, render) each chart at its pinned version.
511 Returns a list of human-readable error strings (empty == every chart is
512 resolvable and renderable). Runs against an isolated Helm home so it is
513 safe to invoke on a developer machine.
515 Retry model, outer layer: the per-command retry in ``_run_with_retry``
516 rides out blips that clear within one command's ~40-second attempt
517 window, but a registry outage lasting a few minutes fails several charts
518 on every inner attempt and used to fail the job (observed live: a rerun
519 of the unchanged job passed). So after the first sweep, the charts that
520 failed get up to ``passes - 1`` additional sweeps, each preceded by a
521 ``repass_delay`` pause and a fresh repo add + index refresh. Only
522 failures that survive every pass are reported; a genuinely bad pin fails
523 every pass and still surfaces.
524 """
525 errors: list[str] = []
526 if not refs:
527 return errors
529 with tempfile.TemporaryDirectory(prefix="gco-helm-validate-") as tmp:
530 env = _helm_env(Path(tmp))
531 classic = {ref.repo_name: ref.repo_url for ref in refs if not ref.use_oci}
533 repo_errors = _sync_classic_repos(classic, helm_binary, env, verbose=verbose)
534 failures = _validate_refs(
535 refs, helm_binary, env, skip_template=skip_template, verbose=verbose
536 )
538 for extra_pass in range(2, max(passes, 1) + 1):
539 if not failures and not repo_errors:
540 break
541 if verbose:
542 print(
543 f"re-pass {extra_pass}/{passes}: retrying "
544 f"{len(failures)} failed chart(s) in {repass_delay:.0f}s "
545 "(fresh repo index)"
546 )
547 time.sleep(repass_delay)
548 repo_errors = _sync_classic_repos(classic, helm_binary, env, verbose=verbose)
549 retry_refs = [ref for ref in refs if ref.name in failures]
550 failures = _validate_refs(
551 retry_refs, helm_binary, env, skip_template=skip_template, verbose=verbose
552 )
554 errors.extend(repo_errors)
555 for ref in refs:
556 errors.extend(failures.get(ref.name, []))
558 return errors
561# ---------------------------------------------------------------------------
562# Gateway API / aws-load-balancer-controller lockstep
563#
564# The controller is built against an exact ``sigs.k8s.io/gateway-api`` release
565# (declared in its go.mod) and its own gateway CRDs ship per controller tag.
566# GCO pins three coupled artifacts in two different files:
567#
568# * the aws-load-balancer-controller chart version (charts.yaml, this file's
569# usual input),
570# * the ``gateway-api-standard-vX.Y.Z`` CRD bundle, and
571# * the ``aws-lbc-gateway-vX.Y.Z`` CRD bundle
572# (both in lambda/helm-installer/handler.py PINNED_GATEWAY_CRD_BUNDLES).
573#
574# When the chart moved to 3.5.0 while the standard bundle stayed at v1.5.0,
575# the controller silently stopped reconciling gateways — nothing failed until
576# the live release validation deployed the pair. These checks encode the
577# contract so the drift fails CI instead:
578#
579# offline: the aws-lbc-gateway bundle version must equal the pinned chart
580# version (they ship from the same controller tag).
581# online: the controller tag's go.mod names its required gateway-api
582# release; the pinned standard bundle must be at least that (major.minor).
583# ---------------------------------------------------------------------------
585_HANDLER_PATH = _REPO_ROOT / "lambda" / "helm-installer" / "handler.py"
586_LBC_CHART_KEY = "aws-load-balancer-controller"
587_GATEWAY_API_BUNDLE_RE = re.compile(r'name="gateway-api-standard-v(\d+\.\d+\.\d+)"')
588_LBC_BUNDLE_RE = re.compile(r'name="aws-lbc-gateway-v(\d+\.\d+\.\d+)"')
589_GO_MOD_GATEWAY_API_RE = re.compile(r"^\s*sigs\.k8s\.io/gateway-api\s+v(\d+\.\d+\.\d+)", re.M)
590_LBC_GO_MOD_URL_TEMPLATE = (
591 "https://raw.githubusercontent.com/kubernetes-sigs/"
592 "aws-load-balancer-controller/v{version}/go.mod"
593)
594_GO_MOD_FETCH_ATTEMPTS = 3
595_GO_MOD_FETCH_TIMEOUT_SECONDS = 15
598def parse_pinned_gateway_bundles(handler_source: str) -> tuple[str | None, str | None]:
599 """Return (gateway-api standard bundle version, aws-lbc bundle version)."""
600 gateway_api = _GATEWAY_API_BUNDLE_RE.search(handler_source)
601 lbc = _LBC_BUNDLE_RE.search(handler_source)
602 return (
603 gateway_api.group(1) if gateway_api else None,
604 lbc.group(1) if lbc else None,
605 )
608def gateway_api_requirement_from_go_mod(go_mod_text: str) -> str | None:
609 """Return the gateway-api release the controller's go.mod declares."""
610 match = _GO_MOD_GATEWAY_API_RE.search(go_mod_text)
611 return match.group(1) if match else None
614def _version_tuple(version: str) -> tuple[int, ...]:
615 return tuple(int(part) for part in version.split("."))
618def fetch_lbc_go_mod(controller_version: str) -> str:
619 """Fetch the controller tag's go.mod, retrying transient HTTP failures."""
620 import urllib.error
621 import urllib.request
623 if not re.fullmatch(r"\d+\.\d+\.\d+", controller_version):
624 raise RuntimeError(
625 f"refusing go.mod fetch for non-semver controller version {controller_version!r}"
626 )
627 url = _LBC_GO_MOD_URL_TEMPLATE.format(version=controller_version)
628 last_error: Exception | None = None
629 for attempt in range(1, _GO_MOD_FETCH_ATTEMPTS + 1):
630 try:
631 with urllib.request.urlopen( # nosec B310 # nosemgrep: dynamic-urllib-use-detected - fixed https://raw.githubusercontent.com template; the only variable is a strictly semver-validated version segment, so no scheme or host injection is possible # noqa: S310
632 url, timeout=_GO_MOD_FETCH_TIMEOUT_SECONDS
633 ) as response:
634 return str(response.read().decode("utf-8"))
635 except (urllib.error.URLError, TimeoutError, OSError) as exc:
636 last_error = exc
637 if attempt < _GO_MOD_FETCH_ATTEMPTS:
638 time.sleep(2**attempt)
639 raise RuntimeError(f"could not fetch {url}: {last_error}")
642def validate_gateway_lockstep(
643 charts: dict[str, Any],
644 *,
645 handler_source: str | None = None,
646 go_mod_fetcher: Any = None,
647 online: bool = False,
648 require_entry: bool = True,
649) -> list[str]:
650 """Check the chart / CRD-bundle lockstep contract; return error strings.
652 Offline: the ``aws-lbc-gateway`` bundle version must equal the pinned
653 chart version. Online (adds one HTTPS fetch): the pinned standard
654 gateway-api bundle must satisfy the requirement in the controller tag's
655 go.mod. With ``require_entry`` (the default, used for the repository's
656 real charts.yaml) a missing chart entry or renamed handler constants are
657 reported rather than skipped — this guard exists precisely for refactors
658 that move them. ``main()`` disables ``require_entry`` for explicitly
659 supplied ``--charts`` fixture files that legitimately omit the entry.
660 """
661 errors: list[str] = []
662 entry = charts.get(_LBC_CHART_KEY)
663 if not isinstance(entry, dict) or not entry.get("version"):
664 if require_entry:
665 return [f"gateway lockstep: no {_LBC_CHART_KEY!r} entry with a version in charts.yaml"]
666 return []
667 chart_version = str(entry["version"]).lstrip("v")
669 if handler_source is None:
670 try:
671 handler_source = _HANDLER_PATH.read_text(encoding="utf-8")
672 except OSError as exc:
673 return [f"gateway lockstep: cannot read {_HANDLER_PATH}: {exc}"]
675 gateway_api_pin, lbc_pin = parse_pinned_gateway_bundles(handler_source)
676 if gateway_api_pin is None or lbc_pin is None:
677 return [
678 "gateway lockstep: PINNED_GATEWAY_CRD_BUNDLES in "
679 "lambda/helm-installer/handler.py no longer names a "
680 "'gateway-api-standard-vX.Y.Z' and an 'aws-lbc-gateway-vX.Y.Z' "
681 "bundle; update this check alongside any rename"
682 ]
684 if lbc_pin != chart_version:
685 errors.append(
686 f"gateway lockstep: aws-lbc-gateway CRD bundle v{lbc_pin} does not match "
687 f"the pinned {_LBC_CHART_KEY} chart {chart_version} — the controller's "
688 "gateway CRDs ship per controller tag; bump both together "
689 "(charts.yaml + PINNED_GATEWAY_CRD_BUNDLES in lambda/helm-installer/handler.py)"
690 )
692 if not online:
693 return errors
695 fetcher = go_mod_fetcher or fetch_lbc_go_mod
696 try:
697 go_mod_text = fetcher(chart_version)
698 except RuntimeError as exc:
699 errors.append(f"gateway lockstep: {exc}")
700 return errors
702 required = gateway_api_requirement_from_go_mod(go_mod_text)
703 if required is None:
704 errors.append(
705 f"gateway lockstep: go.mod for {_LBC_CHART_KEY} v{chart_version} does not "
706 "declare sigs.k8s.io/gateway-api — upstream layout changed; update this check"
707 )
708 return errors
710 if _version_tuple(gateway_api_pin)[:2] < _version_tuple(required)[:2]:
711 errors.append(
712 f"gateway lockstep: {_LBC_CHART_KEY} {chart_version} is built against "
713 f"gateway-api v{required} (its go.mod), but the pinned standard CRD bundle "
714 f"is v{gateway_api_pin}. Upgrading the controller without its Gateway API "
715 "CRDs silently stops gateway reconciliation (caught live, 2026-08). Bump "
716 "'gateway-api-standard' in lambda/helm-installer/handler.py to at least "
717 f"v{required} in the same change"
718 )
719 return errors
722# ---------------------------------------------------------------------------
723# Kubeflow Trainer runtime / example / docs lockstep
724#
725# The kubeflow-trainer chart delivers its built-in ClusterTrainingRuntime
726# blueprints through a post-install hook Job (an unpinned run-time network
727# fetch), so GCO disables that hook and ships the ``torch-distributed``
728# runtime itself — extracted verbatim from the pinned chart into
729# ``post-helm-kubeflow-trainer-runtimes.yaml`` with two documented
730# deviations (``automountServiceAccountToken: false`` and NoNewPrivs on the
731# ``node`` container). Three more surfaces
732# repeat the runtime's pinned trainer image so users see exactly what runs:
733#
734# * the shipped runtime manifest (the source of truth in-repo),
735# * ``examples/kubeflow-trainjob.yaml`` ``spec.trainer.image`` (listed
736# explicitly so the image-trust gate validates it), and
737# * the ``docs/DISTRIBUTED_TRAINING.md`` TrainJob snippet.
738#
739# A chart version bump that skips re-extraction would silently run a stale
740# runtime (or torchrun wiring) against a newer controller. These checks
741# encode the contract so the drift fails CI instead:
742#
743# offline: the shipped runtime manifest, the example and every
744# ``pytorch/pytorch`` image mentioned in the distributed-training doc
745# agree on one image.
746# online: rendering the *pinned* chart with its runtime delivery enabled
747# must reproduce the shipped runtime — image called out explicitly,
748# full spec compared after applying the documented deviations, and the
749# upstream ``trainer.kubeflow.org/*`` labels preserved (the
750# ``webhook-validation: disabled`` label is what keeps webhook warm-up
751# from flaking the apply).
752# ---------------------------------------------------------------------------
754_TRAINER_CHART_KEY = "kubeflow-trainer"
755_TRAINER_RUNTIME_NAME = "torch-distributed"
756_TRAINER_RUNTIME_MANIFEST = (
757 _REPO_ROOT
758 / "lambda"
759 / "kubectl-applier-simple"
760 / "manifests"
761 / "post-helm-kubeflow-trainer-runtimes.yaml"
762)
763_TRAINJOB_EXAMPLE = _REPO_ROOT / "examples" / "kubeflow-trainjob.yaml"
764_DISTRIBUTED_TRAINING_DOC = _REPO_ROOT / "docs" / "DISTRIBUTED_TRAINING.md"
765# Only pytorch/pytorch mentions are lockstep-bound: the doc may legitimately
766# show other registries' images, but a pytorch/pytorch tag that differs from
767# the shipped runtime is exactly the stale-doc drift this check exists for.
768_DOC_PYTORCH_IMAGE_RE = re.compile(r"image:\s*(pytorch/pytorch:\S+)")
769# The runtime labels GCO must carry verbatim: framework selection and the
770# upstream pre-validation marker that exempts the built-in runtime from
771# webhook admission (losing it reintroduces webhook warm-up flakes).
772_TRAINER_RUNTIME_LOCKSTEP_LABELS = (
773 "trainer.kubeflow.org/framework",
774 "trainer.kubeflow.org/webhook-validation",
775)
778def parse_shipped_torch_runtime(manifest_text: str) -> dict[str, Any] | None:
779 """Return the one ``torch-distributed`` runtime in the shipped manifest.
781 ``None`` when the manifest does not contain exactly one
782 ``ClusterTrainingRuntime`` named ``torch-distributed`` — the caller turns
783 that into an "update this check" error rather than guessing.
784 """
785 try:
786 docs = [doc for doc in yaml.safe_load_all(manifest_text) if isinstance(doc, dict)]
787 except yaml.YAMLError:
788 return None
789 runtimes = [
790 doc
791 for doc in docs
792 if doc.get("kind") == "ClusterTrainingRuntime"
793 and (doc.get("metadata") or {}).get("name") == _TRAINER_RUNTIME_NAME
794 ]
795 return runtimes[0] if len(runtimes) == 1 else None
798def upstream_torch_runtime_from_render(render_text: str) -> dict[str, Any] | None:
799 """Extract the ``torch-distributed`` runtime from a rendered chart.
801 The chart ships its runtimes as multi-doc YAML inside the
802 ``*runtimes-installer`` ConfigMap's ``runtimes.yaml`` key (the payload its
803 hook Job would kubectl-apply); this digs the runtime out of that payload.
804 """
805 try:
806 docs = [doc for doc in yaml.safe_load_all(render_text) if isinstance(doc, dict)]
807 except yaml.YAMLError:
808 return None
809 for doc in docs:
810 if doc.get("kind") != "ConfigMap":
811 continue
812 name = str((doc.get("metadata") or {}).get("name") or "")
813 if not name.endswith("runtimes-installer"):
814 continue
815 payload = (doc.get("data") or {}).get("runtimes.yaml")
816 if not isinstance(payload, str):
817 continue
818 try:
819 runtimes = [item for item in yaml.safe_load_all(payload) if isinstance(item, dict)]
820 except yaml.YAMLError:
821 return None
822 for runtime in runtimes:
823 if (
824 runtime.get("kind") == "ClusterTrainingRuntime"
825 and (runtime.get("metadata") or {}).get("name") == _TRAINER_RUNTIME_NAME
826 ):
827 return runtime
828 return None
831def trainer_node_image(runtime: dict[str, Any]) -> str | None:
832 """Return the trainer image of a runtime's ``node`` replicated Job."""
833 template_spec = ((runtime.get("spec") or {}).get("template") or {}).get("spec") or {}
834 jobs = template_spec.get("replicatedJobs")
835 if not isinstance(jobs, list):
836 return None
837 for job in jobs:
838 if not isinstance(job, dict) or job.get("name") != "node":
839 continue
840 pod_spec = (((job.get("template") or {}).get("spec") or {}).get("template") or {}).get(
841 "spec"
842 ) or {}
843 containers = pod_spec.get("containers")
844 if not isinstance(containers, list):
845 return None
846 for container in containers:
847 if isinstance(container, dict) and container.get("name") == "node":
848 image = container.get("image")
849 return image if isinstance(image, str) else None
850 return None
853def example_trainer_image(example_text: str) -> str | None:
854 """Return ``spec.trainer.image`` from the TrainJob example manifest."""
855 try:
856 docs = [doc for doc in yaml.safe_load_all(example_text) if isinstance(doc, dict)]
857 except yaml.YAMLError:
858 return None
859 for doc in docs:
860 if doc.get("kind") == "TrainJob":
861 image = ((doc.get("spec") or {}).get("trainer") or {}).get("image")
862 return image if isinstance(image, str) else None
863 return None
866def doc_pytorch_images(doc_text: str) -> list[str]:
867 """Return every ``pytorch/pytorch`` image the doc's snippets mention."""
868 return _DOC_PYTORCH_IMAGE_RE.findall(doc_text)
871def _apply_documented_runtime_deviations(upstream_spec: dict[str, Any]) -> dict[str, Any]:
872 """Return the upstream runtime spec with GCO's sanctioned deviations applied.
874 Exactly two deviations from verbatim extraction are documented in the
875 manifest header, both on the ``node`` pod template:
877 - ``automountServiceAccountToken: false`` (the same security default
878 both submission paths inject into user Jobs), and
879 - ``securityContext.allowPrivilegeEscalation: false`` on the ``node``
880 container (NoNewPrivs; the platform's manifest policy already rejects
881 an explicit ``true``).
883 Anything else that differs from upstream is drift and must fail — a new
884 deliberate deviation belongs here *and* in the manifest header, in the
885 same change.
886 """
887 adjusted = copy.deepcopy(upstream_spec)
888 jobs = ((adjusted.get("template") or {}).get("spec") or {}).get("replicatedJobs")
889 for job in jobs if isinstance(jobs, list) else []:
890 if not isinstance(job, dict) or job.get("name") != "node":
891 continue
892 pod_spec = (((job.get("template") or {}).get("spec") or {}).get("template") or {}).get(
893 "spec"
894 )
895 if not isinstance(pod_spec, dict):
896 continue
897 pod_spec["automountServiceAccountToken"] = False
898 for container in pod_spec.get("containers") or []:
899 if isinstance(container, dict) and container.get("name") == "node":
900 security = container.setdefault("securityContext", {})
901 if isinstance(security, dict):
902 security["allowPrivilegeEscalation"] = False
903 return adjusted
906def fetch_upstream_torch_runtime(
907 entry: dict[str, Any], helm_binary: str = "helm"
908) -> dict[str, Any]:
909 """Render the pinned chart with runtime delivery enabled; return the runtime.
911 Runs against an isolated Helm home (same hermetic setup as the resolve /
912 render pass) and rides the shared retry ladder, so registry blips do not
913 fail the job. Raises ``RuntimeError`` with an actionable message when the
914 chart cannot be rendered or no longer ships the runtime where this check
915 expects it.
916 """
917 refs = build_refs({_TRAINER_CHART_KEY: entry})
918 if not refs:
919 raise RuntimeError(
920 f"cannot build a Helm reference from the {_TRAINER_CHART_KEY!r} charts.yaml entry"
921 )
922 ref = refs[0]
923 with tempfile.TemporaryDirectory(prefix="gco-trainer-lockstep-") as tmp:
924 env = _helm_env(Path(tmp))
925 if not ref.use_oci:
926 _sync_classic_repos({ref.repo_name: ref.repo_url}, helm_binary, env)
927 rc, out, err = _run_with_retry(
928 [
929 helm_binary,
930 "template",
931 ref.name,
932 ref.reference(),
933 "--version",
934 ref.version,
935 "--namespace",
936 ref.namespace,
937 "--set",
938 "runtimes.torchDistributed.enabled=true",
939 ],
940 env,
941 timeout=180,
942 description="helm template kubeflow-trainer (runtime delivery enabled)",
943 )
944 if rc != 0:
945 raise RuntimeError(
946 f"helm template {ref.reference()} at {ref.version!r} (with "
947 f"runtimes.torchDistributed.enabled=true) failed: {_tail(err)}"
948 )
949 runtime = upstream_torch_runtime_from_render(out)
950 if runtime is None:
951 raise RuntimeError(
952 f"chart {ref.version} no longer ships a 'runtimes-installer' ConfigMap "
953 f"containing ClusterTrainingRuntime {_TRAINER_RUNTIME_NAME!r} — upstream "
954 "runtime delivery changed; update this check alongside the re-extraction"
955 )
956 return runtime
959def validate_trainer_runtime_lockstep(
960 charts: dict[str, Any],
961 *,
962 manifest_text: str | None = None,
963 example_text: str | None = None,
964 doc_text: str | None = None,
965 online: bool = False,
966 runtime_fetcher: Any = None,
967 helm_binary: str = "helm",
968 require_entry: bool = True,
969) -> list[str]:
970 """Check the trainer runtime / example / docs lockstep; return error strings.
972 Offline: the shipped runtime manifest, the TrainJob example and the
973 distributed-training doc must agree on the trainer image. Online (adds
974 one chart render): the shipped runtime must reproduce what the pinned
975 chart ships — byte-identical spec after the documented deviations, with
976 the trainer image compared explicitly for an actionable message. With
977 ``require_entry`` (the default, used for the repository's real
978 charts.yaml) a missing chart entry is reported rather than skipped;
979 ``main()`` disables it for ``--charts`` fixture files.
980 """
981 errors: list[str] = []
982 entry = charts.get(_TRAINER_CHART_KEY)
983 if not isinstance(entry, dict) or not entry.get("version"):
984 if require_entry:
985 return [
986 f"trainer runtime lockstep: no {_TRAINER_CHART_KEY!r} entry with a "
987 "version in charts.yaml"
988 ]
989 return []
991 if manifest_text is None:
992 try:
993 manifest_text = _TRAINER_RUNTIME_MANIFEST.read_text(encoding="utf-8")
994 except OSError as exc:
995 return [f"trainer runtime lockstep: cannot read {_TRAINER_RUNTIME_MANIFEST}: {exc}"]
996 if example_text is None:
997 try:
998 example_text = _TRAINJOB_EXAMPLE.read_text(encoding="utf-8")
999 except OSError as exc:
1000 return [f"trainer runtime lockstep: cannot read {_TRAINJOB_EXAMPLE}: {exc}"]
1001 if doc_text is None:
1002 try:
1003 doc_text = _DISTRIBUTED_TRAINING_DOC.read_text(encoding="utf-8")
1004 except OSError as exc:
1005 return [f"trainer runtime lockstep: cannot read {_DISTRIBUTED_TRAINING_DOC}: {exc}"]
1007 shipped = parse_shipped_torch_runtime(manifest_text)
1008 if shipped is None:
1009 return [
1010 "trainer runtime lockstep: post-helm-kubeflow-trainer-runtimes.yaml no "
1011 f"longer contains exactly one ClusterTrainingRuntime named "
1012 f"{_TRAINER_RUNTIME_NAME!r}; update this check alongside any restructure"
1013 ]
1014 shipped_image = trainer_node_image(shipped)
1015 if not shipped_image:
1016 return [
1017 "trainer runtime lockstep: the shipped torch-distributed runtime has no "
1018 "containers[name=node] image; update this check alongside any restructure"
1019 ]
1021 example_image = example_trainer_image(example_text)
1022 if example_image != shipped_image:
1023 errors.append(
1024 f"trainer runtime lockstep: examples/kubeflow-trainjob.yaml pins "
1025 f"spec.trainer.image {example_image!r} but the shipped torch-distributed "
1026 f"runtime pins {shipped_image!r} — the example deliberately lists the "
1027 "runtime's image so the image-trust gate validates exactly what runs; "
1028 "bump both together"
1029 )
1030 for doc_image in dict.fromkeys(doc_pytorch_images(doc_text)):
1031 if doc_image != shipped_image:
1032 errors.append(
1033 f"trainer runtime lockstep: docs/DISTRIBUTED_TRAINING.md shows "
1034 f"{doc_image!r} but the shipped torch-distributed runtime pins "
1035 f"{shipped_image!r} — update the doc snippet in the same change"
1036 )
1038 if not online:
1039 return errors
1041 fetcher = runtime_fetcher or fetch_upstream_torch_runtime
1042 try:
1043 upstream = fetcher(entry, helm_binary)
1044 except RuntimeError as exc:
1045 errors.append(f"trainer runtime lockstep: {exc}")
1046 return errors
1048 upstream_image = trainer_node_image(upstream)
1049 if upstream_image != shipped_image:
1050 errors.append(
1051 f"trainer runtime lockstep: chart {entry.get('version')} ships "
1052 f"torch-distributed with image {upstream_image!r} but "
1053 f"post-helm-kubeflow-trainer-runtimes.yaml pins {shipped_image!r} — "
1054 "re-extract the runtime from the pinned chart (helm template --set "
1055 "runtimes.torchDistributed.enabled=true) and bump the example + doc "
1056 "images in the same change"
1057 )
1058 expected_spec = _apply_documented_runtime_deviations(upstream.get("spec") or {})
1059 if expected_spec != (shipped.get("spec") or {}):
1060 errors.append(
1061 f"trainer runtime lockstep: the shipped torch-distributed runtime spec "
1062 f"differs from what chart {entry.get('version')} ships (beyond the "
1063 "documented deviations in _apply_documented_runtime_deviations) — the manifest "
1064 "header's contract is 'same bytes'; re-extract it from the pinned chart "
1065 "or record a new sanctioned deviation in both the manifest header and "
1066 "_apply_documented_runtime_deviations"
1067 )
1068 shipped_labels = (shipped.get("metadata") or {}).get("labels") or {}
1069 upstream_labels = (upstream.get("metadata") or {}).get("labels") or {}
1070 for label in _TRAINER_RUNTIME_LOCKSTEP_LABELS:
1071 if shipped_labels.get(label) != upstream_labels.get(label):
1072 errors.append(
1073 f"trainer runtime lockstep: label {label!r} is "
1074 f"{shipped_labels.get(label)!r} in the shipped runtime but "
1075 f"{upstream_labels.get(label)!r} upstream — these labels carry "
1076 "upstream semantics (framework selection / webhook pre-validation) "
1077 "and must be preserved verbatim"
1078 )
1079 return errors
1082def emit_chart_ref(charts: dict[str, Any], chart_name: str) -> tuple[str, str]:
1083 """Return ``(text, error)`` for --emit-ref.
1085 The emitted line is "<helm-ref> <version> <namespace> <repo_url>",
1086 space-separated so shell callers can consume it with a plain
1087 ``read -r ref version namespace repo_url``. The reference is built by the
1088 same ``ChartRef.reference()`` the online validator uses, which mirrors
1089 ``handler.install_chart`` — the CI job installs exactly what the
1090 installer Lambda would. ``repo_url`` rides along for classic (non-OCI)
1091 charts, whose reference is ``<repo_name>/<chart>`` and only resolves
1092 after ``helm repo add <repo_name> <repo_url>`` (or via
1093 ``helm pull <chart> --repo <repo_url>``); for OCI charts it is the
1094 ``oci://`` base already embedded in the reference.
1095 """
1096 refs = {ref.name: ref for ref in build_refs(charts)}
1097 ref = refs.get(chart_name)
1098 if ref is None:
1099 known = ", ".join(sorted(refs)) or "(none)"
1100 return "", f"chart {chart_name!r} not found in charts.yaml (known: {known})"
1101 return f"{ref.reference()} {ref.version} {ref.namespace} {ref.repo_url}", ""
1104def emit_chart_values(charts: dict[str, Any], chart_name: str) -> tuple[str, str]:
1105 """Return ``(yaml_text, error)`` for --emit-values.
1107 Fails when the values still carry a ``{{TOKEN}}`` deployment placeholder:
1108 charts.yaml values are the deploy-time *fallback* and must be
1109 standalone-installable (the regional stack only ever layers additional
1110 overrides on top). A token here would mean the fallback contract broke —
1111 better to fail the emit than install a chart with a literal ``{{...}}``.
1112 """
1113 refs = {ref.name: ref for ref in build_refs(charts)}
1114 ref = refs.get(chart_name)
1115 if ref is None:
1116 known = ", ".join(sorted(refs)) or "(none)"
1117 return "", f"chart {chart_name!r} not found in charts.yaml (known: {known})"
1118 text = yaml.safe_dump(ref.values, default_flow_style=False, sort_keys=False)
1119 if "{{" in text:
1120 tokens = sorted(set(re.findall(r"\{\{[A-Z0-9_]+\}\}", text)))
1121 return "", (
1122 f"{chart_name}: values contain deployment tokens {tokens} — "
1123 "charts.yaml values must be standalone-installable fallbacks"
1124 )
1125 return text, ""
1128def _build_parser() -> argparse.ArgumentParser:
1129 parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
1130 parser.add_argument(
1131 "--charts",
1132 type=Path,
1133 default=_DEFAULT_CHARTS,
1134 help="Path to charts.yaml (defaults to lambda/helm-installer/charts.yaml).",
1135 )
1136 parser.add_argument(
1137 "--mode",
1138 choices=("auto", "offline", "online"),
1139 default="auto",
1140 help=(
1141 "auto (default): structural checks always, online checks when helm "
1142 "is on PATH. offline: structural only. online: require helm and run "
1143 "the resolve/render checks (fail if helm is missing)."
1144 ),
1145 )
1146 parser.add_argument(
1147 "--skip-template",
1148 action="store_true",
1149 help="Online mode only: resolve each chart but skip the helm template render.",
1150 )
1151 parser.add_argument(
1152 "--enabled-only",
1153 action="store_true",
1154 help="Validate only charts with enabled: true (default: validate every entry).",
1155 )
1156 parser.add_argument(
1157 "--helm-binary",
1158 default="helm",
1159 help="Helm executable to use (default: 'helm' on PATH).",
1160 )
1161 parser.add_argument(
1162 "-v",
1163 "--verbose",
1164 action="store_true",
1165 help="Print a per-chart resolve/render line during the online pass.",
1166 )
1167 emit = parser.add_mutually_exclusive_group()
1168 emit.add_argument(
1169 "--emit-ref",
1170 metavar="CHART",
1171 help=(
1172 "Print '<helm-ref> <version> <namespace>' for one charts.yaml entry "
1173 "and exit (query mode for CI jobs that helm-install the pinned chart)."
1174 ),
1175 )
1176 emit.add_argument(
1177 "--emit-values",
1178 metavar="CHART",
1179 help=(
1180 "Print the shipped values block for one charts.yaml entry as YAML "
1181 "and exit; fails if the values carry {{TOKEN}} placeholders."
1182 ),
1183 )
1184 return parser
1187def main(argv: list[str] | None = None) -> int:
1188 args = _build_parser().parse_args(argv)
1190 try:
1191 charts = load_charts(args.charts)
1192 except FileNotFoundError:
1193 print(f"ERROR: charts file not found: {args.charts}", file=sys.stderr)
1194 return 2
1195 except (yaml.YAMLError, ValueError) as exc:
1196 print(f"ERROR: could not parse {args.charts}: {exc}", file=sys.stderr)
1197 return 2
1199 if args.emit_ref or args.emit_values:
1200 emitter = emit_chart_ref if args.emit_ref else emit_chart_values
1201 text, error = emitter(charts, args.emit_ref or args.emit_values)
1202 if error:
1203 print(f"ERROR: {error}", file=sys.stderr)
1204 return 2
1205 print(text, end="" if text.endswith("\n") else "\n")
1206 return 0
1208 errors = validate_structure(charts, enabled_only=args.enabled_only)
1210 helm_available = shutil.which(args.helm_binary) is not None
1211 run_online = False
1212 if args.mode == "online":
1213 if not helm_available:
1214 print(
1215 f"ERROR: --mode online requires the '{args.helm_binary}' binary on PATH.",
1216 file=sys.stderr,
1217 )
1218 return 2
1219 run_online = True
1220 elif args.mode == "auto":
1221 run_online = helm_available
1222 if not helm_available:
1223 print(
1224 "note: 'helm' not found on PATH; running structural checks only. "
1225 "Pass --mode online in CI to require the resolve/render checks."
1226 )
1228 # Fixture chart files passed via --charts may legitimately omit the
1229 # controller / trainer entries; the repository's real charts.yaml may not.
1230 is_default_charts = args.charts.resolve() == _DEFAULT_CHARTS.resolve()
1231 errors.extend(
1232 validate_gateway_lockstep(
1233 charts,
1234 online=run_online,
1235 require_entry=is_default_charts,
1236 )
1237 )
1238 errors.extend(
1239 validate_trainer_runtime_lockstep(
1240 charts,
1241 online=run_online,
1242 helm_binary=args.helm_binary,
1243 require_entry=is_default_charts,
1244 )
1245 )
1247 refs = build_refs(charts, enabled_only=args.enabled_only)
1248 if run_online:
1249 errors.extend(
1250 validate_online(
1251 refs,
1252 helm_binary=args.helm_binary,
1253 skip_template=args.skip_template,
1254 verbose=args.verbose,
1255 )
1256 )
1258 if errors:
1259 print()
1260 print(f"ERROR: {len(errors)} Helm chart validation problem(s) found:")
1261 for err in errors:
1262 print(f" - {err}")
1263 print()
1264 print(
1265 "Fix the pinned chart name/version in lambda/helm-installer/charts.yaml "
1266 "so every entry is a real, installable Helm chart."
1267 )
1268 return 1
1270 scope = "enabled" if args.enabled_only else "all"
1271 if run_online:
1272 rendered = "" if args.skip_template else " + rendered"
1273 print(
1274 f"OK: {len(refs)} Helm chart(s) ({scope}) are well-formed and "
1275 f"resolvable{rendered} at their pinned versions."
1276 )
1277 else:
1278 print(f"OK: {len(refs)} Helm chart(s) ({scope}) are structurally valid.")
1279 return 0
1282if __name__ == "__main__":
1283 sys.exit(main())