Coverage for .github / scripts / validate_helm_charts.py: 100.00%
608 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-10-28 16:19 +0000
1"""Validate every Helm chart (name, version) pinned in charts.yaml.
3Confirms that each chart entry in ``lambda/helm-installer/charts.yaml`` is a
4real, installable Helm chart at the pinned version — i.e. the
5``(chart, version)`` combination is something Helm can actually pull and
6render for a cluster, not a typo or a tag that never shipped. The
7helm-installer Lambda trusts these pins blindly at deploy time, so a bad pin
8today only surfaces as a failed ``helm upgrade --install`` mid-deploy; this
9check moves that failure left into CI.
11Two layers, matching what each CI stage can afford:
13 Structural (offline, always): pure-Python checks on the parsed YAML — no
14 network, no ``helm`` binary. Every entry needs a ``chart``, a SemVer-ish
15 ``version`` and a ``repo_url``; an ``oci://`` ``repo_url`` must set
16 ``use_oci: true`` (and vice-versa); a classic HTTP(S) repo needs a
17 ``repo_name`` for ``helm repo add``. Catches the obvious mistakes before
18 spending a network round-trip.
20 Online (needs ``helm`` + network): for every chart, build the *same*
21 reference the installer Lambda builds (see ``handler.install_chart``),
22 then
24 * ``helm show chart <ref> --version <ver>`` — proves Helm can pull the
25 chart metadata at exactly the pinned version; and
26 * ``helm template <ref> --version <ver> --values <configured>`` —
27 proves the chart renders to Kubernetes manifests (installable) with
28 the values ``charts.yaml`` ships.
30 Two retry layers ride out intermittent registry failures instead of
31 forcing a manual job rerun. Inner: every network-touching helm call
32 (repo add/update, show chart, template) goes through ``_run_with_retry``
33 — a fixed number of attempts with exponential backoff, first success
34 wins. Outer: charts still failing after the first sweep get a bounded
35 number of re-passes (default one more, ~30s later) behind a fresh
36 ``helm repo add`` + index refresh, covering outages that outlast a
37 single command's retry window. A genuinely bad pin fails every attempt
38 of every pass and still surfaces.
40By default *every* chart in the file is validated, including entries with
41``enabled: false``: those are toggled on via ``cdk.json``, so their pinned
42``(name, version)`` must be valid too.
44Three lockstep contracts ride along (each with an offline half that always
45runs and an online half): the AWS Load Balancer Controller chart against the
46Gateway API CRD bundles the helm-installer pins, the Kubeflow Trainer chart
47against the shipped ``torch-distributed`` runtime, and the ``slinky-slurm``
48slurmrestd image against ``SLURM_API_VERSION`` in ``gco/slurm_config.py``
49(the online half runs ``slurmrestd -d list`` inside the pinned image, so it
50needs a container runtime: ``docker`` by default, ``--container-runtime``
51for podman/finch).
53Usage::
55 # Structural only (no helm needed):
56 python3 .github/scripts/validate_helm_charts.py --mode offline
58 # Full check (requires helm + docker on PATH, and network) — the CI gate:
59 python3 .github/scripts/validate_helm_charts.py --mode online
61 # Same, with another container runtime for the slurmrestd image (the
62 # repo-wide GCO_CONTAINER_RUNTIME variable sets the default):
63 python3 .github/scripts/validate_helm_charts.py --mode online --container-runtime finch
65 # Auto: structural always, online when helm happens to be installed:
66 python3 .github/scripts/validate_helm_charts.py
68 # Point at a non-default charts.yaml (used by the test suite):
69 python3 .github/scripts/validate_helm_charts.py --charts /path/to/charts.yaml
71 # Emit one chart's pinned reference / shipped values (consumed by the
72 # integration:kind:examples-smoke job so its `helm install` uses the
73 # exact pins and values the installer Lambda would — no copies in CI):
74 python3 .github/scripts/validate_helm_charts.py --emit-ref mlflow
75 python3 .github/scripts/validate_helm_charts.py --emit-values mlflow
77Exit codes::
79 0 all validated charts are well-formed (and, when online, resolvable
80 + renderable at their pinned versions)
81 1 one or more charts failed validation
82 2 unexpected I/O / argument error (charts.yaml missing or unparseable,
83 or --mode online requested without a helm binary or container runtime)
85The module is importable from the test suite — call ``validate_structure()``,
86``build_refs()`` or ``validate_online()`` directly to exercise the logic
87against fixtures.
88"""
90from __future__ import annotations
92import argparse
93import contextlib
94import copy
95import os
96import re
97import shutil
98import subprocess # used only to invoke the pinned `helm` binary with fixed argv
99import sys
100import tempfile
101import time
102from dataclasses import dataclass
103from pathlib import Path
104from typing import Any
106import yaml
108# charts.yaml lives next to the helm-installer Lambda handler. This script is
109# .github/scripts/validate_helm_charts.py, so the repo root is two parents up.
110_REPO_ROOT = Path(__file__).resolve().parents[2]
111_DEFAULT_CHARTS = _REPO_ROOT / "lambda" / "helm-installer" / "charts.yaml"
113# Lenient SemVer-ish matcher for the pinned chart ``version``. Helm requires
114# SemVer2 chart versions, so anything failing this is guaranteed to fail a real
115# ``helm ... --version`` resolve too — we just catch it offline first. Accepts
116# an optional leading "v" (cert-manager / aws-efa tag their charts that way)
117# and an optional pre-release / build-metadata suffix.
118_VERSION_RE = re.compile(r"^v?\d+(?:\.\d+){1,2}(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$")
121@dataclass(frozen=True)
122class ChartRef:
123 """Everything needed to resolve one ``charts.yaml`` entry with Helm."""
125 name: str
126 chart: str
127 version: str
128 repo_name: str
129 repo_url: str
130 use_oci: bool
131 namespace: str
132 enabled: bool
133 values: dict[str, Any]
135 def reference(self) -> str:
136 """Build the Helm chart reference exactly like ``handler.install_chart``.
138 OCI charts are addressed by their full ``oci://.../<chart>`` URL;
139 classic HTTP(S) repos are addressed as ``<repo_name>/<chart>`` after
140 the repo has been added with ``helm repo add``.
141 """
142 if self.use_oci:
143 return f"{self.repo_url}/{self.chart}"
144 return f"{self.repo_name}/{self.chart}"
147def load_charts(path: Path) -> dict[str, Any]:
148 """Parse ``charts.yaml`` and return the ``charts:`` mapping.
150 Raises ``FileNotFoundError`` when the file is absent and ``ValueError``
151 when the top-level ``charts:`` mapping is missing or malformed; ``main()``
152 turns both into exit code 2.
153 """
154 with open(path, encoding="utf-8") as fh:
155 data = yaml.safe_load(fh) or {}
156 charts = data.get("charts")
157 if not isinstance(charts, dict):
158 raise ValueError("missing or malformed top-level 'charts:' mapping")
159 return charts
162def validate_structure(charts: dict[str, Any], *, enabled_only: bool = False) -> list[str]:
163 """Return a list of structural problems (empty == all well-formed).
165 These are the offline, pure-Python checks: enough to guarantee we can even
166 build a Helm reference for each entry and that OCI/classic metadata is
167 self-consistent. Every returned string names the offending chart so the
168 failure is actionable from the CI log alone.
169 """
170 errors: list[str] = []
171 if not charts:
172 return ["charts.yaml contains no chart entries under 'charts:'"]
174 for name, cfg in charts.items():
175 if not isinstance(cfg, dict):
176 errors.append(f"{name}: entry is not a mapping")
177 continue
178 if enabled_only and not cfg.get("enabled", False):
179 continue
181 chart = cfg.get("chart")
182 version = cfg.get("version")
183 repo_url = cfg.get("repo_url")
184 repo_name = cfg.get("repo_name")
185 use_oci = bool(cfg.get("use_oci", False))
187 if not (isinstance(chart, str) and chart.strip()):
188 errors.append(f"{name}: missing or empty 'chart'")
190 if not (isinstance(version, str) and version.strip()):
191 errors.append(f"{name}: missing or empty 'version'")
192 elif not _VERSION_RE.match(version.strip()):
193 errors.append(f"{name}: version {version!r} is not a valid SemVer chart version")
195 if not (isinstance(repo_url, str) and repo_url.strip()):
196 errors.append(f"{name}: missing or empty 'repo_url'")
197 else:
198 is_oci_url = repo_url.startswith("oci://")
199 if is_oci_url and not use_oci:
200 errors.append(f"{name}: repo_url is oci:// but use_oci is not set to true")
201 if use_oci and not is_oci_url:
202 errors.append(
203 f"{name}: use_oci is true but repo_url {repo_url!r} is not an oci:// URL"
204 )
205 if not use_oci and not is_oci_url:
206 if not repo_url.startswith(("http://", "https://")):
207 errors.append(
208 f"{name}: classic repo_url {repo_url!r} must be http(s):// or oci://"
209 )
210 if not (isinstance(repo_name, str) and repo_name.strip()):
211 errors.append(f"{name}: non-OCI chart needs a 'repo_name' for 'helm repo add'")
213 return errors
216def build_refs(charts: dict[str, Any], *, enabled_only: bool = False) -> list[ChartRef]:
217 """Build a ``ChartRef`` for every entry well-formed enough to resolve.
219 Entries too malformed to build a reference (no chart / version / repo_url,
220 or a classic repo with no repo_name) are skipped here — ``validate_structure``
221 already reports them, so there is no point trying to hit Helm for them.
222 """
223 refs: list[ChartRef] = []
224 for name, cfg in charts.items():
225 if not isinstance(cfg, dict):
226 continue
227 if enabled_only and not cfg.get("enabled", False):
228 continue
230 chart = cfg.get("chart")
231 version = cfg.get("version")
232 repo_url = cfg.get("repo_url")
233 use_oci = bool(cfg.get("use_oci", False))
234 repo_name = cfg.get("repo_name") or ""
236 if not (isinstance(chart, str) and chart.strip()):
237 continue
238 if not (isinstance(version, str) and version.strip()):
239 continue
240 if not (isinstance(repo_url, str) and repo_url.strip()):
241 continue
242 if not use_oci and not str(repo_name).strip():
243 continue
245 values = cfg.get("values")
246 refs.append(
247 ChartRef(
248 name=str(name),
249 chart=chart.strip(),
250 version=version.strip(),
251 repo_name=str(repo_name).strip(),
252 repo_url=repo_url.strip(),
253 use_oci=use_oci,
254 namespace=str(cfg.get("namespace", "default")),
255 enabled=bool(cfg.get("enabled", False)),
256 values=values if isinstance(values, dict) else {},
257 )
258 )
259 return refs
262def _run(cmd: list[str], env: dict[str, str], *, timeout: int = 120) -> tuple[int, str, str]:
263 """Run a command with a fixed argv (never a shell), returning (rc, out, err).
265 A subprocess timeout maps to ``(-1, "", "timeout: ...")`` so callers get a
266 uniform failure contract instead of an exception — mirrors the same pattern
267 in the helm-installer Lambda's ``run_helm``.
268 """
269 try:
270 proc = (
271 subprocess.run( # nosemgrep: dangerous-subprocess-use-audit - fixed argv, no shell=True
272 cmd,
273 capture_output=True,
274 text=True,
275 env=env,
276 timeout=timeout,
277 check=False,
278 )
279 )
280 except subprocess.TimeoutExpired as exc:
281 return -1, "", f"timeout: command exceeded {exc.timeout}s"
282 return proc.returncode, proc.stdout, proc.stderr
285def _run_with_retry(
286 cmd: list[str],
287 env: dict[str, str],
288 *,
289 attempts: int = 4,
290 base_delay: float = 2.0,
291 max_delay: float = 20.0,
292 timeout: int = 120,
293 verbose: bool = False,
294 description: str = "",
295) -> tuple[int, str, str]:
296 """Run a network-touching helm command up to ``attempts`` times; first success wins.
298 The command is retried on any non-zero exit, regardless of why it failed:
299 the first attempt that succeeds (rc 0) is returned immediately, and if none
300 do, the last failure is returned. Between attempts we sleep an exponentially
301 growing delay (``base_delay`` doubling each round, capped at ``max_delay``)
302 to give a blipping registry a moment to recover.
304 This is the guard against intermittent registry failures (timeouts, resets,
305 5xx) that otherwise force a manual rerun of the ``integration:helm:charts-valid``
306 job. A genuinely bad pin fails on every attempt and still surfaces, just a
307 few seconds later.
308 """
309 result: tuple[int, str, str] = (1, "", "")
310 for attempt in range(1, attempts + 1):
311 result = _run(cmd, env, timeout=timeout)
312 if result[0] == 0:
313 return result
314 if attempt < attempts:
315 delay = min(base_delay * (2 ** (attempt - 1)), max_delay)
316 if verbose:
317 label = description or " ".join(cmd)
318 print(
319 f" attempt {attempt}/{attempts} failed, retrying in "
320 f"{delay:.1f}s — {label}: {_tail(result[2], 200)}"
321 )
322 time.sleep(delay)
323 return result
326def _helm_env(helm_home: Path) -> dict[str, str]:
327 """Return an environment that isolates Helm's cache/config/data in a temp dir.
329 Keeps the check hermetic: it never reads or clobbers a developer's real
330 ``helm repo`` list, and CI starts from a clean slate every run.
331 """
332 env = os.environ.copy()
333 env["HELM_CACHE_HOME"] = str(helm_home / "cache")
334 env["HELM_CONFIG_HOME"] = str(helm_home / "config")
335 env["HELM_DATA_HOME"] = str(helm_home / "data")
336 return env
339def _tail(text: str, limit: int = 400) -> str:
340 """Trim helm stderr to the last ``limit`` chars so reports stay readable."""
341 text = (text or "").strip()
342 if len(text) <= limit:
343 return text
344 return "..." + text[-limit:]
347def _chart_version_from_show(show_output: str) -> str | None:
348 """Extract the ``version`` field from ``helm show chart`` YAML output."""
349 try:
350 meta = yaml.safe_load(show_output)
351 except yaml.YAMLError:
352 return None
353 if isinstance(meta, dict):
354 version = meta.get("version")
355 if isinstance(version, str):
356 return version
357 return None
360def _versions_match(resolved: str, requested: str) -> bool:
361 """Compare versions ignoring a leading ``v`` on either side."""
362 return resolved.lstrip("v") == requested.lstrip("v")
365def _render_chart(
366 ref: ChartRef,
367 ref_str: str,
368 helm_binary: str,
369 env: dict[str, str],
370 *,
371 verbose: bool = False,
372) -> str | None:
373 """``helm template`` the chart with its shipped values; return an error or None.
375 Rendering with the exact ``values`` block from ``charts.yaml`` proves the
376 chart is installable *as GCO configures it*, not just with upstream
377 defaults. No cluster is contacted, but templating a remote ``--version``
378 ref still pulls the chart from its registry, so it goes through
379 ``_run_with_retry`` to ride out the same transient blips as the resolve
380 step.
381 """
382 args = [
383 helm_binary,
384 "template",
385 ref.name,
386 ref_str,
387 "--version",
388 ref.version,
389 "--namespace",
390 ref.namespace,
391 ]
393 values_path: str | None = None
394 if ref.values:
395 fd, values_path = tempfile.mkstemp(suffix=".yaml", prefix=f"{ref.name}-values-")
396 try:
397 # fdopen takes ownership of fd; the with-block closes it on any exit,
398 # so a failing dump must NOT close it again (that raised EBADF and
399 # masked the real error). Remove the half-written file instead.
400 with os.fdopen(fd, "w", encoding="utf-8") as fh:
401 yaml.safe_dump(ref.values, fh)
402 except Exception:
403 with contextlib.suppress(OSError):
404 os.remove(values_path)
405 raise
406 args.extend(["--values", values_path])
408 try:
409 rc, _out, err = _run_with_retry(
410 args,
411 env,
412 timeout=180,
413 verbose=verbose,
414 description=f"helm template {ref.name}",
415 )
416 finally:
417 if values_path:
418 with contextlib.suppress(OSError):
419 os.remove(values_path)
421 if rc != 0:
422 return f"chart failed to render (helm template): {_tail(err)}"
423 return None
426def _sync_classic_repos(
427 classic: dict[str, str],
428 helm_binary: str,
429 env: dict[str, str],
430 *,
431 verbose: bool = False,
432) -> list[str]:
433 """``helm repo add`` every classic repo and refresh the index once.
435 Returns error strings for repos that could not be added. The index
436 refresh is network-bound too — it goes through the same retry so a blip
437 there doesn't cascade into spurious "cannot resolve" errors downstream.
438 """
439 errors: list[str] = []
440 for repo_name, repo_url in classic.items():
441 rc, _out, err = _run_with_retry(
442 [helm_binary, "repo", "add", repo_name, repo_url, "--force-update"],
443 env,
444 verbose=verbose,
445 description=f"helm repo add {repo_name}",
446 )
447 if rc != 0:
448 errors.append(f"helm repo add {repo_name} ({repo_url}) failed: {_tail(err)}")
449 if classic:
450 _run_with_retry(
451 [helm_binary, "repo", "update"],
452 env,
453 timeout=180,
454 verbose=verbose,
455 description="helm repo update",
456 )
457 return errors
460def _validate_refs(
461 refs: list[ChartRef],
462 helm_binary: str,
463 env: dict[str, str],
464 *,
465 skip_template: bool = False,
466 verbose: bool = False,
467) -> dict[str, list[str]]:
468 """Resolve + render each chart; return ``{chart name: [errors]}`` for failures."""
469 failures: dict[str, list[str]] = {}
470 for ref in refs:
471 ref_str = ref.reference()
472 label = f"{ref.name} {ref.version} ({ref_str})"
473 chart_errors: list[str] = []
475 rc, out, err = _run_with_retry(
476 [helm_binary, "show", "chart", ref_str, "--version", ref.version],
477 env,
478 verbose=verbose,
479 description=f"helm show chart {ref.name}",
480 )
481 if rc != 0:
482 chart_errors.append(
483 f"{ref.name}: helm cannot resolve {ref_str} at version "
484 f"{ref.version!r}: {_tail(err)}"
485 )
486 if verbose:
487 print(f"FAIL resolve {label}")
488 failures[ref.name] = chart_errors
489 continue
491 resolved = _chart_version_from_show(out)
492 if resolved and not _versions_match(resolved, ref.version):
493 chart_errors.append(
494 f"{ref.name}: requested version {ref.version!r} but helm resolved {resolved!r}"
495 )
496 if verbose:
497 print(f"ok resolve {label}")
499 if not skip_template:
500 render_error = _render_chart(ref, ref_str, helm_binary, env, verbose=verbose)
501 if render_error:
502 chart_errors.append(f"{ref.name}: {render_error}")
503 if verbose:
504 print(f"FAIL render {label}")
505 elif verbose:
506 print(f"ok render {label}")
508 if chart_errors:
509 failures[ref.name] = chart_errors
510 return failures
513def validate_online(
514 refs: list[ChartRef],
515 *,
516 helm_binary: str = "helm",
517 skip_template: bool = False,
518 verbose: bool = False,
519 passes: int = 2,
520 repass_delay: float = 30.0,
521) -> list[str]:
522 """Resolve (and, unless skipped, render) each chart at its pinned version.
524 Returns a list of human-readable error strings (empty == every chart is
525 resolvable and renderable). Runs against an isolated Helm home so it is
526 safe to invoke on a developer machine.
528 Retry model, outer layer: the per-command retry in ``_run_with_retry``
529 rides out blips that clear within one command's ~40-second attempt
530 window, but a registry outage lasting a few minutes fails several charts
531 on every inner attempt and used to fail the job (observed live: a rerun
532 of the unchanged job passed). So after the first sweep, the charts that
533 failed get up to ``passes - 1`` additional sweeps, each preceded by a
534 ``repass_delay`` pause and a fresh repo add + index refresh. Only
535 failures that survive every pass are reported; a genuinely bad pin fails
536 every pass and still surfaces.
537 """
538 errors: list[str] = []
539 if not refs:
540 return errors
542 with tempfile.TemporaryDirectory(prefix="gco-helm-validate-") as tmp:
543 env = _helm_env(Path(tmp))
544 classic = {ref.repo_name: ref.repo_url for ref in refs if not ref.use_oci}
546 repo_errors = _sync_classic_repos(classic, helm_binary, env, verbose=verbose)
547 failures = _validate_refs(
548 refs, helm_binary, env, skip_template=skip_template, verbose=verbose
549 )
551 for extra_pass in range(2, max(passes, 1) + 1):
552 if not failures and not repo_errors:
553 break
554 if verbose:
555 print(
556 f"re-pass {extra_pass}/{passes}: retrying "
557 f"{len(failures)} failed chart(s) in {repass_delay:.0f}s "
558 "(fresh repo index)"
559 )
560 time.sleep(repass_delay)
561 repo_errors = _sync_classic_repos(classic, helm_binary, env, verbose=verbose)
562 retry_refs = [ref for ref in refs if ref.name in failures]
563 failures = _validate_refs(
564 retry_refs, helm_binary, env, skip_template=skip_template, verbose=verbose
565 )
567 errors.extend(repo_errors)
568 for ref in refs:
569 errors.extend(failures.get(ref.name, []))
571 return errors
574# ---------------------------------------------------------------------------
575# Gateway API / aws-load-balancer-controller lockstep
576#
577# The controller is built against an exact ``sigs.k8s.io/gateway-api`` release
578# (declared in its go.mod) and its own gateway CRDs ship per controller tag.
579# GCO pins three coupled artifacts in two different files:
580#
581# * the aws-load-balancer-controller chart version (charts.yaml, this file's
582# usual input),
583# * the ``gateway-api-standard-vX.Y.Z`` CRD bundle, and
584# * the ``aws-lbc-gateway-vX.Y.Z`` CRD bundle
585# (both in lambda/helm-installer/handler.py PINNED_GATEWAY_CRD_BUNDLES).
586#
587# When the chart moved to 3.5.0 while the standard bundle stayed at v1.5.0,
588# the controller silently stopped reconciling gateways — nothing failed until
589# the live release validation deployed the pair. These checks encode the
590# contract so the drift fails CI instead:
591#
592# offline: the aws-lbc-gateway bundle version must equal the pinned chart
593# version (they ship from the same controller tag).
594# online: the controller tag's go.mod names its required gateway-api
595# release; the pinned standard bundle must be at least that (major.minor).
596# ---------------------------------------------------------------------------
598_HANDLER_PATH = _REPO_ROOT / "lambda" / "helm-installer" / "handler.py"
599_LBC_CHART_KEY = "aws-load-balancer-controller"
600_GATEWAY_API_BUNDLE_RE = re.compile(r'name="gateway-api-standard-v(\d+\.\d+\.\d+)"')
601_LBC_BUNDLE_RE = re.compile(r'name="aws-lbc-gateway-v(\d+\.\d+\.\d+)"')
602_GO_MOD_GATEWAY_API_RE = re.compile(r"^\s*sigs\.k8s\.io/gateway-api\s+v(\d+\.\d+\.\d+)", re.M)
603_LBC_GO_MOD_URL_TEMPLATE = (
604 "https://raw.githubusercontent.com/kubernetes-sigs/"
605 "aws-load-balancer-controller/v{version}/go.mod"
606)
607_GO_MOD_FETCH_ATTEMPTS = 3
608_GO_MOD_FETCH_TIMEOUT_SECONDS = 15
611def parse_pinned_gateway_bundles(handler_source: str) -> tuple[str | None, str | None]:
612 """Return (gateway-api standard bundle version, aws-lbc bundle version)."""
613 gateway_api = _GATEWAY_API_BUNDLE_RE.search(handler_source)
614 lbc = _LBC_BUNDLE_RE.search(handler_source)
615 return (
616 gateway_api.group(1) if gateway_api else None,
617 lbc.group(1) if lbc else None,
618 )
621def gateway_api_requirement_from_go_mod(go_mod_text: str) -> str | None:
622 """Return the gateway-api release the controller's go.mod declares."""
623 match = _GO_MOD_GATEWAY_API_RE.search(go_mod_text)
624 return match.group(1) if match else None
627def _version_tuple(version: str) -> tuple[int, ...]:
628 return tuple(int(part) for part in version.split("."))
631def fetch_lbc_go_mod(controller_version: str) -> str:
632 """Fetch the controller tag's go.mod, retrying transient HTTP failures."""
633 import urllib.error
634 import urllib.request
636 if not re.fullmatch(r"\d+\.\d+\.\d+", controller_version):
637 raise RuntimeError(
638 f"refusing go.mod fetch for non-semver controller version {controller_version!r}"
639 )
640 url = _LBC_GO_MOD_URL_TEMPLATE.format(version=controller_version)
641 last_error: Exception | None = None
642 for attempt in range(1, _GO_MOD_FETCH_ATTEMPTS + 1):
643 try:
644 with urllib.request.urlopen( # nosec B310 # nosemgrep: dynamic-urllib-use-detected - fixed https://raw.githubusercontent.com template; the only variable is a strictly semver-validated version segment, so no scheme or host injection is possible
645 url, timeout=_GO_MOD_FETCH_TIMEOUT_SECONDS
646 ) as response:
647 return str(response.read().decode("utf-8"))
648 except (urllib.error.URLError, TimeoutError, OSError) as exc:
649 last_error = exc
650 if attempt < _GO_MOD_FETCH_ATTEMPTS:
651 time.sleep(2**attempt)
652 raise RuntimeError(f"could not fetch {url}: {last_error}")
655def validate_gateway_lockstep(
656 charts: dict[str, Any],
657 *,
658 handler_source: str | None = None,
659 go_mod_fetcher: Any = None,
660 online: bool = False,
661 require_entry: bool = True,
662) -> list[str]:
663 """Check the chart / CRD-bundle lockstep contract; return error strings.
665 Offline: the ``aws-lbc-gateway`` bundle version must equal the pinned
666 chart version. Online (adds one HTTPS fetch): the pinned standard
667 gateway-api bundle must satisfy the requirement in the controller tag's
668 go.mod. With ``require_entry`` (the default, used for the repository's
669 real charts.yaml) a missing chart entry or renamed handler constants are
670 reported rather than skipped — this guard exists precisely for refactors
671 that move them. ``main()`` disables ``require_entry`` for explicitly
672 supplied ``--charts`` fixture files that legitimately omit the entry.
673 """
674 errors: list[str] = []
675 entry = charts.get(_LBC_CHART_KEY)
676 if not isinstance(entry, dict) or not entry.get("version"):
677 if require_entry:
678 return [f"gateway lockstep: no {_LBC_CHART_KEY!r} entry with a version in charts.yaml"]
679 return []
680 chart_version = str(entry["version"]).lstrip("v")
682 if handler_source is None:
683 try:
684 handler_source = _HANDLER_PATH.read_text(encoding="utf-8")
685 except OSError as exc:
686 return [f"gateway lockstep: cannot read {_HANDLER_PATH}: {exc}"]
688 gateway_api_pin, lbc_pin = parse_pinned_gateway_bundles(handler_source)
689 if gateway_api_pin is None or lbc_pin is None:
690 return [
691 "gateway lockstep: PINNED_GATEWAY_CRD_BUNDLES in "
692 "lambda/helm-installer/handler.py no longer names a "
693 "'gateway-api-standard-vX.Y.Z' and an 'aws-lbc-gateway-vX.Y.Z' "
694 "bundle; update this check alongside any rename"
695 ]
697 if lbc_pin != chart_version:
698 errors.append(
699 f"gateway lockstep: aws-lbc-gateway CRD bundle v{lbc_pin} does not match "
700 f"the pinned {_LBC_CHART_KEY} chart {chart_version} — the controller's "
701 "gateway CRDs ship per controller tag; bump both together "
702 "(charts.yaml + PINNED_GATEWAY_CRD_BUNDLES in lambda/helm-installer/handler.py)"
703 )
705 if not online:
706 return errors
708 fetcher = go_mod_fetcher or fetch_lbc_go_mod
709 try:
710 go_mod_text = fetcher(chart_version)
711 except RuntimeError as exc:
712 errors.append(f"gateway lockstep: {exc}")
713 return errors
715 required = gateway_api_requirement_from_go_mod(go_mod_text)
716 if required is None:
717 errors.append(
718 f"gateway lockstep: go.mod for {_LBC_CHART_KEY} v{chart_version} does not "
719 "declare sigs.k8s.io/gateway-api — upstream layout changed; update this check"
720 )
721 return errors
723 if _version_tuple(gateway_api_pin)[:2] < _version_tuple(required)[:2]:
724 errors.append(
725 f"gateway lockstep: {_LBC_CHART_KEY} {chart_version} is built against "
726 f"gateway-api v{required} (its go.mod), but the pinned standard CRD bundle "
727 f"is v{gateway_api_pin}. Upgrading the controller without its Gateway API "
728 "CRDs silently stops gateway reconciliation (caught live, 2026-08). Bump "
729 "'gateway-api-standard' in lambda/helm-installer/handler.py to at least "
730 f"v{required} in the same change"
731 )
732 return errors
735# ---------------------------------------------------------------------------
736# Kubeflow Trainer runtime / example / docs lockstep
737#
738# The kubeflow-trainer chart delivers its built-in ClusterTrainingRuntime
739# blueprints through a post-install hook Job (an unpinned run-time network
740# fetch), so GCO disables that hook and ships the ``torch-distributed``
741# runtime itself — extracted verbatim from the pinned chart into
742# ``post-helm-kubeflow-trainer-runtimes.yaml`` with two documented
743# deviations (``automountServiceAccountToken: false`` and NoNewPrivs on the
744# ``node`` container). Three more surfaces
745# repeat the runtime's pinned trainer image so users see exactly what runs:
746#
747# * the shipped runtime manifest (the source of truth in-repo),
748# * ``examples/kubeflow-trainjob.yaml`` ``spec.trainer.image`` (listed
749# explicitly so the image-trust gate validates it), and
750# * the ``docs/DISTRIBUTED_TRAINING.md`` TrainJob snippet.
751#
752# A chart version bump that skips re-extraction would silently run a stale
753# runtime (or torchrun wiring) against a newer controller. These checks
754# encode the contract so the drift fails CI instead:
755#
756# offline: the shipped runtime manifest, the example and every
757# ``pytorch/pytorch`` image mentioned in the distributed-training doc
758# agree on one image.
759# online: rendering the *pinned* chart with its runtime delivery enabled
760# must reproduce the shipped runtime — image called out explicitly,
761# full spec compared after applying the documented deviations, and the
762# upstream ``trainer.kubeflow.org/*`` labels preserved (the
763# ``webhook-validation: disabled`` label is what keeps webhook warm-up
764# from flaking the apply).
765# ---------------------------------------------------------------------------
767_TRAINER_CHART_KEY = "kubeflow-trainer"
768_TRAINER_RUNTIME_NAME = "torch-distributed"
769_TRAINER_RUNTIME_MANIFEST = (
770 _REPO_ROOT
771 / "lambda"
772 / "kubectl-applier-simple"
773 / "manifests"
774 / "post-helm-kubeflow-trainer-runtimes.yaml"
775)
776_TRAINJOB_EXAMPLE = _REPO_ROOT / "examples" / "kubeflow-trainjob.yaml"
777_DISTRIBUTED_TRAINING_DOC = _REPO_ROOT / "docs" / "DISTRIBUTED_TRAINING.md"
778# Only pytorch/pytorch mentions are lockstep-bound: the doc may legitimately
779# show other registries' images, but a pytorch/pytorch tag that differs from
780# the shipped runtime is exactly the stale-doc drift this check exists for.
781_DOC_PYTORCH_IMAGE_RE = re.compile(r"image:\s*(pytorch/pytorch:\S+)")
782# The runtime labels GCO must carry verbatim: framework selection and the
783# upstream pre-validation marker that exempts the built-in runtime from
784# webhook admission (losing it reintroduces webhook warm-up flakes).
785_TRAINER_RUNTIME_LOCKSTEP_LABELS = (
786 "trainer.kubeflow.org/framework",
787 "trainer.kubeflow.org/webhook-validation",
788)
791def parse_shipped_torch_runtime(manifest_text: str) -> dict[str, Any] | None:
792 """Return the one ``torch-distributed`` runtime in the shipped manifest.
794 ``None`` when the manifest does not contain exactly one
795 ``ClusterTrainingRuntime`` named ``torch-distributed`` — the caller turns
796 that into an "update this check" error rather than guessing.
797 """
798 try:
799 docs = [doc for doc in yaml.safe_load_all(manifest_text) if isinstance(doc, dict)]
800 except yaml.YAMLError:
801 return None
802 runtimes = [
803 doc
804 for doc in docs
805 if doc.get("kind") == "ClusterTrainingRuntime"
806 and (doc.get("metadata") or {}).get("name") == _TRAINER_RUNTIME_NAME
807 ]
808 return runtimes[0] if len(runtimes) == 1 else None
811def upstream_torch_runtime_from_render(render_text: str) -> dict[str, Any] | None:
812 """Extract the ``torch-distributed`` runtime from a rendered chart.
814 The chart ships its runtimes as multi-doc YAML inside the
815 ``*runtimes-installer`` ConfigMap's ``runtimes.yaml`` key (the payload its
816 hook Job would kubectl-apply); this digs the runtime out of that payload.
817 """
818 try:
819 docs = [doc for doc in yaml.safe_load_all(render_text) if isinstance(doc, dict)]
820 except yaml.YAMLError:
821 return None
822 for doc in docs:
823 if doc.get("kind") != "ConfigMap":
824 continue
825 name = str((doc.get("metadata") or {}).get("name") or "")
826 if not name.endswith("runtimes-installer"):
827 continue
828 payload = (doc.get("data") or {}).get("runtimes.yaml")
829 if not isinstance(payload, str):
830 continue
831 try:
832 runtimes = [item for item in yaml.safe_load_all(payload) if isinstance(item, dict)]
833 except yaml.YAMLError:
834 return None
835 for runtime in runtimes:
836 if (
837 runtime.get("kind") == "ClusterTrainingRuntime"
838 and (runtime.get("metadata") or {}).get("name") == _TRAINER_RUNTIME_NAME
839 ):
840 return runtime
841 return None
844def trainer_node_image(runtime: dict[str, Any]) -> str | None:
845 """Return the trainer image of a runtime's ``node`` replicated Job."""
846 template_spec = ((runtime.get("spec") or {}).get("template") or {}).get("spec") or {}
847 jobs = template_spec.get("replicatedJobs")
848 if not isinstance(jobs, list):
849 return None
850 for job in jobs:
851 if not isinstance(job, dict) or job.get("name") != "node":
852 continue
853 pod_spec = (((job.get("template") or {}).get("spec") or {}).get("template") or {}).get(
854 "spec"
855 ) or {}
856 containers = pod_spec.get("containers")
857 if not isinstance(containers, list):
858 return None
859 for container in containers:
860 if isinstance(container, dict) and container.get("name") == "node":
861 image = container.get("image")
862 return image if isinstance(image, str) else None
863 return None
866def example_trainer_image(example_text: str) -> str | None:
867 """Return ``spec.trainer.image`` from the TrainJob example manifest."""
868 try:
869 docs = [doc for doc in yaml.safe_load_all(example_text) if isinstance(doc, dict)]
870 except yaml.YAMLError:
871 return None
872 for doc in docs:
873 if doc.get("kind") == "TrainJob":
874 image = ((doc.get("spec") or {}).get("trainer") or {}).get("image")
875 return image if isinstance(image, str) else None
876 return None
879def doc_pytorch_images(doc_text: str) -> list[str]:
880 """Return every ``pytorch/pytorch`` image the doc's snippets mention."""
881 return _DOC_PYTORCH_IMAGE_RE.findall(doc_text)
884def _apply_documented_runtime_deviations(upstream_spec: dict[str, Any]) -> dict[str, Any]:
885 """Return the upstream runtime spec with GCO's sanctioned deviations applied.
887 Exactly two deviations from verbatim extraction are documented in the
888 manifest header, both on the ``node`` pod template:
890 - ``automountServiceAccountToken: false`` (the same security default
891 both submission paths inject into user Jobs), and
892 - ``securityContext.allowPrivilegeEscalation: false`` on the ``node``
893 container (NoNewPrivs; the platform's manifest policy already rejects
894 an explicit ``true``).
896 Anything else that differs from upstream is drift and must fail — a new
897 deliberate deviation belongs here *and* in the manifest header, in the
898 same change.
899 """
900 adjusted = copy.deepcopy(upstream_spec)
901 jobs = ((adjusted.get("template") or {}).get("spec") or {}).get("replicatedJobs")
902 for job in jobs if isinstance(jobs, list) else []:
903 if not isinstance(job, dict) or job.get("name") != "node":
904 continue
905 pod_spec = (((job.get("template") or {}).get("spec") or {}).get("template") or {}).get(
906 "spec"
907 )
908 if not isinstance(pod_spec, dict):
909 continue
910 pod_spec["automountServiceAccountToken"] = False
911 for container in pod_spec.get("containers") or []:
912 if isinstance(container, dict) and container.get("name") == "node":
913 security = container.setdefault("securityContext", {})
914 if isinstance(security, dict):
915 security["allowPrivilegeEscalation"] = False
916 return adjusted
919def fetch_upstream_torch_runtime(
920 entry: dict[str, Any], helm_binary: str = "helm"
921) -> dict[str, Any]:
922 """Render the pinned chart with runtime delivery enabled; return the runtime.
924 Runs against an isolated Helm home (same hermetic setup as the resolve /
925 render pass) and rides the shared retry ladder, so registry blips do not
926 fail the job. Raises ``RuntimeError`` with an actionable message when the
927 chart cannot be rendered or no longer ships the runtime where this check
928 expects it.
929 """
930 refs = build_refs({_TRAINER_CHART_KEY: entry})
931 if not refs:
932 raise RuntimeError(
933 f"cannot build a Helm reference from the {_TRAINER_CHART_KEY!r} charts.yaml entry"
934 )
935 ref = refs[0]
936 with tempfile.TemporaryDirectory(prefix="gco-trainer-lockstep-") as tmp:
937 env = _helm_env(Path(tmp))
938 if not ref.use_oci:
939 _sync_classic_repos({ref.repo_name: ref.repo_url}, helm_binary, env)
940 rc, out, err = _run_with_retry(
941 [
942 helm_binary,
943 "template",
944 ref.name,
945 ref.reference(),
946 "--version",
947 ref.version,
948 "--namespace",
949 ref.namespace,
950 "--set",
951 "runtimes.torchDistributed.enabled=true",
952 ],
953 env,
954 timeout=180,
955 description="helm template kubeflow-trainer (runtime delivery enabled)",
956 )
957 if rc != 0:
958 raise RuntimeError(
959 f"helm template {ref.reference()} at {ref.version!r} (with "
960 f"runtimes.torchDistributed.enabled=true) failed: {_tail(err)}"
961 )
962 runtime = upstream_torch_runtime_from_render(out)
963 if runtime is None:
964 raise RuntimeError(
965 f"chart {ref.version} no longer ships a 'runtimes-installer' ConfigMap "
966 f"containing ClusterTrainingRuntime {_TRAINER_RUNTIME_NAME!r} — upstream "
967 "runtime delivery changed; update this check alongside the re-extraction"
968 )
969 return runtime
972def validate_trainer_runtime_lockstep(
973 charts: dict[str, Any],
974 *,
975 manifest_text: str | None = None,
976 example_text: str | None = None,
977 doc_text: str | None = None,
978 online: bool = False,
979 runtime_fetcher: Any = None,
980 helm_binary: str = "helm",
981 require_entry: bool = True,
982) -> list[str]:
983 """Check the trainer runtime / example / docs lockstep; return error strings.
985 Offline: the shipped runtime manifest, the TrainJob example and the
986 distributed-training doc must agree on the trainer image. Online (adds
987 one chart render): the shipped runtime must reproduce what the pinned
988 chart ships — byte-identical spec after the documented deviations, with
989 the trainer image compared explicitly for an actionable message. With
990 ``require_entry`` (the default, used for the repository's real
991 charts.yaml) a missing chart entry is reported rather than skipped;
992 ``main()`` disables it for ``--charts`` fixture files.
993 """
994 errors: list[str] = []
995 entry = charts.get(_TRAINER_CHART_KEY)
996 if not isinstance(entry, dict) or not entry.get("version"):
997 if require_entry:
998 return [
999 f"trainer runtime lockstep: no {_TRAINER_CHART_KEY!r} entry with a "
1000 "version in charts.yaml"
1001 ]
1002 return []
1004 if manifest_text is None:
1005 try:
1006 manifest_text = _TRAINER_RUNTIME_MANIFEST.read_text(encoding="utf-8")
1007 except OSError as exc:
1008 return [f"trainer runtime lockstep: cannot read {_TRAINER_RUNTIME_MANIFEST}: {exc}"]
1009 if example_text is None:
1010 try:
1011 example_text = _TRAINJOB_EXAMPLE.read_text(encoding="utf-8")
1012 except OSError as exc:
1013 return [f"trainer runtime lockstep: cannot read {_TRAINJOB_EXAMPLE}: {exc}"]
1014 if doc_text is None:
1015 try:
1016 doc_text = _DISTRIBUTED_TRAINING_DOC.read_text(encoding="utf-8")
1017 except OSError as exc:
1018 return [f"trainer runtime lockstep: cannot read {_DISTRIBUTED_TRAINING_DOC}: {exc}"]
1020 shipped = parse_shipped_torch_runtime(manifest_text)
1021 if shipped is None:
1022 return [
1023 "trainer runtime lockstep: post-helm-kubeflow-trainer-runtimes.yaml no "
1024 f"longer contains exactly one ClusterTrainingRuntime named "
1025 f"{_TRAINER_RUNTIME_NAME!r}; update this check alongside any restructure"
1026 ]
1027 shipped_image = trainer_node_image(shipped)
1028 if not shipped_image:
1029 return [
1030 "trainer runtime lockstep: the shipped torch-distributed runtime has no "
1031 "containers[name=node] image; update this check alongside any restructure"
1032 ]
1034 example_image = example_trainer_image(example_text)
1035 if example_image != shipped_image:
1036 errors.append(
1037 f"trainer runtime lockstep: examples/kubeflow-trainjob.yaml pins "
1038 f"spec.trainer.image {example_image!r} but the shipped torch-distributed "
1039 f"runtime pins {shipped_image!r} — the example deliberately lists the "
1040 "runtime's image so the image-trust gate validates exactly what runs; "
1041 "bump both together"
1042 )
1043 for doc_image in dict.fromkeys(doc_pytorch_images(doc_text)):
1044 if doc_image != shipped_image:
1045 errors.append(
1046 f"trainer runtime lockstep: docs/DISTRIBUTED_TRAINING.md shows "
1047 f"{doc_image!r} but the shipped torch-distributed runtime pins "
1048 f"{shipped_image!r} — update the doc snippet in the same change"
1049 )
1051 if not online:
1052 return errors
1054 fetcher = runtime_fetcher or fetch_upstream_torch_runtime
1055 try:
1056 upstream = fetcher(entry, helm_binary)
1057 except RuntimeError as exc:
1058 errors.append(f"trainer runtime lockstep: {exc}")
1059 return errors
1061 upstream_image = trainer_node_image(upstream)
1062 if upstream_image != shipped_image:
1063 errors.append(
1064 f"trainer runtime lockstep: chart {entry.get('version')} ships "
1065 f"torch-distributed with image {upstream_image!r} but "
1066 f"post-helm-kubeflow-trainer-runtimes.yaml pins {shipped_image!r} — "
1067 "re-extract the runtime from the pinned chart (helm template --set "
1068 "runtimes.torchDistributed.enabled=true) and bump the example + doc "
1069 "images in the same change"
1070 )
1071 expected_spec = _apply_documented_runtime_deviations(upstream.get("spec") or {})
1072 if expected_spec != (shipped.get("spec") or {}):
1073 errors.append(
1074 f"trainer runtime lockstep: the shipped torch-distributed runtime spec "
1075 f"differs from what chart {entry.get('version')} ships (beyond the "
1076 "documented deviations in _apply_documented_runtime_deviations) — the manifest "
1077 "header's contract is 'same bytes'; re-extract it from the pinned chart "
1078 "or record a new sanctioned deviation in both the manifest header and "
1079 "_apply_documented_runtime_deviations"
1080 )
1081 shipped_labels = (shipped.get("metadata") or {}).get("labels") or {}
1082 upstream_labels = (upstream.get("metadata") or {}).get("labels") or {}
1083 for label in _TRAINER_RUNTIME_LOCKSTEP_LABELS:
1084 if shipped_labels.get(label) != upstream_labels.get(label):
1085 errors.append(
1086 f"trainer runtime lockstep: label {label!r} is "
1087 f"{shipped_labels.get(label)!r} in the shipped runtime but "
1088 f"{upstream_labels.get(label)!r} upstream — these labels carry "
1089 "upstream semantics (framework selection / webhook pre-validation) "
1090 "and must be preserved verbatim"
1091 )
1092 return errors
1095# ---------------------------------------------------------------------------
1096# Slurm REST API version lockstep
1097#
1098# Every GCO surface that talks to slurmrestd addresses one OpenAPI plugin
1099# version (``/slurm/<version>/...``): the API client behind ``/api/v1/slurm``,
1100# the ``gco slurm`` CLI through it, ``examples/slurm-cluster-job.yaml`` and the
1101# live-validation probe (both call slurmrestd directly from inside the
1102# cluster) and the API doc's sample payload. The pin is
1103# ``SLURM_API_VERSION`` in ``gco/slurm_config.py``, beside the image tag it
1104# moves with.
1105#
1106# The version is a property of the Slurm release the ``slinky-slurm`` images
1107# are built from: each release ships its newest ``data_parser`` plugin plus
1108# the three before it and drops the oldest, so bumping the image tag in
1109# charts.yaml without re-checking the pin eventually leaves the API addressing
1110# a version the image no longer serves — every request 404s. Negotiating the
1111# version at runtime is deliberately not the fix: request and response bodies
1112# differ between plugin versions, so the client is written against exactly
1113# one. These checks encode the contract so the drift fails CI instead:
1114#
1115# offline: every surface that repeats the pin names the same version as
1116# ``gco/slurm_config.py``.
1117# online: ``slurmrestd -d list`` inside the pinned slurmrestd image names
1118# the plugin versions it ships; the pin must be one of them. Needs a
1119# container runtime (``docker`` on the CI runner; ``--container-runtime``
1120# for podman/finch). A newer served version is noted, not an error:
1121# moving to it is a deliberate re-check of the body shapes.
1122# ---------------------------------------------------------------------------
1124_SLURM_CHART_KEY = "slinky-slurm"
1125_SLURM_CONFIG_MODULE = _REPO_ROOT / "gco" / "slurm_config.py"
1126# The files that repeat the pin outside gco/slurm_config.py. The route module
1127# is included for the comment that records which version's job_desc_msg shape
1128# the submission body was written against.
1129_SLURM_API_VERSION_SURFACES: tuple[Path, ...] = (
1130 _REPO_ROOT / "examples" / "slurm-cluster-job.yaml",
1131 _REPO_ROOT / "scripts" / "live_release_validation" / "manifests" / "slurm-smoke-job.yaml",
1132 _REPO_ROOT / "docs" / "API.md",
1133 _REPO_ROOT / "gco" / "services" / "api_routes" / "slurm.py",
1134)
1135_SLURM_API_VERSION_PIN_RE = re.compile(r'^SLURM_API_VERSION\s*=\s*"(v\d+\.\d+\.\d+)"', re.MULTILINE)
1136# slurmrestd plugin versions are v0.0.NN; any such token in a Slurm surface is
1137# a repeat of the pin (nothing else in those files is versioned that way).
1138_SLURM_API_VERSION_MENTION_RE = re.compile(r"\bv0\.0\.\d+\b")
1139# One line per plugin in the ``slurmrestd -d list`` output ("data_parser/v0.0.44").
1140_SLURMRESTD_PLUGIN_RE = re.compile(r"data_parser/(v\d+\.\d+\.\d+)")
1143def parse_slurm_api_version(config_source: str) -> str | None:
1144 """Return the ``SLURM_API_VERSION`` pin from ``gco/slurm_config.py`` source."""
1145 match = _SLURM_API_VERSION_PIN_RE.search(config_source)
1146 return match.group(1) if match else None
1149def slurm_api_version_mentions(text: str) -> list[tuple[int, str]]:
1150 """Return ``(line_number, version)`` for every slurmrestd version token in ``text``."""
1151 return [
1152 (number, match.group(0))
1153 for number, line in enumerate(text.splitlines(), start=1)
1154 for match in _SLURM_API_VERSION_MENTION_RE.finditer(line)
1155 ]
1158def parse_served_slurm_api_versions(listing: str) -> set[str]:
1159 """Return the data_parser plugin versions a ``slurmrestd -d list`` listing names."""
1160 return set(_SLURMRESTD_PLUGIN_RE.findall(listing))
1163def slurmrestd_image(entry: dict[str, Any]) -> str | None:
1164 """Return ``repository:tag`` of the slurmrestd image a ``slinky-slurm`` entry pins."""
1165 values = entry.get("values") or {}
1166 restapi = values.get("restapi") or {}
1167 image = (restapi.get("slurmrestd") or {}).get("image")
1168 if not isinstance(image, dict) or not image.get("repository") or not image.get("tag"):
1169 return None
1170 return f"{image['repository']}:{image['tag']}"
1173def _plugin_sort_key(version: str) -> tuple[int, ...]:
1174 return _version_tuple(version.lstrip("v"))
1177def _repo_relative(path: Path) -> str:
1178 """Display form of a surface path: repo-relative when under the repo root."""
1179 try:
1180 return str(path.relative_to(_REPO_ROOT))
1181 except ValueError:
1182 return str(path)
1185def fetch_served_slurm_api_versions(image: str, container_runtime: str = "docker") -> set[str]:
1186 """Run ``slurmrestd -d list`` inside the pinned image; return the versions it ships.
1188 The pull + run rides the shared retry ladder so a registry blip does not
1189 fail the job. The container gets no network (the listing is local to the
1190 binary). Raises ``RuntimeError`` with an actionable message when the image
1191 cannot run or no longer prints a plugin listing where this check expects
1192 it.
1193 """
1194 cmd = [
1195 container_runtime,
1196 "run",
1197 "--rm",
1198 "--network",
1199 "none",
1200 "--entrypoint",
1201 "slurmrestd",
1202 image,
1203 "-d",
1204 "list",
1205 ]
1206 rc, out, err = _run_with_retry(
1207 cmd,
1208 os.environ.copy(),
1209 timeout=600,
1210 description=f"{container_runtime} run {image} slurmrestd -d list",
1211 )
1212 if rc != 0:
1213 raise RuntimeError(
1214 f"{container_runtime} run {image} (slurmrestd -d list) failed: {_tail(err or out)}"
1215 )
1216 versions = parse_served_slurm_api_versions(out + "\n" + err)
1217 if not versions:
1218 raise RuntimeError(
1219 f"slurmrestd -d list in {image} printed no data_parser/vX.Y.Z plugin lines — "
1220 "upstream output changed; update this check alongside the new format"
1221 )
1222 return versions
1225def validate_slurm_api_version_lockstep(
1226 charts: dict[str, Any],
1227 *,
1228 config_source: str | None = None,
1229 surface_texts: dict[str, str] | None = None,
1230 online: bool = False,
1231 versions_fetcher: Any = None,
1232 container_runtime: str = "docker",
1233 require_entry: bool = True,
1234) -> list[str]:
1235 """Check the slurmrestd API version lockstep; return error strings.
1237 Offline: every surface that repeats ``SLURM_API_VERSION`` (the example
1238 job, the live-validation probe, the API doc and the route module's shape
1239 note) names the version pinned in ``gco/slurm_config.py``. Online (adds
1240 one image pull + run): the slurmrestd image pinned in charts.yaml must
1241 serve that version. With ``require_entry`` (the default, used for the
1242 repository's real charts.yaml) a missing chart entry is reported rather
1243 than skipped; ``main()`` disables it for ``--charts`` fixture files.
1244 ``surface_texts`` maps a display name to file text (tests); ``None``
1245 reads the repository files.
1246 """
1247 errors: list[str] = []
1248 entry = charts.get(_SLURM_CHART_KEY)
1249 if not isinstance(entry, dict) or not entry.get("version"):
1250 if require_entry:
1251 return [
1252 f"slurm api lockstep: no {_SLURM_CHART_KEY!r} entry with a version in charts.yaml"
1253 ]
1254 return []
1256 if config_source is None:
1257 try:
1258 config_source = _SLURM_CONFIG_MODULE.read_text(encoding="utf-8")
1259 except OSError as exc:
1260 return [f"slurm api lockstep: cannot read {_SLURM_CONFIG_MODULE}: {exc}"]
1261 pin = parse_slurm_api_version(config_source)
1262 if pin is None:
1263 return [
1264 'slurm api lockstep: gco/slurm_config.py no longer defines SLURM_API_VERSION = "vX.Y.Z"; '
1265 "update this check alongside any rename"
1266 ]
1268 if surface_texts is None:
1269 surface_texts = {}
1270 for path in _SLURM_API_VERSION_SURFACES:
1271 try:
1272 surface_texts[_repo_relative(path)] = path.read_text(encoding="utf-8")
1273 except OSError as exc:
1274 return [f"slurm api lockstep: cannot read {path}: {exc}"]
1275 for name, text in surface_texts.items():
1276 for line, version in slurm_api_version_mentions(text):
1277 if version != pin:
1278 errors.append(
1279 f"slurm api lockstep: {name}:{line} names slurmrestd {version} but "
1280 f"SLURM_API_VERSION in gco/slurm_config.py is {pin} — every surface "
1281 "addresses one plugin version; update them in the same change"
1282 )
1284 if not online:
1285 return errors
1287 image = slurmrestd_image(entry)
1288 if image is None:
1289 errors.append(
1290 f"slurm api lockstep: charts.yaml {_SLURM_CHART_KEY}.values.restapi.slurmrestd.image "
1291 "no longer pins a repository and tag; update this check alongside any restructure"
1292 )
1293 return errors
1295 fetcher = versions_fetcher or fetch_served_slurm_api_versions
1296 try:
1297 served = fetcher(image, container_runtime)
1298 except RuntimeError as exc:
1299 errors.append(f"slurm api lockstep: {exc}")
1300 return errors
1302 served_list = ", ".join(sorted(served, key=_plugin_sort_key))
1303 if pin not in served:
1304 errors.append(
1305 f"slurm api lockstep: {image} serves data_parser plugin version(s) {served_list} "
1306 f"but SLURM_API_VERSION is {pin} — the GCO API would get 404 for every Slurm "
1307 "request. Bump SLURM_API_VERSION in gco/slurm_config.py (and the surfaces that "
1308 "repeat it) to a version the image ships, re-checking the request and response "
1309 "shapes gco/services/slurm_client.py and api_routes/slurm.py rely on"
1310 )
1311 return errors
1313 newer = [
1314 v
1315 for v in sorted(served, key=_plugin_sort_key)
1316 if _plugin_sort_key(v) > _plugin_sort_key(pin)
1317 ]
1318 if newer:
1319 print(
1320 f"note: {image} serves SLURM_API_VERSION {pin} and also newer plugin version(s) "
1321 f"{', '.join(newer)}; the newest stays supported longest across Slurm releases, "
1322 "moving to it is a deliberate re-check of the body shapes (not required now)."
1323 )
1324 return errors
1327def emit_chart_ref(charts: dict[str, Any], chart_name: str) -> tuple[str, str]:
1328 """Return ``(text, error)`` for --emit-ref.
1330 The emitted line is "<helm-ref> <version> <namespace> <repo_url>",
1331 space-separated so shell callers can consume it with a plain
1332 ``read -r ref version namespace repo_url``. The reference is built by the
1333 same ``ChartRef.reference()`` the online validator uses, which mirrors
1334 ``handler.install_chart`` — the CI job installs exactly what the
1335 installer Lambda would. ``repo_url`` rides along for classic (non-OCI)
1336 charts, whose reference is ``<repo_name>/<chart>`` and only resolves
1337 after ``helm repo add <repo_name> <repo_url>`` (or via
1338 ``helm pull <chart> --repo <repo_url>``); for OCI charts it is the
1339 ``oci://`` base already embedded in the reference.
1340 """
1341 refs = {ref.name: ref for ref in build_refs(charts)}
1342 ref = refs.get(chart_name)
1343 if ref is None:
1344 known = ", ".join(sorted(refs)) or "(none)"
1345 return "", f"chart {chart_name!r} not found in charts.yaml (known: {known})"
1346 return f"{ref.reference()} {ref.version} {ref.namespace} {ref.repo_url}", ""
1349def emit_chart_values(charts: dict[str, Any], chart_name: str) -> tuple[str, str]:
1350 """Return ``(yaml_text, error)`` for --emit-values.
1352 Fails when the values still carry a ``{{TOKEN}}`` deployment placeholder:
1353 charts.yaml values are the deploy-time *fallback* and must be
1354 standalone-installable (the regional stack only ever layers additional
1355 overrides on top). A token here would mean the fallback contract broke —
1356 better to fail the emit than install a chart with a literal ``{{...}}``.
1357 """
1358 refs = {ref.name: ref for ref in build_refs(charts)}
1359 ref = refs.get(chart_name)
1360 if ref is None:
1361 known = ", ".join(sorted(refs)) or "(none)"
1362 return "", f"chart {chart_name!r} not found in charts.yaml (known: {known})"
1363 text = yaml.safe_dump(ref.values, default_flow_style=False, sort_keys=False)
1364 if "{{" in text:
1365 tokens = sorted(set(re.findall(r"\{\{[A-Z0-9_]+\}\}", text)))
1366 return "", (
1367 f"{chart_name}: values contain deployment tokens {tokens} — "
1368 "charts.yaml values must be standalone-installable fallbacks"
1369 )
1370 return text, ""
1373def _build_parser() -> argparse.ArgumentParser:
1374 parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
1375 parser.add_argument(
1376 "--charts",
1377 type=Path,
1378 default=_DEFAULT_CHARTS,
1379 help="Path to charts.yaml (defaults to lambda/helm-installer/charts.yaml).",
1380 )
1381 parser.add_argument(
1382 "--mode",
1383 choices=("auto", "offline", "online"),
1384 default="auto",
1385 help=(
1386 "auto (default): structural checks always, online checks when helm "
1387 "is on PATH. offline: structural only. online: require helm and run "
1388 "the resolve/render checks (fail if helm is missing)."
1389 ),
1390 )
1391 parser.add_argument(
1392 "--skip-template",
1393 action="store_true",
1394 help="Online mode only: resolve each chart but skip the helm template render.",
1395 )
1396 parser.add_argument(
1397 "--enabled-only",
1398 action="store_true",
1399 help="Validate only charts with enabled: true (default: validate every entry).",
1400 )
1401 parser.add_argument(
1402 "--helm-binary",
1403 default="helm",
1404 help="Helm executable to use (default: 'helm' on PATH).",
1405 )
1406 parser.add_argument(
1407 "--container-runtime",
1408 default=os.environ.get("GCO_CONTAINER_RUNTIME", "docker"),
1409 help=(
1410 "Container CLI used by the online slurmrestd API-version check to run the "
1411 "pinned slurmrestd image (default: $GCO_CONTAINER_RUNTIME or 'docker'; podman "
1412 "and finch work too). Required by --mode online; auto mode skips that one check "
1413 "when it is missing."
1414 ),
1415 )
1416 parser.add_argument(
1417 "-v",
1418 "--verbose",
1419 action="store_true",
1420 help="Print a per-chart resolve/render line during the online pass.",
1421 )
1422 emit = parser.add_mutually_exclusive_group()
1423 emit.add_argument(
1424 "--emit-ref",
1425 metavar="CHART",
1426 help=(
1427 "Print '<helm-ref> <version> <namespace>' for one charts.yaml entry "
1428 "and exit (query mode for CI jobs that helm-install the pinned chart)."
1429 ),
1430 )
1431 emit.add_argument(
1432 "--emit-values",
1433 metavar="CHART",
1434 help=(
1435 "Print the shipped values block for one charts.yaml entry as YAML "
1436 "and exit; fails if the values carry {{TOKEN}} placeholders."
1437 ),
1438 )
1439 return parser
1442def main(argv: list[str] | None = None) -> int:
1443 args = _build_parser().parse_args(argv)
1445 try:
1446 charts = load_charts(args.charts)
1447 except FileNotFoundError:
1448 print(f"ERROR: charts file not found: {args.charts}", file=sys.stderr)
1449 return 2
1450 except (yaml.YAMLError, ValueError) as exc:
1451 print(f"ERROR: could not parse {args.charts}: {exc}", file=sys.stderr)
1452 return 2
1454 if args.emit_ref or args.emit_values:
1455 emitter = emit_chart_ref if args.emit_ref else emit_chart_values
1456 text, error = emitter(charts, args.emit_ref or args.emit_values)
1457 if error:
1458 print(f"ERROR: {error}", file=sys.stderr)
1459 return 2
1460 print(text, end="" if text.endswith("\n") else "\n")
1461 return 0
1463 errors = validate_structure(charts, enabled_only=args.enabled_only)
1465 helm_available = shutil.which(args.helm_binary) is not None
1466 run_online = False
1467 if args.mode == "online":
1468 if not helm_available:
1469 print(
1470 f"ERROR: --mode online requires the '{args.helm_binary}' binary on PATH.",
1471 file=sys.stderr,
1472 )
1473 return 2
1474 run_online = True
1475 elif args.mode == "auto":
1476 run_online = helm_available
1477 if not helm_available:
1478 print(
1479 "note: 'helm' not found on PATH; running structural checks only. "
1480 "Pass --mode online in CI to require the resolve/render checks."
1481 )
1483 # The slurmrestd API-version check runs the pinned image, so its online
1484 # half additionally needs a container runtime: required alongside helm in
1485 # --mode online, skipped with a note in auto mode.
1486 run_slurm_online = run_online
1487 if run_online and shutil.which(args.container_runtime) is None:
1488 if args.mode == "online":
1489 print(
1490 f"ERROR: --mode online requires the '{args.container_runtime}' binary on PATH "
1491 "(the slurmrestd API-version check runs the pinned slurmrestd image; pass "
1492 "--container-runtime podman|finch for another runtime).",
1493 file=sys.stderr,
1494 )
1495 return 2
1496 run_slurm_online = False
1497 print(
1498 f"note: '{args.container_runtime}' not found on PATH; skipping the online "
1499 "slurmrestd API-version check. Pass --mode online in CI to require it "
1500 "(--container-runtime selects podman/finch)."
1501 )
1503 # Fixture chart files passed via --charts may legitimately omit the
1504 # controller / trainer / Slurm entries; the repository's real charts.yaml
1505 # may not.
1506 is_default_charts = args.charts.resolve() == _DEFAULT_CHARTS.resolve()
1507 errors.extend(
1508 validate_gateway_lockstep(
1509 charts,
1510 online=run_online,
1511 require_entry=is_default_charts,
1512 )
1513 )
1514 errors.extend(
1515 validate_trainer_runtime_lockstep(
1516 charts,
1517 online=run_online,
1518 helm_binary=args.helm_binary,
1519 require_entry=is_default_charts,
1520 )
1521 )
1522 errors.extend(
1523 validate_slurm_api_version_lockstep(
1524 charts,
1525 online=run_slurm_online,
1526 container_runtime=args.container_runtime,
1527 require_entry=is_default_charts,
1528 )
1529 )
1531 refs = build_refs(charts, enabled_only=args.enabled_only)
1532 if run_online:
1533 errors.extend(
1534 validate_online(
1535 refs,
1536 helm_binary=args.helm_binary,
1537 skip_template=args.skip_template,
1538 verbose=args.verbose,
1539 )
1540 )
1542 if errors:
1543 print()
1544 print(f"ERROR: {len(errors)} Helm chart validation problem(s) found:")
1545 for err in errors:
1546 print(f" - {err}")
1547 print()
1548 print(
1549 "Fix the pinned chart name/version in lambda/helm-installer/charts.yaml "
1550 "so every entry is a real, installable Helm chart."
1551 )
1552 return 1
1554 scope = "enabled" if args.enabled_only else "all"
1555 if run_online:
1556 rendered = "" if args.skip_template else " + rendered"
1557 print(
1558 f"OK: {len(refs)} Helm chart(s) ({scope}) are well-formed and "
1559 f"resolvable{rendered} at their pinned versions."
1560 )
1561 else:
1562 print(f"OK: {len(refs)} Helm chart(s) ({scope}) are structurally valid.")
1563 return 0
1566if __name__ == "__main__":
1567 sys.exit(main())