← all scripts

.github/scripts/dependency-scan.sh

1404 of 1404 statements covered (100.00%).

coveredmissednever traced by Bash (not counted)A line ending in … continues the statement above it and shares its fate.

1#!/usr/bin/env bash
2# =============================================================================
3# dependency-scan.sh — check Python, Node.js, Docker, Helm, and EKS versions
4# =============================================================================
5#
6# Invoked by .github/workflows/deps-scan.yml (monthly schedule).
7#
8# Checks for drift across:
9#
10# - Python packages pinned in pyproject.toml
11# - Docker images referenced from workflows, K8s manifests, examples,
12# local live-validation manifests, and Helm chart values
13# - Helm chart versions from charts.yaml
14# - EKS add-on versions from gco/stacks/constants.py (AWS creds)
15# - EKS Kubernetes minor from cdk.json (AWS creds)
16# - Aurora PostgreSQL engine versions (AWS creds)
17# - EMR Serverless release labels (AWS creds)
18# - Bedrock default model ids from cdk.json
19# context.bedrock.mission_default_model_id (Mission sampling),
20# context.bedrock.capacity_advisor_default_model_id (capacity advisor),
21# context.bedrock.claude_code_default_model_id (Claude Autopilot),
22# context.bedrock.codex_default_model_id (Codex Autopilot),
23# context.bedrock.embedding_model_id (Mission memory), and
24# context.vector_store.embedding_model_id (workload RAG corpus), each
25# compared against the newest same-family release — inference profiles
26# for the generation keys, EMBEDDING foundation models for the embedding
27# keys (AWS creds)
28# - Accelerator catalog and Karpenter NodePool policy (offline), plus live
29# NVIDIA GPU / AWS Neuron EC2 catalog drift across enabled Regions (AWS creds)
30# - Dockerfile.dev ARG pins (Node LTS major, npm, CDK CLI, kubectl,
31# AWS CLI v2, Docker CLI, Docker Buildx, uv) and the immutable AWS CLI
32# runtime image in gco/services/inference_monitor.py — public registries,
33# no AWS creds needed
34# - GCO Autopilot pins from cli/autopilot.py: CLAUDE_CODE_VERSION and
35# CODEX_VERSION install pins vs their npm latest dist-tags, plus companion
36# MCP server liveness (missing/deprecated/yanked on npm or PyPI) — public
37# endpoints, no AWS creds needed
38# - Pre-commit hook revisions in .pre-commit-config.yaml compared
39# against the latest tag published upstream (GitHub API)
40# - CDK enum constants from gco/stacks/constants.py compared against the
41# installed aws-cdk-lib (LAMBDA_PYTHON_RUNTIME, LAMBDA_NODEJS_RUNTIME;
42# the Aurora engine is a plain version string checked against live RDS)
43# - Latest stable Python release from endoflife.date — public endpoint
44# - CI tooling the workflows install by hand: Trivy (the install-trivy
45# composite action's version default),
46# actionlint (ACTIONLINT_VERSION), Helm and kubectl (HELM_VERSION /
47# KUBECTL_VERSION), kubeconform (KUBECONFORM_VERSION), Calico
48# (CALICO_VERSION), Metrics Server (METRICS_SERVER_VERSION), and kind + its
49# node image — public endpoints, no AWS creds
50# - Version consistency: ruff (pyproject / pre-commit / lint workflow),
51# Python and Node runtime pins, npm packageManager + CDK CLI pins, every
52# owned npm graph's lockfile/Dependabot coverage, duplicated *_VERSION
53# workflow environment pins, every per-Lambda requirements.txt pin
54# against the version resolved centrally, and digest-pinned images
55# carrying two digests under one tag
56# - Base-image security epochs (APT_SECURITY_EPOCH / DNF_SECURITY_EPOCH)
57# older than SECURITY_EPOCH_STALE_DAYS
58# - Suppression expiries: .trivyignore / .pip-audit-ignore /
59# .npm-audit-ignore entries expiring within SUPPRESSION_EXPIRY_WARN_DAYS
60# (before the CI validator hard-fails)
61# - Lockfile freshness: direct deps in pyproject.toml missing from or pinned
62# differently in requirements-lock.txt
63#
64# Ports the `.dependency-scan-script` YAML anchor from the retired
65# GitLab pipeline into a standalone shell script. Two behavior changes:
66#
67# 1. Workflow file input. The GitLab version grepped `.gitlab-ci.yml` for
68# CI image tags. This version scans every file under
69# `$WORKFLOWS_DIR` (default: `.github/workflows`).
70# 2. Reporting. The GitLab version POSTed directly to the GitLab issues
71# API. This version writes a Markdown report to a file and emits
72# `has_drift=true|false`, `scan_complete=true|false`, and `report_path=…`
73# on $GITHUB_OUTPUT so the calling workflow can manage a rolling issue.
74#
75# Environment inputs:
76# WORKFLOWS_DIR default: .github/workflows
77#
78# Outputs (via $GITHUB_OUTPUT):
79# has_drift "true" when any version is outdated, else "false"
80# scan_complete "false" when any check fails or is explicitly skipped,
81# else "true"
82# report_path path to the Markdown report (only set when has_drift=true)
83# =============================================================================
8426set -uo pipefail
85
8626WORKFLOWS_DIR="${WORKFLOWS_DIR:-.github/workflows}"
8752REPORT_FILE="$(mktemp -t dep-scan-XXXXXX.md 2>/dev/null || mktemp --suffix=.md)"
8852INCOMPLETE_REASONS_FILE="$(mktemp -t dep-scan-incomplete-XXXXXX 2>/dev/null || mktemp)"
89
90# Persist incomplete reasons to a file because many checks run in pipeline
91# subshells. A shell variable assignment made there would be lost, while an
92# append to this channel survives and is consumed by the final completeness
93# predicate. Messages go to stderr so they cannot corrupt command substitutions.
94mark_scan_incomplete() {
9595 local reason="$1"
9695 printf '%s\n' "$reason" >> "$INCOMPLETE_REASONS_FILE"
9795 echo " INCOMPLETE: $reason" >&2
98}
99
100join_scan_incomplete_reasons() {
10111 sort -u "$INCOMPLETE_REASONS_FILE" \
10211 | awk 'BEGIN { first = 1 } { if (!first) printf "; "; printf "%s", $0; first = 0 } END { print "" }'
103}
104
105# Source shared functions (also used by BATS tests)
106104SCAN_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
107# shellcheck source=.github/scripts/lib_dependency_scan.sh
10826source "${SCAN_SCRIPT_DIR}/lib_dependency_scan.sh"
109
110# ---------------------------------------------------------------------------
111# Report helpers
112#
113# Small Markdown emitters shared by every section of the drift report so the
114# table formatting lives in one place — previously each section hand-rolled
115# its own ``| … |`` header + separator, which drifted as sections were added.
116# ``emit_md_table`` turns a pipe-delimited results file into a GitHub table;
117# ``md_anchor`` builds the in-page heading slug the top-of-report summary
118# links to.
119#
120# Thresholds for the recurring-hygiene checks. Tunable in one place:
121# SUPPRESSION_EXPIRY_WARN_DAYS surface .trivyignore / .pip-audit-ignore /
122# .npm-audit-ignore entries expiring within
123# this many days
124# (the CI validator still hard-fails on the
125# day itself — this is the early warning).
126# SECURITY_EPOCH_STALE_DAYS flag a Dockerfile APT/DNF security epoch
127# older than this many days.
128# ---------------------------------------------------------------------------
12926SUPPRESSION_EXPIRY_WARN_DAYS="${SUPPRESSION_EXPIRY_WARN_DAYS:-30}"
13026SECURITY_EPOCH_STALE_DAYS="${SECURITY_EPOCH_STALE_DAYS:-45}"
131
132# md_anchor <title> — GitHub heading slug (lowercase, punctuation dropped,
133# spaces → hyphens). Close enough to GitHub's own algorithm for the
134# summary-table links to resolve.
135md_anchor() {
136418 printf '%s' "$1" \
137418 | tr '[:upper:]' '[:lower:]' \
138418 | sed -E 's/[^a-z0-9 -]//g; s/ /-/g'
139}
140
141# emit_md_table <header> <results-file> [wrap]
142#
143# <header> pipe-delimited column labels, e.g. "Package|Current|Latest"
144# <results-file> file of pipe-delimited rows with the same column count
145# [wrap] when "code", every non-empty cell is wrapped in backticks
146#
147# Prints a GitHub-flavoured Markdown table. With no wrap, cells are emitted
148# verbatim, so a caller that wants a link in a cell just writes the
149# ``[text](url)`` markdown straight into the results file.
150emit_md_table() {
15177 local header="$1" file="$2" wrap="${3:-}"
15277 local -a cells cols
153154 IFS='|' read -r -a cells <<< "$header"
15477 local head="|" sep="|" c
155274 for c in "${cells[@]}"; do
156274 head+=" ${c} |"
157274 sep+="---|"
158 done
15977 printf '%s\n%s\n' "$head" "$sep"
160728 while IFS='|' read -r -a cols; do
161287 [ "${#cols[@]}" -eq 0 ] && continue
162287 local row="|" cell
163982 for cell in "${cols[@]}"; do
1641012 if [ "$wrap" = "code" ] && [ -n "$cell" ]; then
16530 row+=" \`${cell}\` |"
166 else
167952 row+=" ${cell} |"
168 fi
169 done
170287 printf '%s\n' "$row"
171 done < "$file"
172}
173
174# days_until <YYYY-MM-DD> — integer days from today to the given date
175# (negative when the date is in the past). Empty output on a malformed date.
176days_until() {
17776 python3 -c "
178import datetime, sys
179try:
180 d = datetime.date.fromisoformat(sys.argv[1])
181except Exception:
182 sys.exit(0)
183print((d - datetime.date.today()).days)
184" "$1" 2>/dev/null
185}
186
187# days_since <YYYY-MM-DD> — integer days from the given date to today
188# (negative when the date is in the future). Empty output on a malformed date.
189days_since() {
19084 python3 -c "
191import datetime, sys
192try:
193 d = datetime.date.fromisoformat(sys.argv[1])
194except Exception:
195 sys.exit(0)
196print((datetime.date.today() - d).days)
197" "$1" 2>/dev/null
198}
199
200# ---------------------------------------------------------------------------
201# Python packages
202#
203# We run ``pip list --outdated`` on the installed interpreter, but
204# filter the JSON result down to packages we pin *directly* in
205# ``pyproject.toml::[project.dependencies]`` or the
206# ``[project.optional-dependencies]`` groups. Every other outdated
207# entry is a transitive dependency — its version is controlled by
208# something we pin (``jsii``, ``aws-cdk-lib``, ``fastmcp``,
209# ``botocore``, …) and bumping it ourselves either does nothing or
210# breaks the resolver. Leaving those entries in the monthly scan
211# report was creating noise: the operator had no action to take on
212# them beyond "wait for upstream". Filter them out so the report
213# only lists packages we can act on.
214#
215# The ``[build-system]`` requires pins (the exact-pinned build backend)
216# are appended to the same surface below via a direct PyPI lookup — pip
217# resolves them inside build isolation, so ``pip list`` never sees them.
218# ---------------------------------------------------------------------------
21926echo "=== Checking for outdated Python dependencies ==="
22026
22126# Install the project with EVERY optional-dependency group, not just the
22226# base dependencies: ``pip list --outdated`` can only report packages that
22326# are installed, so a base-only install silently dropped pins that live
22426# exclusively in an extras group (``aws-cdk-lib`` in ``cdk``, ``playwright``
22526# in ``diagrams``, ``mypy`` in ``typecheck``, ...) even though
22626# ``extract_direct_python_deps`` already includes them in the direct-pin
22726# filter below. Groups are enumerated from pyproject.toml so a new extras
22826# group joins the surface automatically; if enumeration fails, fall back to
22926# the old base-only install rather than dropping the report section.
23078PYTHON_EXTRAS="$(extract_python_extras pyproject.toml | paste -sd, -)"
23126if [ -z "$PYTHON_EXTRAS" ]; then
2321 mark_scan_incomplete "Could not enumerate optional dependency groups from pyproject.toml."
233fi
23426if [ -n "$PYTHON_EXTRAS" ]; then
23525 echo "Installing with extras: [${PYTHON_EXTRAS}]"
23625 if ! pip install -e ".[${PYTHON_EXTRAS}]" --quiet --root-user-action=ignore; then
2371 mark_scan_incomplete "Python dependency installation failed."
238 fi
239else
2401 if ! pip install -e . --quiet --root-user-action=ignore; then
2411 mark_scan_incomplete "Python dependency installation failed and optional groups could not be enumerated."
242 fi
243fi
24452if ! OUTDATED_RAW="$(pip list --outdated --format=json)"; then
2451 mark_scan_incomplete "pip list --outdated failed."
2461 OUTDATED_RAW="[]"
24750elif ! printf '%s' "$OUTDATED_RAW" | jq -e 'type == "array"' >/dev/null 2>&1; then
2481 mark_scan_incomplete "pip list --outdated returned malformed JSON."
2491 OUTDATED_RAW="[]"
250fi
251
252# Build a newline-separated list of PEP-503-normalised direct-dep names.
253# An empty list disables the filter so we never silently hide drift when
254# the TOML parse breaks. In practice the file always parses — we just
255# can't risk a dropped report section.
25652DIRECT_DEPS="$(extract_direct_python_deps pyproject.toml)"
25726if [ -z "$DIRECT_DEPS" ]; then
2581 mark_scan_incomplete "Could not parse direct Python dependencies from pyproject.toml."
259fi
260
26178if ! OUTDATED="$(printf '%s' "$OUTDATED_RAW" | python3 -c "
26278import json, re, sys
26378raw = sys.stdin.read()
26478direct = set(
26578 line.strip() for line in '''$DIRECT_DEPS'''.splitlines() if line.strip()
26678)
26778try:
26878 data = json.loads(raw) if raw else []
26978except json.JSONDecodeError:
27078 raise SystemExit(1)
27178if direct:
27278 data = [
27378 e for e in data
27478 if re.sub(r'[-_.]+', '-', e.get('name', '')).lower() in direct
27578 ]
27678print(json.dumps(data))
27778")"; then
2781 mark_scan_incomplete "Could not parse or filter pip's outdated-package response."
2791 OUTDATED="[]"
280fi
281
282# Build-backend pins ([build-system] requires) are Python dependencies
283# too, but they are invisible to ``pip list --outdated``: pip resolves
284# them inside build isolation, not in this venv (a Python 3.14 venv does
285# not even ship setuptools). Compare each exact pin against its PyPI
286# latest and report it through the same Python-packages surface as every
287# other pyproject pin. Non-exact entries are skipped here — the
288# version-consistency section flags those as a policy finding.
28952BUILD_SYSTEM_PINS="$(extract_build_system_pins pyproject.toml)"
29026if [ -z "$BUILD_SYSTEM_PINS" ]; then
2911 mark_scan_incomplete "Could not parse [build-system] requirements from pyproject.toml."
292else
293100 while IFS='|' read -r bs_name bs_version bs_raw; do
294 # Only exact pins are compared; non-exact entries surface through
295 # the version-consistency policy check instead.
29650 if [ -z "$bs_name" ] || [ -z "$bs_version" ]; then
2971 continue
298 fi
29995 if ! bs_latest="$(curl -fsSL --max-time 15 \
300 "https://pypi.org/pypi/${bs_name}/json" 2>/dev/null \
301 | jq -r '.info.version // empty' 2>/dev/null)" || [ -z "$bs_latest" ]; then
3021 mark_scan_incomplete "PyPI lookup failed or returned an invalid version for build dependency ${bs_name}."
3031 continue
304 fi
30546 if [ "$(compare_semver "$bs_version" "$bs_latest")" = "newer" ]; then
30612 OUTDATED="$(echo "$OUTDATED" | jq \
307 --arg name "$bs_name" --arg cur "$bs_version" --arg latest "$bs_latest" \
308 '. + [{"name": $name, "version": $cur, "latest_version": $latest}]')"
309 fi
310 done <<< "$BUILD_SYSTEM_PINS"
311fi
312
31378PYTHON_COUNT="$(echo "$OUTDATED" | jq 'length')"
31426if [ "$PYTHON_COUNT" -eq 0 ]; then
31522 echo "All Python dependencies are up to date."
31622 PYTHON_OUTDATED=""
317else
3184 echo "Found $PYTHON_COUNT outdated Python package(s) (direct dependencies only — transitive bumps are upstream's job)"
3198 echo "$OUTDATED" | jq -r '.[] | " - \(.name): \(.version) -> \(.latest_version)"'
3204 PYTHON_OUTDATED="$OUTDATED"
321fi
322
323# ---------------------------------------------------------------------------
324# npm packages (every repository-owned graph)
325# ---------------------------------------------------------------------------
326# Direct npm dependencies never had a drift surface of their own: aws-cdk and
327# markdownlint-cli2 only leaked into the report indirectly (via the
328# Dockerfile.dev ARG and the pre-commit hook rev), and the
329# inference-streaming-proxy's @aws-sdk clients were reported nowhere at all.
330# This walks the same repository-owned graphs the npm package-management
331# check validates and compares each exact direct pin against the registry's
332# ``latest`` dist-tag — the npm analogue of the Python Packages surface.
33326echo ""
33426echo "=== Checking for outdated npm packages ==="
33526
33652NPM_RESULTS="$(mktemp)"
33726NPM_COUNT=0
338
33952NPM_PACKAGE_DIRS="$(list_npm_package_dirs .)"
34026if [ -z "$NPM_PACKAGE_DIRS" ]; then
3411 mark_scan_incomplete "Could not enumerate repository-owned npm package manifests."
342fi
343108while IFS= read -r package_dir; do
34429 [ -n "$package_dir" ] || continue
34527 manifest="${package_dir}/package.json"
34654 package_pins="$(extract_npm_direct_pins "$manifest")"
34727 if [ -z "$package_pins" ]; then
3481 mark_scan_incomplete "Could not parse exact npm dependency pins from ${manifest}."
3491 continue
350 fi
351120 while IFS='|' read -r pkg_name pkg_version; do
35234 [ -n "$pkg_name" ] || continue
353 # Scoped names carry a '/', which must be encoded in the registry URL.
354102 encoded_name="$(printf '%s' "$pkg_name" | sed 's|/|%2F|g')"
355135 if ! pkg_latest="$(curl -fsSL --max-time 15 \
356 "https://registry.npmjs.org/${encoded_name}/latest" 2>/dev/null \
357 | jq -r '.version // empty' 2>/dev/null)" || [ -z "$pkg_latest" ]; then
3581 mark_scan_incomplete "npm registry lookup failed or returned an invalid version for ${pkg_name}."
3591 continue
360 fi
36166 if [ "$(compare_semver "$pkg_version" "$pkg_latest")" = "newer" ]; then
3629 echo " - ${package_dir}: ${pkg_name} ${pkg_version} -> ${pkg_latest}"
3639 echo "${package_dir}|${pkg_name}|${pkg_version}|${pkg_latest}" >> "$NPM_RESULTS"
364 fi
365 done <<< "$package_pins"
366done <<< "$NPM_PACKAGE_DIRS"
367
36878NPM_COUNT="$(wc -l < "$NPM_RESULTS" | tr -d ' ')"
36926if [ "$NPM_COUNT" -eq 0 ]; then
37022 echo "All npm direct dependencies are up to date."
371else
3724 echo "Found $NPM_COUNT outdated npm package pin(s) across the owned graphs"
373fi
374
375# ---------------------------------------------------------------------------
376# Docker image tags
377# ---------------------------------------------------------------------------
37826echo ""
37926echo "=== Checking for outdated Docker images ==="
38026
38152DOCKER_RESULTS="$(mktemp)"
38252ALL_IMAGES="$(mktemp)"
383
384check_image() {
385262 local image="$1"
386262 local current_tag="$2"
387
388 # Only handle semver tags
389262 if ! is_semver_tag "$current_tag"; then
39023 return
391 fi
392 # Skip images we build in this project
393239 if is_project_image "$image"; then
39423 return
395 fi
396
397216 local parsed registry repo
398432 parsed="$(parse_image_registry "$image")"
399648 registry="$(echo "$parsed" | cut -d'|' -f1)"
400648 repo="$(echo "$parsed" | cut -d'|' -f2)"
401
402 # Fetch and filter separately. A registry/network failure marks the scan
403 # incomplete, while "the registry answered and nothing newer exists in
404 # this variant family" is an up-to-date pin. The previous single pipeline
405 # conflated the two under pipefail: the strict bare-semver grep matched
406 # nothing for suffix-tagged repositories (…-py3, …-cuda…, …-ubuntu…), so
407 # they reported "tag lookup failed" every month even though the registry
408 # was fine.
409216 local raw_tags=""
410857 if ! raw_tags="$(skopeo list-tags --retry-times 3 "docker://${registry}/${repo}" 2>/dev/null \
411 | jq -r '.Tags[]' 2>/dev/null)" || [ -z "$raw_tags" ]; then
41214 mark_scan_incomplete "Container registry tag lookup failed for ${registry}/${repo}."
41314 return
414 fi
415
416202 local latest_tag
417606 latest_tag="$(printf '%s\n' "$raw_tags" | newer_same_variant_tag "$current_tag")" || latest_tag=""
418
419202 if [ -n "$latest_tag" ]; then
42050 echo " - ${image}:${current_tag} -> ${latest_tag}"
42150 echo "${image}|${current_tag}|${latest_tag}" >> "$DOCKER_RESULTS"
42250 return
423 fi
424
425 # No newer family member. If the pinned tag itself is no longer listed,
426 # the pin points at something the registry stopped advertising (renamed
427 # variant scheme, withdrawn tag) — that deserves eyes, not silence.
428 # tag_listed reads the list from an argument, not a printf pipe: under
429 # pipefail, grep -q's early exit gave printf SIGPIPE on large tag lists
430 # and inverted "tag present" into a false INCOMPLETE (2026-09 scan).
431152 if ! tag_listed "$current_tag" "$raw_tags"; then
4325 mark_scan_incomplete "Pinned tag ${current_tag} is no longer listed by ${registry}/${repo}."
433 fi
434}
435
436# Collect image:tag pairs from workflow files (bare `image:` references in
437# container specs and `uses: …@sha` are handled by Dependabot; here we look
438# for free-form image references in run steps).
43926echo "Checking workflow files in $WORKFLOWS_DIR..."
44026if [ -d "$WORKFLOWS_DIR" ]; then
44125 grep -rhoE "image: [a-zA-Z0-9_./-]+:[a-zA-Z0-9._-]+" "$WORKFLOWS_DIR" 2>/dev/null \
44248 | sed 's/image: //' >> "$ALL_IMAGES" || true
44325 grep -rhoE "[a-zA-Z0-9_./-]+:[a-zA-Z0-9._-]+" "$WORKFLOWS_DIR" 2>/dev/null \
44425 | grep -E '^(alpine|hadolint|koalaman|semgrep|bridgecrew|checkmarx|trufflesecurity|zricethezav|aquasec|bats|python):' \
44525 | sed 's/[[:space:]]*$//' >> "$ALL_IMAGES" || true
446fi
447
44826echo "Checking K8s manifest images..."
44926grep -rhoE "image: [a-zA-Z0-9_./-]+:[a-zA-Z0-9._-]+" lambda/kubectl-applier-simple/manifests/ 2>/dev/null \
45053 | grep -v '{{' | sed 's/image: //' >> "$ALL_IMAGES" || true
451
45226echo "Checking example manifest images..."
45326grep -rhoE "image: [a-zA-Z0-9_./-]+:[a-zA-Z0-9._-]+" examples/ 2>/dev/null \
45427 | sed 's/image: //' >> "$ALL_IMAGES" || true
455
45626echo "Checking local live-validation manifest images..."
45726grep -rhoE "image: [a-zA-Z0-9_./-]+:[a-zA-Z0-9._-]+" scripts/live_release_validation/manifests/ 2>/dev/null \
45827 | sed 's/image: //' >> "$ALL_IMAGES" || true
459
46026echo "Checking Helm chart value images..."
461# Registry-aware walk over every charts.yaml values block (see
462# extract_chart_value_images in lib_dependency_scan.sh — moved there so BATS
463# exercises the real logic). charts.yaml always pins values images, so an
464# empty result means the parse broke — surface that as an incomplete scan
465# rather than silently dropping the sweep.
46652CHART_VALUE_IMAGES="$(extract_chart_value_images lambda/helm-installer/charts.yaml)"
46726if [ -n "$CHART_VALUE_IMAGES" ]; then
46823 printf '%s\n' "$CHART_VALUE_IMAGES" >> "$ALL_IMAGES"
469else
4703 mark_scan_incomplete "Could not parse Helm chart value images."
471fi
472
473# Mooncake default image — pinned as a Python constant in cli/images.py
474# (_DISAGGREGATED_DEFAULT_IMAGE), so it is invisible to Dependabot (docker
475# ecosystem) and to the manifest/workflow sweeps above. Add it here so a newer
476# upstream vLLM release shows up in the monthly drift report — the cue to
477# validate and bump the pin.
47826echo "Checking Mooncake default image (cli/images.py)..."
47952MOONCAKE_IMAGE="$(extract_mooncake_default_image cli/images.py)"
48026if [ -n "$MOONCAKE_IMAGE" ]; then
48125 echo "$MOONCAKE_IMAGE" >> "$ALL_IMAGES"
482else
4831 mark_scan_incomplete "Could not parse the Mooncake default image from cli/images.py."
484fi
485
486# The model-sync init container uses an official AWS CLI image pinned by both
487# version and manifest-list digest. Strip only the digest for the registry tag
488# lookup; the offline provenance contract separately requires the digest.
48926echo "Checking AWS CLI runtime image (gco/services/inference_monitor.py)..."
49052AWS_CLI_RUNTIME_IMAGE="$(extract_python_string_constant \
491 AWS_CLI_IMAGE gco/services/inference_monitor.py)"
492# check_pinned_digest <repo:tag@sha256:digest> <origin label>
493#
494# Shared digest-freshness check for every digest-pinned image the repository
495# commits: verify the tag's currently published manifest-list digest still
496# equals the committed one. A moved digest is a drift row (the tag was
497# re-pushed upstream — the pin is stale); an unreachable registry or an
498# implausible response marks the scan incomplete.
499check_pinned_digest() {
50053 local pinned_ref="$1" origin="$2" parts repository tag committed published
501106 if ! parts="$(split_pinned_image_ref "$pinned_ref")"; then
5021 mark_scan_incomplete "Could not parse an immutable image reference from ${origin}."
5031 return
504 fi
505156 repository="$(echo "$parts" | cut -d'|' -f1)"
506156 tag="$(echo "$parts" | cut -d'|' -f2)"
507156 committed="$(echo "$parts" | cut -d'|' -f3)"
50852 printf '%s:%s\n' "$repository" "$tag" >> "$ALL_IMAGES"
509
510104 if ! published="$(published_manifest_digest "${repository}:${tag}")"; then
5112 mark_scan_incomplete "Container manifest lookup failed for ${repository}:${tag}."
5122 return
513 fi
51450 if [ "$committed" != "$published" ]; then
51512 echo " - ${repository}:${tag}: committed digest does not match the tag (${origin})"
51612 echo "${repository}|${tag}@${committed}|${tag}@${published}" >> "$DOCKER_RESULTS"
517 fi
518}
519
52026if [[ "$AWS_CLI_RUNTIME_IMAGE" =~ ^[^@]+:[^@]+@sha256:[0-9a-f]{64}$ ]]; then
52125 check_pinned_digest "$AWS_CLI_RUNTIME_IMAGE" "gco/services/inference_monitor.py"
522else
5231 mark_scan_incomplete "Could not parse an immutable AWS_CLI_IMAGE from gco/services/inference_monitor.py."
524fi
525
526# The live-validation smoke manifests pin every image by tag AND manifest-list
527# digest (tests/test_live_release_validation.py enforces the shape). The tag
528# half already rides the normal drift check above; this pass keeps the digest
529# half honest too, so an upstream same-tag re-push shows up as drift instead
530# of silently diverging from what a validation run would actually pull.
53126echo "Checking live-validation smoke image digests (scripts/live_release_validation/manifests)..."
532104SMOKE_PINNED_REFS="$(grep -rhoE \
533 "image: [a-zA-Z0-9_./-]+:[a-zA-Z0-9._-]+@sha256:[0-9a-f]{64}" \
534 scripts/live_release_validation/manifests/ 2>/dev/null | sed 's/^image: //' | sort -u)"
53526if [ -z "$SMOKE_PINNED_REFS" ]; then
5361 mark_scan_incomplete "No digest-pinned smoke images found under scripts/live_release_validation/manifests/."
537else
53853 while read -r pinned_ref; do
53928 [ -z "$pinned_ref" ] && continue
54028 check_pinned_digest "$pinned_ref" "live-validation smoke manifest"
541 done <<< "$SMOKE_PINNED_REFS"
542fi
543
544314sort -u "$ALL_IMAGES" | while read -r img; do
545262 [ -z "$img" ] && continue
546786 image="$(echo "$img" | cut -d':' -f1)"
547786 tag="$(echo "$img" | cut -d':' -f2)"
548262 check_image "$image" "$tag"
549done
55026rm -f "$ALL_IMAGES"
551
55278DOCKER_COUNT="$(wc -l < "$DOCKER_RESULTS" | tr -d ' ')"
55326[ -z "$DOCKER_COUNT" ] && DOCKER_COUNT=0
554
555# ---------------------------------------------------------------------------
556# Helm chart versions
557# ---------------------------------------------------------------------------
55826echo ""
55926echo "=== Checking Helm chart versions ==="
56026
56152HELM_RESULTS="$(mktemp)"
56226CHARTS_FILE="lambda/helm-installer/charts.yaml"
563
56426if [ -f "$CHARTS_FILE" ]; then
56550 CHART_ENTRIES="$(extract_helm_charts "$CHARTS_FILE")"
56625 if [ -z "$CHART_ENTRIES" ]; then
5671 mark_scan_incomplete "Could not parse Helm chart pins from ${CHARTS_FILE}."
568 fi
569200 while IFS= read -r entry; do
57076 [ -n "$entry" ] || continue
571222 chart_name="$(echo "$entry" | jq -r '.name // empty')"
572222 repo_url="$(echo "$entry" | jq -r '.repo_url // empty')"
573222 chart="$(echo "$entry" | jq -r '.chart // empty')"
574222 current="$(echo "$entry" | jq -r '.version // empty')"
575222 use_oci="$(echo "$entry" | jq -r '.use_oci // false')"
576295 if [ -z "$chart_name" ] || [ -z "$repo_url" ] || [ -z "$chart" ] || [ -z "$current" ]; then
5771 mark_scan_incomplete "Helm chart parser emitted an incomplete record from ${CHARTS_FILE}."
5781 continue
579 fi
580
58173 latest=""
58273 if [ "$use_oci" = "true" ]; then
583132 if ! latest="$(helm show chart "${repo_url}/${chart}" 2>/dev/null | grep '^version:' | awk '{print $2}')" \
58431 || [ -z "$latest" ]; then
5852 mark_scan_incomplete "Helm OCI lookup failed for ${repo_url}/${chart}."
5862 continue
587 fi
588 else
58940 if ! helm repo add "$chart_name" "$repo_url" --force-update > /dev/null 2>&1; then
5901 mark_scan_incomplete "Helm repository refresh failed for ${chart_name} (${repo_url})."
5911 continue
592 fi
593156 if ! latest="$(helm search repo "${chart_name}/${chart}" --output json 2>/dev/null \
594 | jq -r '.[0].version // empty')" || [ -z "$latest" ]; then
5952 mark_scan_incomplete "Helm chart lookup failed for ${chart_name}/${chart}."
5962 continue
597 fi
598 fi
599
60068 if [ "$current" != "$latest" ]; then
60121 current_stripped="${current#v}"
60221 latest_stripped="${latest#v}"
60321 if [ "$current_stripped" != "$latest_stripped" ]; then
60421 echo " - ${chart_name} (${chart}): ${current} -> ${latest}"
60521 echo "${chart_name}|${chart}|${current}|${latest}" >> "$HELM_RESULTS"
606 fi
607 fi
608 done <<< "$CHART_ENTRIES"
609else
6101 mark_scan_incomplete "${CHARTS_FILE} is missing."
611fi
612
61378HELM_COUNT="$(wc -l < "$HELM_RESULTS" 2>/dev/null | tr -d ' ')"
61426[ -z "$HELM_COUNT" ] && HELM_COUNT=0
615
616# ---------------------------------------------------------------------------
617# EKS add-on versions (best-effort — requires AWS credentials)
618#
619# Pre-flight: probe for usable AWS credentials. If `sts get-caller-identity`
620# fails the scan is skipped entirely and a one-line note goes into both the
621# console log and the Markdown report — this is more honest than silently
622# dropping the section. Wire AWS creds through OIDC (see the deps-scan
623# section in .github/CI.md) to enable the check.
624# ---------------------------------------------------------------------------
62526echo ""
62626echo "=== Checking EKS add-on versions ==="
62726
62852ADDON_RESULTS="$(mktemp)"
62926ADDON_SKIP_REASON=""
63052K8S_VERSION="$(extract_k8s_version "")"
631
63226if ! aws sts get-caller-identity >/dev/null 2>&1; then
6333 ADDON_SKIP_REASON="No AWS credentials available (scan needs eks:DescribeAddonVersions). Configure OIDC to enable."
6343 echo " $ADDON_SKIP_REASON"
635else
63646 EKS_ADDONS="$(extract_eks_addons "gco/stacks/regional_stack.py")"
63723 if [ -z "$EKS_ADDONS" ]; then
6381 ADDON_SKIP_REASON="Could not read EKS add-on pins from gco/stacks/regional_stack.py."
6391 echo " $ADDON_SKIP_REASON"
640 else
641244 while IFS='|' read -r addon_name current_version; do
642102 [ -z "$addon_name" ] && continue
643204 latest="$(aws eks describe-addon-versions \
644 --addon-name "$addon_name" \
645 --kubernetes-version "$K8S_VERSION" \
646 --query 'addons[0].addonVersions[0].addonVersion' \
647 --output text 2>/dev/null)" || latest=""
648
649102 if ! [[ "$latest" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-eksbuild\.[0-9]+$ ]]; then
6502 ADDON_SKIP_REASON="EKS add-on lookup failed or returned an invalid version for ${addon_name}."
6512 echo " $ADDON_SKIP_REASON"
6522 break
653 fi
654100 if [ "$current_version" != "$latest" ]; then
65520 echo " - ${addon_name}: ${current_version} -> ${latest}"
65620 echo "${addon_name}|${current_version}|${latest}" >> "$ADDON_RESULTS"
657 fi
658 done <<< "$EKS_ADDONS"
659 fi
660fi
661
66278ADDON_COUNT="$(wc -l < "$ADDON_RESULTS" 2>/dev/null | tr -d ' ')"
66326[ -z "$ADDON_COUNT" ] && ADDON_COUNT=0
664
665# ---------------------------------------------------------------------------
666# EKS Kubernetes version (best-effort — requires AWS credentials)
667#
668# Compares ``kubernetes_version`` in cdk.json against the latest minor
669# available from ``aws eks describe-cluster-versions`` (filtered to
670# ``STANDARD_SUPPORT``). If a newer minor is available, we also report
671# when standard support for the currently-pinned minor ends so the
672# upgrade urgency is visible in the PR.
673#
674# IAM action: ``eks:DescribeClusterVersions``. Same credential preflight
675# as the EKS add-on / Aurora / EMR checks.
676# ---------------------------------------------------------------------------
67726echo ""
67826echo "=== Checking EKS Kubernetes version ==="
67926
68052EKS_K8S_RESULTS="$(mktemp)"
68126EKS_K8S_SKIP_REASON=""
682
68326if ! aws sts get-caller-identity >/dev/null 2>&1; then
6843 EKS_K8S_SKIP_REASON="No AWS credentials available (scan needs eks:DescribeClusterVersions). Configure OIDC to enable."
6853 echo " $EKS_K8S_SKIP_REASON"
686else
687 # ``--version-status STANDARD_SUPPORT`` returns every minor still in
688 # standard support — we don't want to flag the extended-support
689 # lifecycle as "newer." It must be the only selector: the API rejects
690 # combining it with ``--include-all`` ("Only one of the defaultOnly,
691 # clusterVersions, includeAll or status request parameters is accepted
692 # at a time"), which is exactly how this check silently broke once.
693 # stderr is captured into the skip reason so the next API-shape change
694 # is diagnosable from the report instead of reading as a generic skip.
69546 EKS_K8S_ERR_FILE="$(mktemp)"
69647 CLUSTER_VERSIONS_JSON="$(aws eks describe-cluster-versions \
697 --version-status STANDARD_SUPPORT \
698 --output json 2>"$EKS_K8S_ERR_FILE")" || CLUSTER_VERSIONS_JSON=""
699
70023 if [ -z "$CLUSTER_VERSIONS_JSON" ]; then
7013 EKS_K8S_ERR="$(head -n 1 "$EKS_K8S_ERR_FILE" 2>/dev/null | tr -d '\r')"
7021 EKS_K8S_SKIP_REASON="EKS Kubernetes version lookup failed${EKS_K8S_ERR:+: ${EKS_K8S_ERR}}"
7031 [ -z "$EKS_K8S_ERR" ] && EKS_K8S_SKIP_REASON="EKS Kubernetes version lookup returned an empty response."
7041 echo " $EKS_K8S_SKIP_REASON"
7051 rm -f "$EKS_K8S_ERR_FILE"
706 else
70722 rm -f "$EKS_K8S_ERR_FILE"
708 # Max of ``clusterVersion`` across all rows is the newest standard-
709 # support minor. We use Python for a proper numeric sort so 1.10
710 # beats 1.9 (sort -V already does this, but Python keeps the data
711 # wrangling in one place).
71267 LATEST_K8S="$(echo "$CLUSTER_VERSIONS_JSON" | python3 -c '
713import json, sys
714try:
715 data = json.load(sys.stdin)
716except Exception:
717 sys.exit(0)
718versions = sorted(
719 {row["clusterVersion"] for row in data.get("clusterVersions", [])},
720 key=lambda v: tuple(int(p) for p in v.split(".")),
721)
722print(versions[-1] if versions else "")
723' 2>/dev/null)" || LATEST_K8S=""
724
72522 if [ -z "$LATEST_K8S" ]; then
7262 EKS_K8S_SKIP_REASON="EKS Kubernetes version response contained no parseable standard-support versions."
7272 echo " $EKS_K8S_SKIP_REASON"
728 else
72940 CURRENT_K8S="$(extract_k8s_version "cdk.json")"
730
73120 if [ "$CURRENT_K8S" != "$LATEST_K8S" ] \
7326 && [ "$(compare_semver "$CURRENT_K8S" "$LATEST_K8S")" = "newer" ]; then
733 # Grab the standard-support end date for the currently-pinned
734 # minor. Blank when EKS hasn't published one yet (brand-new release).
7359 EOS_DATE="$(echo "$CLUSTER_VERSIONS_JSON" | python3 -c "
7369import json, sys
7379cv = sys.argv[1]
7389try:
7399 data = json.load(sys.stdin)
7409except Exception:
7419 sys.exit(0)
7429for row in data.get('clusterVersions', []):
7439 if row.get('clusterVersion') == cv:
7449 ts = row.get('endOfStandardSupportDate', '')
7459 # Strip time-of-day; the date is what the report cares about.
7469 print(str(ts).split('T', 1)[0].split(' ', 1)[0])
7479 break
7489" "$CURRENT_K8S" 2>/dev/null)" || EOS_DATE=""
749
7503 echo " - kubernetes_version: ${CURRENT_K8S} -> ${LATEST_K8S} (std support ends ${EOS_DATE:-unknown})"
7513 echo "kubernetes_version|${CURRENT_K8S}|${LATEST_K8S}|${EOS_DATE:-unknown}" >> "$EKS_K8S_RESULTS"
752 fi
753 fi
754 fi
755fi
756
75778EKS_K8S_COUNT="$(wc -l < "$EKS_K8S_RESULTS" 2>/dev/null | tr -d ' ')"
75826[ -z "$EKS_K8S_COUNT" ] && EKS_K8S_COUNT=0
759
760# ---------------------------------------------------------------------------
761# Aurora PostgreSQL engine versions (best-effort — requires AWS credentials)
762#
763# Checks whether the Aurora PostgreSQL engine version pinned in
764# regional_stack.py has a newer minor or major release available.
765# Uses the same credential gate as the EKS add-on check above.
766# ---------------------------------------------------------------------------
76726echo ""
76826echo "=== Checking Aurora PostgreSQL engine versions ==="
76926
77052AURORA_RESULTS="$(mktemp)"
77126AURORA_SKIP_REASON=""
772
77326if ! aws sts get-caller-identity >/dev/null 2>&1; then
7743 AURORA_SKIP_REASON="No AWS credentials available (scan needs rds:DescribeDBEngineVersions). Configure OIDC to enable."
7753 echo " $AURORA_SKIP_REASON"
776else
777 # The pinned Aurora PostgreSQL version (AURORA_POSTGRES_VERSION in
778 # gco/stacks/constants.py — a plain version string applied through
779 # AuroraPostgresEngineVersion.of(), so no CDK enum is involved).
78046 AURORA_VERSIONS="$(extract_aurora_versions "gco/stacks/regional_stack.py")"
78123 if [ -z "$AURORA_VERSIONS" ]; then
7821 AURORA_SKIP_REASON="Could not read AURORA_POSTGRES_VERSION from gco/stacks/constants.py."
7831 echo " $AURORA_SKIP_REASON"
784 else
78543 while read -r current_ver; do
78622 [ -z "$current_ver" ] && continue
78766 major="$(echo "$current_ver" | cut -d. -f1)"
788
789 # Query the latest available engine version for this major line.
790110 latest="$(aws rds describe-db-engine-versions \
791 --engine aurora-postgresql \
792 --query "DBEngineVersions[?starts_with(EngineVersion, '${major}.')].EngineVersion" \
793 --output text 2>/dev/null \
794 | tr '\t' '\n' | sort -V | tail -1)" || latest=""
795
79622 if ! [[ "$latest" =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?$ ]]; then
7971 AURORA_SKIP_REASON="Aurora PostgreSQL engine lookup failed or returned an invalid version for major ${major}."
7981 echo " $AURORA_SKIP_REASON"
7991 break
800 fi
80121 if [ "$current_ver" != "$latest" ]; then
8024 echo " - aurora-postgresql: ${current_ver} -> ${latest}"
8034 echo "aurora-postgresql|${current_ver}|${latest}" >> "$AURORA_RESULTS"
804 fi
805 done <<< "$AURORA_VERSIONS"
806 fi
807fi
808
80978AURORA_COUNT="$(wc -l < "$AURORA_RESULTS" 2>/dev/null | tr -d ' ')"
81026[ -z "$AURORA_COUNT" ] && AURORA_COUNT=0
811
812# ---------------------------------------------------------------------------
813# EMR Serverless release labels (best-effort — requires AWS credentials)
814#
815# Checks whether the EMR Serverless release label pinned in
816# gco/stacks/constants.py has a newer release available. Uses the same
817# credential gate as the EKS add-on / Aurora checks above.
818#
819# AWS CLI note: the `list-release-labels` subcommand lives on the classic
820# `aws emr` service, not on `aws emr-serverless`. Classic EMR and EMR
821# Serverless share the same release-label namespace (e.g. emr-7.13.0),
822# so calling the classic service returns the labels usable by Serverless
823# applications. The IAM action is ``elasticmapreduce:ListReleaseLabels``
824# (which is what the OIDC policy grants) and is shared between the two
825# services — the CLI routing is just a surface-level difference.
826# ---------------------------------------------------------------------------
82726echo ""
82826echo "=== Checking EMR Serverless release labels ==="
82926
83052EMR_RESULTS="$(mktemp)"
83126EMR_SKIP_REASON=""
832
83326if ! aws sts get-caller-identity >/dev/null 2>&1; then
8343 EMR_SKIP_REASON="No AWS credentials available (scan needs elasticmapreduce:ListReleaseLabels). Configure OIDC to enable."
8353 echo " $EMR_SKIP_REASON"
836else
83746 EMR_VERSIONS="$(extract_emr_versions "gco/stacks/constants.py")"
83823 if [ -z "$EMR_VERSIONS" ]; then
8391 EMR_SKIP_REASON="Could not read the EMR Serverless release-label pin from gco/stacks/constants.py."
8401 echo " $EMR_SKIP_REASON"
841 else
84241 while read -r current_label; do
84322 [ -z "$current_label" ] && continue
844 # current_label looks like "emr-7.13.0". Filter labels to ones that
845 # start with "emr-<major>." and take the latest by semver-ish sort.
846 # Skip preview/nightly tags (``-preview``, ``-beta``, ``-rc*``). The
847 # latest release label is what we compare against.
84866 major="$(echo "$current_label" | sed -E 's/^emr-([0-9]+)\..*/\1/')"
84944 release_labels="$(aws emr list-release-labels \
850 --region us-east-1 \
851 --query 'ReleaseLabels[]' --output text 2>/dev/null)" || release_labels=""
852
85343 if [ -z "$release_labels" ] || [ "$release_labels" = "None" ]; then
8541 EMR_SKIP_REASON="EMR release-label lookup failed or returned an empty response."
8551 echo " $EMR_SKIP_REASON"
8561 break
857 fi
858
859128 latest="$(echo "$release_labels" \
860 | tr '\t' '\n' \
861 | grep -E "^emr-${major}\.[0-9]+\.[0-9]+$" \
862 | sort -V | tail -1)" || true
863
864 # Also check whether a newer major release line exists.
865128 latest_any="$(echo "$release_labels" \
866 | tr '\t' '\n' \
867 | grep -E "^emr-[0-9]+\.[0-9]+\.[0-9]+$" \
868 | sort -V | tail -1)" || true
869
87021 if [ -z "$latest_any" ]; then
8712 EMR_SKIP_REASON="EMR release-label response contained no parseable stable releases."
8722 echo " $EMR_SKIP_REASON"
8732 break
874 fi
87538 if [ -n "$latest" ] && [ "$current_label" != "$latest" ]; then
8762 echo " - emr-serverless: ${current_label} -> ${latest}"
8772 echo "emr-serverless|${current_label}|${latest}" >> "$EMR_RESULTS"
87817 elif [ "$current_label" != "$latest_any" ] \
8792 && [ "$(compare_semver "${current_label#emr-}" "${latest_any#emr-}")" = "newer" ]; then
880 # Same minor — no new release in our pinned major — but a new
881 # major exists.
8821 echo " - emr-serverless: ${current_label} -> ${latest_any} (new major available)"
8831 echo "emr-serverless|${current_label}|${latest_any}" >> "$EMR_RESULTS"
884 fi
885 done <<< "$EMR_VERSIONS"
886 fi
887fi
888
88978EMR_COUNT="$(wc -l < "$EMR_RESULTS" 2>/dev/null | tr -d ' ')"
89026[ -z "$EMR_COUNT" ] && EMR_COUNT=0
891
892# ---------------------------------------------------------------------------
893# Bedrock default model (best-effort — requires AWS credentials)
894#
895# Compares each configured Bedrock model default in cdk.json —
896# context.bedrock.mission_default_model_id (Mission sampling),
897# context.bedrock.capacity_advisor_default_model_id (the capacity
898# advisor), context.bedrock.claude_code_default_model_id (the session
899# model GCO Autopilot hands to Claude Code),
900# context.bedrock.codex_default_model_id (the session model it hands to Codex),
901# and context.bedrock.embedding_model_id (Mission memory's text-embedding
902# model) — against the newest release in the SAME model family. The three
903# generation keys compare against system-defined inference profiles
904# (aws bedrock list-inference-profiles); the embedding key is a plain
905# foundation model, so it compares against
906# aws bedrock list-foundation-models --by-output-modality EMBEDDING.
907# Every consumer resolves its key through gco.bedrock, so the scan and
908# the runtime paths cannot silently diverge; the keys are independent
909# knobs and each gets its own drift row.
910#
911# Same-family scoping (see bedrock_model_family) means we only flag a newer
912# release of the same model line (e.g. a newer global Amazon Nova Lite) — never a
913# different tier or provider, since switching those is a human decision,
914# not drift. When a newer release is reported, update the flagged key in
915# cdk.json; for the Mission key also re-capture the scaffold
916# fixture with scripts/capture_scaffold_fixtures.py. For the embedding
917# key, remember stored vectors are only comparable to vectors from the
918# same model: adopting a newer embedding model means re-embedding or
919# segregating existing Mission-memory data, not just bumping the pin.
920#
921# IAM actions: bedrock:ListInferenceProfiles and
922# bedrock:ListFoundationModels. Pinned to us-east-1 (the advisor +
923# Mission sampling + Mission memory default region) regardless of the
924# workflow's configured region. Same credential preflight as the EKS
925# add-on / Aurora / EMR checks.
926# ---------------------------------------------------------------------------
92726echo ""
92826echo "=== Checking Bedrock default model ==="
92926
93052BEDROCK_MODEL_RESULTS="$(mktemp)"
93126BEDROCK_MODEL_SKIP_REASON=""
932
93326if ! aws sts get-caller-identity >/dev/null 2>&1; then
9343 BEDROCK_MODEL_SKIP_REASON="No AWS credentials available (scan needs bedrock:ListInferenceProfiles). Configure OIDC to enable."
9353 echo " $BEDROCK_MODEL_SKIP_REASON"
936else
937115 for BEDROCK_MODEL_LEAF in mission_default_model_id capacity_advisor_default_model_id claude_code_default_model_id codex_default_model_id embedding_model_id; do
938230 CURRENT_BEDROCK_MODEL="$(extract_default_bedrock_model cdk.json "$BEDROCK_MODEL_LEAF")"
939115 if [ -z "$CURRENT_BEDROCK_MODEL" ]; then
9405 BEDROCK_MODEL_SKIP_REASON="Could not read context.bedrock.${BEDROCK_MODEL_LEAF} from cdk.json."
9415 echo " $BEDROCK_MODEL_SKIP_REASON"
9425 continue
943 fi
944110 if [ "$BEDROCK_MODEL_LEAF" = "embedding_model_id" ]; then
945 # Embedding defaults are foundation models, not inference profiles.
94644 LATEST_BEDROCK_MODEL="$(get_latest_bedrock_embedding_model "$CURRENT_BEDROCK_MODEL" us-east-1)" || LATEST_BEDROCK_MODEL=""
947 else
948176 LATEST_BEDROCK_MODEL="$(get_latest_bedrock_model "$CURRENT_BEDROCK_MODEL" us-east-1)" || LATEST_BEDROCK_MODEL=""
949 fi
950110 if [ -z "$LATEST_BEDROCK_MODEL" ]; then
95110 BEDROCK_MODEL_SKIP_REASON="Bedrock model lookup failed or returned no active release in the model family of context.bedrock.${BEDROCK_MODEL_LEAF}."
95210 echo " $BEDROCK_MODEL_SKIP_REASON"
953100 elif [ "$CURRENT_BEDROCK_MODEL" != "$LATEST_BEDROCK_MODEL" ] \
95430 && [ "$(compare_bedrock_model "$CURRENT_BEDROCK_MODEL" "$LATEST_BEDROCK_MODEL")" = "newer" ]; then
95515 echo " - bedrock ${BEDROCK_MODEL_LEAF}: ${CURRENT_BEDROCK_MODEL} -> ${LATEST_BEDROCK_MODEL}"
95615 echo "context.bedrock.${BEDROCK_MODEL_LEAF}|${CURRENT_BEDROCK_MODEL}|${LATEST_BEDROCK_MODEL}" >> "$BEDROCK_MODEL_RESULTS"
957 fi
958 done
959 # The vector store keeps its own embedding model at
960 # context.vector_store.embedding_model_id (independent of mission memory's
961 # bedrock.embedding_model_id by design). Same foundation-model drift check,
962 # same re-embed caveat: adopting a newer model means re-ingesting the corpus.
963 # The key is optional (the block ships with defaults), so absence is not a
964 # skip condition for the whole check.
96546 VECTOR_STORE_MODEL="$(extract_default_bedrock_model cdk.json embedding_model_id vector_store)"
96623 if [ -n "$VECTOR_STORE_MODEL" ]; then
96744 LATEST_VECTOR_STORE_MODEL="$(get_latest_bedrock_embedding_model "$VECTOR_STORE_MODEL" us-east-1)" || LATEST_VECTOR_STORE_MODEL=""
96822 if [ -z "$LATEST_VECTOR_STORE_MODEL" ]; then
9692 BEDROCK_MODEL_SKIP_REASON="Bedrock model lookup failed or returned no active release in the model family of context.vector_store.embedding_model_id."
9702 echo " $BEDROCK_MODEL_SKIP_REASON"
97120 elif [ "$VECTOR_STORE_MODEL" != "$LATEST_VECTOR_STORE_MODEL" ] \
9726 && [ "$(compare_bedrock_model "$VECTOR_STORE_MODEL" "$LATEST_VECTOR_STORE_MODEL")" = "newer" ]; then
9733 echo " - vector_store embedding_model_id: ${VECTOR_STORE_MODEL} -> ${LATEST_VECTOR_STORE_MODEL}"
9743 echo "context.vector_store.embedding_model_id|${VECTOR_STORE_MODEL}|${LATEST_VECTOR_STORE_MODEL}" >> "$BEDROCK_MODEL_RESULTS"
975 fi
976 fi
977fi
978
97978BEDROCK_MODEL_COUNT="$(wc -l < "$BEDROCK_MODEL_RESULTS" 2>/dev/null | tr -d ' ')"
98026[ -z "$BEDROCK_MODEL_COUNT" ] && BEDROCK_MODEL_COUNT=0
981
982
983# ---------------------------------------------------------------------------
984# Dockerfile.dev ARG pins
985#
986# Checks the tooling versions pinned in ``Dockerfile.dev`` (Node.js
987# release, AWS CDK CLI, kubectl, AWS CLI v2, Docker CLI, Docker Buildx). These ARGs sit
988# outside the main dependency surfaces above — Dependabot watches the
989# ``FROM python:…`` base image but not the ARG pins — so drift here has
990# historically gone undetected until someone rebuilt the image.
991#
992# Each pin has its own upstream:
993#
994# NODE_VERSION github://nodejs/Release → schedule.json (active LTS
995# major) + nodejs.org/dist/index.json (newest release
996# on that major)
997# NPM_VERSION registry.npmjs.org/npm/latest
998# CDK_VERSION registry.npmjs.org/aws-cdk/latest
999# KUBECTL_VERSION https://dl.k8s.io/release/stable-<minor>.txt
1000# (minor from cdk.json::kubernetes_version)
1001# AWSCLI_VERSION github://aws/aws-cli/tags (v2.x.y semver, no GitHub Releases)
1002# DOCKER_VERSION github://moby/moby/releases/latest (``docker-v<ver>``)
1003# BUILDX_VERSION github://docker/buildx/releases/latest (v<ver>)
1004# UV_VERSION github://astral-sh/uv/releases/latest (bare semver)
1005#
1006# All endpoints are public — no AWS credentials needed.
1007# ---------------------------------------------------------------------------
100826echo ""
100926echo "=== Checking Dockerfile.dev ARG pins ==="
101026
101152DOCKERFILE_RESULTS="$(mktemp)"
101226DOCKERFILE_PIN_FILE="Dockerfile.dev"
1013
1014check_dockerfile_pin() {
1015192 local name="$1" current="$2" latest=""
1016648 case "$name" in
1017 NODE_VERSION)
1018 # Two drift signals folded into one compare. First pick the
1019 # highest major with an active LTS window from the release
1020 # schedule (lts <= today AND (end missing or end > today)),
1021 # then resolve that major's newest release from the official
1022 # dist index — the same origin the Dockerfile downloads from.
1023 # A new LTS line and a new patch on the current line both
1024 # surface as ``newer``.
102524 local lts_major
102673 lts_major="$(curl -fsSL --max-time 15 \
1027 "https://raw.githubusercontent.com/nodejs/Release/main/schedule.json" 2>/dev/null \
1028 | python3 -c '
1029import sys, json, datetime
1030try:
1031 data = json.load(sys.stdin)
1032except Exception:
1033 sys.exit(0)
1034today = datetime.date.today().isoformat()
1035candidates = []
1036for k, v in data.items():
1037 if not k.startswith("v") or "lts" not in v:
1038 continue
1039 if v["lts"] > today:
1040 continue
1041 if v.get("end", "9999-12-31") <= today:
1042 continue
1043 try:
1044 candidates.append(int(k[1:]))
1045 except ValueError:
1046 continue
1047if candidates:
1048 print(max(candidates))
1049' 2>/dev/null)" || true
105024 if [ -z "$lts_major" ]; then
10511 mark_scan_incomplete "Node.js release-schedule lookup failed or returned no active LTS major."
10521 return
1053 fi
1054 # index.json is newest-first per line, so the first entry whose
1055 # version sits on the LTS major is that major's latest release.
105669 latest="$(curl -fsSL --max-time 15 \
1057 "https://nodejs.org/dist/index.json" 2>/dev/null \
1058 | jq -r --arg prefix "v${lts_major}." \
1059 '[.[].version | select(startswith($prefix))][0] // empty' 2>/dev/null)" || true
1060 ;;
1061 CDK_VERSION)
106273 latest="$(curl -fsSL --max-time 15 \
1063 "https://registry.npmjs.org/aws-cdk/latest" 2>/dev/null \
1064 | jq -r '.version // empty' 2>/dev/null)" || true
1065 ;;
1066 NPM_VERSION)
1067 # ``npm`` is part of the dev container's pinned tooling — the
1068 # npm bundled inside the Node dist tarball is fixed per Node
1069 # release but lags npm's own line, so the Dockerfile installs a
1070 # specific ``npm@X.Y.Z`` to keep rebuilds reproducible (same
1071 # rationale as CDK_VERSION above). The canonical "latest" is the
1072 # ``latest`` dist-tag on npmjs.org, same source CDK uses.
107373 latest="$(curl -fsSL --max-time 15 \
1074 "https://registry.npmjs.org/npm/latest" 2>/dev/null \
1075 | jq -r '.version // empty' 2>/dev/null)" || true
1076 ;;
1077 KUBECTL_VERSION)
1078 # Match the minor line already committed to cdk.json so the pin
1079 # and the EKS cluster stay within the ±1 minor skew policy.
108024 local k8s_minor
108148 k8s_minor="$(extract_k8s_version "cdk.json")"
108273 latest="$(curl -fsSL --max-time 15 \
1083 "https://dl.k8s.io/release/stable-${k8s_minor}.txt" 2>/dev/null | tr -d '[:space:]')" || true
1084 ;;
1085 AWSCLI_VERSION)
1086 # aws/aws-cli doesn't publish GitHub Releases for v2; tags are the
1087 # canonical source. First page (per_page=20) is newest-first;
1088 # filter to 2.x.y semver and take the top match.
108973 latest="$(curl -fsSL --max-time 15 \
1090 "https://api.github.com/repos/aws/aws-cli/tags?per_page=20" 2>/dev/null \
1091 | jq -r '[.[].name | select(test("^2\\.[0-9]+\\.[0-9]+$"))][0] // empty' 2>/dev/null)" || true
1092 ;;
1093 DOCKER_VERSION)
1094 # moby/moby tags releases as ``docker-v<semver>``; strip the
1095 # prefix so compare_semver can handle the value.
109697 latest="$(curl -fsSL --max-time 15 \
1097 "https://api.github.com/repos/moby/moby/releases/latest" 2>/dev/null \
1098 | jq -r '.tag_name // empty' 2>/dev/null \
1099 | sed -E 's/^(docker-)?v//')" || true
1100 ;;
1101 BUILDX_VERSION)
1102 # docker/buildx publishes GitHub Releases tagged v<semver>
1103 # (e.g. v0.35.0). The release tag is the canonical source;
1104 # compare_semver strips the leading v on both sides. The
1105 # Dockerfile installs the plugin binary buildx-<tag>.linux-<arch>.
110673 latest="$(curl -fsSL --max-time 15 \
1107 "https://api.github.com/repos/docker/buildx/releases/latest" 2>/dev/null \
1108 | jq -r '.tag_name // empty' 2>/dev/null)" || true
1109 ;;
1110 UV_VERSION)
1111 # astral-sh/uv publishes GitHub Releases tagged with a bare semver
1112 # (e.g. 0.12.1). The dev container ships uv/uvx for the `gco
1113 # autopilot` companion MCP servers; bump the two SHA256 ARGs in
1114 # lockstep from the per-artifact *.sha256 release files.
111573 latest="$(curl -fsSL --max-time 15 \
1116 "https://api.github.com/repos/astral-sh/uv/releases/latest" 2>/dev/null \
1117 | jq -r '.tag_name // empty' 2>/dev/null)" || true
1118 ;;
1119 esac
1120 # extract_dockerfile_pins only emits the names above. Should its allowlist
1121 # ever grow without a matching arm here, ``latest`` stays empty and the
1122 # pin is reported as an incomplete lookup below, rather than skipped in
1123 # silence.
1124
1125191 if [ -z "$latest" ]; then
11267 mark_scan_incomplete "Upstream version lookup failed for Dockerfile.dev pin ${name}."
11277 return
1128 fi
1129
1130 # Every pin is a semver, NODE_VERSION included. compare_semver strips
1131 # a leading ``v`` on both sides, matching the kubectl and buildx pins
1132 # that keep the prefix.
1133184 local relation
1134368 relation="$(compare_semver "$current" "$latest")"
1135
1136184 if [ "$relation" = "newer" ]; then
113732 echo " - ${name}: ${current} -> ${latest}"
113832 echo "${name}|${current}|${latest}" >> "$DOCKERFILE_RESULTS"
1139 fi
1140}
1141
114226if [ -f "$DOCKERFILE_PIN_FILE" ]; then
114350 DOCKERFILE_PINS="$(extract_dockerfile_pins "$DOCKERFILE_PIN_FILE")"
114425 if [ -z "$DOCKERFILE_PINS" ]; then
11451 mark_scan_incomplete "Could not parse tooling pins from ${DOCKERFILE_PIN_FILE}."
1146 fi
1147436 while IFS='|' read -r pin_name pin_value; do
1148194 [ -z "$pin_name" ] && continue
1149192 check_dockerfile_pin "$pin_name" "$pin_value"
1150 done <<< "$DOCKERFILE_PINS"
1151else
11521 mark_scan_incomplete "$DOCKERFILE_PIN_FILE is missing."
1153fi
1154
115578DOCKERFILE_COUNT="$(wc -l < "$DOCKERFILE_RESULTS" 2>/dev/null | tr -d ' ')"
115626[ -z "$DOCKERFILE_COUNT" ] && DOCKERFILE_COUNT=0
1157
1158# ---------------------------------------------------------------------------
1159# GCO Autopilot pins (agent CLI releases + companion MCP servers)
1160#
1161# ``gco autopilot`` (cli/autopilot.py) carries three dependency surfaces that
1162# live in Python constants, invisible to Dependabot and to every sweep above:
1163#
1164# CLAUDE_CODE_VERSION exact @anthropic-ai/claude-code release installed
1165# by the default engine on first use.
1166# CODEX_VERSION exact @openai/codex release installed by the Codex
1167# engine on first use.
1168# COMPANION_MCP_SERVERS the npx/uvx-launched companion MCP servers wired
1169# into every autopilot session. Nothing pins them
1170# (they resolve at launch), so the risk isn't
1171# staleness — it's disappearance: a package that is
1172# unpublished, deprecated, or yanked breaks every
1173# new session. Each is resolved on its registry and
1174# reported when unhealthy. This is exactly how
1175# mcp-server-fetch and mcp-server-calculator broke
1176# before being pruned in 2026-08.
1177#
1178# Remediation for companion findings: replace or drop the server in
1179# cli/autopilot.py *and* the "Recommended Companion MCP Servers" tables in
1180# gco_mcp/README.md — tests/test_cli_autopilot.py fails the PR until the two
1181# agree. All endpoints are public — no AWS credentials needed.
1182# ---------------------------------------------------------------------------
118326echo ""
118426echo "=== Checking GCO Autopilot pins ==="
118526
118652AUTOPILOT_RESULTS="$(mktemp)"
118726AUTOPILOT_SKIP_REASON=""
118826AUTOPILOT_SOURCE="cli/autopilot.py"
1189
119026if [ ! -f "$AUTOPILOT_SOURCE" ]; then
11912 AUTOPILOT_SKIP_REASON="${AUTOPILOT_SOURCE} not found."
11922 echo " $AUTOPILOT_SKIP_REASON"
1193else
1194 check_autopilot_npm_pin() {
119548 local constant_name="$1"
119648 local package_name="$2"
119748 local extractor="$3"
119848 local pin="" package_status="" latest="" verdict=""
119948 local package_url="https://www.npmjs.com/package/${package_name}"
1200
120196 pin="$("$extractor" "$AUTOPILOT_SOURCE")"
120248 if [ -z "$pin" ]; then
12032 AUTOPILOT_SKIP_REASON="${constant_name} not found in ${AUTOPILOT_SOURCE}."
12042 echo " $AUTOPILOT_SKIP_REASON"
12052 return
1206 fi
1207
120892 package_status="$(get_registry_package_status npm "$package_name")"
120946 if [ -z "$package_status" ]; then
12102 AUTOPILOT_SKIP_REASON="npm lookup for ${package_name} failed (network)."
12112 echo " $AUTOPILOT_SKIP_REASON"
12122 return
1213 fi
1214
121544 latest="${package_status#*|}"
121644 case "$package_status" in
1217 ok\|*)
121842 if [ -n "$latest" ] \
121984 && [ "$(compare_semver "$pin" "$latest")" = "newer" ]; then
12208 echo " - ${constant_name}: ${pin} -> ${latest}"
12218 echo "${package_name} (${constant_name})|${pin}|${latest}|${package_url}" >> "$AUTOPILOT_RESULTS"
1222 fi
1223 ;;
1224 *)
12252 verdict="${package_status%%|*}"
12262 echo " - ${package_name}: ${verdict}"
12272 echo "${package_name} (${constant_name})|${pin}|${verdict}|${package_url}" >> "$AUTOPILOT_RESULTS"
1228 ;;
1229 esac
1230 }
1231
123224 check_autopilot_npm_pin \
1233 "CLAUDE_CODE_VERSION" "@anthropic-ai/claude-code" extract_claude_code_pin
123424 check_autopilot_npm_pin \
1235 "CODEX_VERSION" "@openai/codex" extract_codex_pin
1236
1237 # Companion MCP server liveness. Missing/deprecated/yanked is drift; a
1238 # network failure marks the scan incomplete rather than inventing findings.
1239180 while IFS='|' read -r companion_name companion_registry companion_package; do
124066 [ -z "$companion_name" ] && continue
1241132 companion_status="$(get_registry_package_status "$companion_registry" "$companion_package")"
124266 if [ -z "$companion_status" ]; then
12434 if [ -z "$AUTOPILOT_SKIP_REASON" ]; then
12441 AUTOPILOT_SKIP_REASON="Registry lookup failed for ${companion_package} (${companion_registry}); companion liveness incomplete."
12451 echo " $AUTOPILOT_SKIP_REASON"
1246 fi
12474 continue
1248 fi
124962 case "$companion_status" in
1250 ok\|*)
1251 ;;
1252 *)
12534 companion_verdict="${companion_status%%|*}"
12544 companion_detail="${companion_status#*|}"
12556 [ -n "$companion_detail" ] && companion_verdict="${companion_verdict}: ${companion_detail}"
12564 if [ "$companion_registry" = "npm" ]; then
12572 companion_url="https://www.npmjs.com/package/${companion_package}"
1258 else
12592 companion_url="https://pypi.org/project/${companion_package}/"
1260 fi
12614 echo " - companion ${companion_name}: ${companion_verdict}"
12624 echo "companion ${companion_name} (${companion_registry}: ${companion_package})|launch-time (unpinned)|${companion_verdict}|${companion_url}" >> "$AUTOPILOT_RESULTS"
1263 ;;
1264 esac
1265 done < <(extract_companion_mcp_packages "$AUTOPILOT_SOURCE")
126624fi
1267
126878AUTOPILOT_COUNT="$(wc -l < "$AUTOPILOT_RESULTS" 2>/dev/null | tr -d ' ')"
126926[ -z "$AUTOPILOT_COUNT" ] && AUTOPILOT_COUNT=0
1270
1271# ---------------------------------------------------------------------------
1272# Pre-commit hook revisions
1273#
1274# Compares the ``rev:`` pinned for each ``repo:`` block in
1275# ``.pre-commit-config.yaml`` against the latest semver-shaped tag
1276# published by the upstream Git host. This catches drift Dependabot
1277# can't see — pre-commit pins live in YAML, not in the package
1278# ecosystems Dependabot monitors — and matters in practice because
1279# stale hook pins quietly miss new lint rules and bug fixes.
1280#
1281# Each hook's repo URL is resolved to a tag list via the GitHub API
1282# (the only host we use today). Full SHA-1/SHA-256 object ids are accepted as
1283# immutable exemptions. Other unsupported refs (branches, floating labels,
1284# prereleases) mark the scan incomplete rather than silently disappearing.
1285# Calls are unauthenticated; we make one request per hook, which is well below
1286# the 60 req/h public limit.
1287# ---------------------------------------------------------------------------
128826echo ""
128926echo "=== Checking pre-commit hook revisions ==="
129026
129152PRECOMMIT_RESULTS="$(mktemp)"
129226PRECOMMIT_CONFIG=".pre-commit-config.yaml"
1293
129426if [ -f "$PRECOMMIT_CONFIG" ]; then
129550 PRECOMMIT_HOOKS="$(extract_precommit_hooks "$PRECOMMIT_CONFIG")"
129625 if [ -z "$PRECOMMIT_HOOKS" ]; then
12971 mark_scan_incomplete "Could not parse hook pins from ${PRECOMMIT_CONFIG}."
1298 fi
1299162 while IFS='|' read -r repo current_rev; do
130057 [ -z "$repo" ] && continue
130155 [ -z "$current_rev" ] && continue
1302 # Complete Git object ids are immutable and intentionally have no release
1303 # drift lookup. Every other non-semver ref may move or cannot be compared
1304 # safely, so it makes this scan incomplete instead of receiving a blanket
1305 # "SHA" exemption.
130655 if is_full_git_commit_sha "$current_rev"; then
13071 continue
1308 fi
130954 if ! [[ "$current_rev" =~ ^v?[0-9]+\.[0-9]+(\.[0-9]+)?$ ]]; then
13101 mark_scan_incomplete "Unsupported mutable or non-semver pre-commit rev '${current_rev}' for ${repo}."
13111 continue
1312 fi
1313
1314106 latest_rev="$(get_latest_precommit_hook_release "$repo")"
131553 if [ -z "$latest_rev" ]; then
13163 mark_scan_incomplete "Pre-commit tag lookup failed for ${repo}."
13173 continue
1318 fi
1319
1320 # Strip ``v`` so compare_semver ranks ``v0.22.1`` vs ``v0.22.2``
1321 # (and the rare unprefixed ``1.38.0`` from yamllint historically)
1322 # consistently. We keep the original ``current_rev`` / ``latest_rev``
1323 # strings in the report so the operator copy-pastes the exact
1324 # value pre-commit expects.
132550 if [ "$current_rev" != "$latest_rev" ] \
132624 && [ "$(compare_semver "$current_rev" "$latest_rev")" = "newer" ]; then
132712 echo " - ${repo}: ${current_rev} -> ${latest_rev}"
132812 echo "${repo}|${current_rev}|${latest_rev}" >> "$PRECOMMIT_RESULTS"
1329 fi
1330 done <<< "$PRECOMMIT_HOOKS"
1331else
13321 mark_scan_incomplete "$PRECOMMIT_CONFIG is missing."
1333fi
1334
133578PRECOMMIT_COUNT="$(wc -l < "$PRECOMMIT_RESULTS" 2>/dev/null | tr -d ' ')"
133626[ -z "$PRECOMMIT_COUNT" ] && PRECOMMIT_COUNT=0
1337
1338# ---------------------------------------------------------------------------
1339# CDK enum constants
1340#
1341# Compares the CDK-enum-name constants pinned in
1342# ``gco/stacks/constants.py`` against the highest enum members exposed
1343# by the installed ``aws-cdk-lib``. This catches the case where
1344# aws-cdk-lib already supports a newer enum (because we bumped the
1345# library, or simply because the latest published release added one)
1346# but ``constants.py`` still pins an older one.
1347#
1348# Two enums are tracked today:
1349#
1350# - ``LAMBDA_PYTHON_RUNTIME`` → ``aws_cdk.aws_lambda.Runtime.PYTHON_X_Y``
1351# - ``LAMBDA_NODEJS_RUNTIME`` → ``aws_cdk.aws_lambda.Runtime.NODEJS_<major>_X``
1352#
1353# The Aurora engine deliberately is NOT an enum: constants.py pins a plain
1354# version string applied through ``AuroraPostgresEngineVersion.of()``, and
1355# the "Aurora PostgreSQL engine" section validates it against the live RDS
1356# API — the authoritative source — instead of the CDK library's catalog.
1357#
1358# The deps-scan workflow installs the latest ``aws-cdk-lib`` for this
1359# section; locally the helper just reflects whatever's already on the
1360# active interpreter. If aws-cdk-lib isn't importable we skip with a
1361# one-line note (mirrors the AWS-creds skip pattern used elsewhere).
1362# ---------------------------------------------------------------------------
136326echo ""
136426echo "=== Checking CDK enum constants ==="
136526
136652CDK_ENUM_RESULTS="$(mktemp)"
136726CDK_ENUM_SKIP_REASON=""
1368
136926if ! python3 -c "import aws_cdk" 2>/dev/null; then
13702 CDK_ENUM_SKIP_REASON="aws-cdk-lib not importable. Install with 'pip install aws-cdk-lib' to enable."
13712 echo " $CDK_ENUM_SKIP_REASON"
1372else
1373 # Lambda Python runtime enum
137448 LAMBDA_RT_CURRENT="$(extract_constant_value LAMBDA_PYTHON_RUNTIME)"
137548 LAMBDA_RT_LATEST="$(get_latest_lambda_python_runtime)"
137647 if [ -z "$LAMBDA_RT_CURRENT" ] || [ -z "$LAMBDA_RT_LATEST" ]; then
13772 mark_scan_incomplete "Could not parse the current or latest Lambda Python runtime enum."
137822 elif [ "$LAMBDA_RT_CURRENT" != "$LAMBDA_RT_LATEST" ]; then
1379 # Convert PYTHON_3_14 → 3.14 so compare_semver can rank them.
13809 cur_v="$(echo "$LAMBDA_RT_CURRENT" | sed -E 's/^PYTHON_([0-9]+)_([0-9]+)$/\1.\2/')"
13819 lat_v="$(echo "$LAMBDA_RT_LATEST" | sed -E 's/^PYTHON_([0-9]+)_([0-9]+)$/\1.\2/')"
13826 if [ "$(compare_semver "$cur_v" "$lat_v")" = "newer" ]; then
13833 echo " - LAMBDA_PYTHON_RUNTIME: ${LAMBDA_RT_CURRENT} -> ${LAMBDA_RT_LATEST}"
13843 echo "LAMBDA_PYTHON_RUNTIME|aws_lambda.Runtime|${LAMBDA_RT_CURRENT}|${LAMBDA_RT_LATEST}" >> "$CDK_ENUM_RESULTS"
1385 fi
1386 fi
1387
1388 # Lambda Node.js runtime enum
138948 LAMBDA_NODE_RT_CURRENT="$(extract_constant_value LAMBDA_NODEJS_RUNTIME)"
139048 LAMBDA_NODE_RT_LATEST="$(get_latest_lambda_nodejs_runtime)"
139147 if [ -z "$LAMBDA_NODE_RT_CURRENT" ] || [ -z "$LAMBDA_NODE_RT_LATEST" ]; then
13922 mark_scan_incomplete "Could not parse the current or latest Lambda Node.js runtime enum."
139322 elif [ "$LAMBDA_NODE_RT_CURRENT" != "$LAMBDA_NODE_RT_LATEST" ]; then
13943 cur_major="${LAMBDA_NODE_RT_CURRENT#NODEJS_}"
13953 cur_major="${cur_major%_X}"
13963 lat_major="${LAMBDA_NODE_RT_LATEST#NODEJS_}"
13973 lat_major="${lat_major%_X}"
13986 if [[ "$cur_major" =~ ^[0-9]+$ ]] && [[ "$lat_major" =~ ^[0-9]+$ ]] \
13993 && [ "$lat_major" -gt "$cur_major" ]; then
14003 echo " - LAMBDA_NODEJS_RUNTIME: ${LAMBDA_NODE_RT_CURRENT} -> ${LAMBDA_NODE_RT_LATEST}"
14013 echo "LAMBDA_NODEJS_RUNTIME|aws_lambda.Runtime|${LAMBDA_NODE_RT_CURRENT}|${LAMBDA_NODE_RT_LATEST}" >> "$CDK_ENUM_RESULTS"
1402 fi
1403 fi
1404
1405fi
1406
140778CDK_ENUM_COUNT="$(wc -l < "$CDK_ENUM_RESULTS" 2>/dev/null | tr -d ' ')"
140826[ -z "$CDK_ENUM_COUNT" ] && CDK_ENUM_COUNT=0
1409
1410# ---------------------------------------------------------------------------
1411# Python release
1412#
1413# Compares the Lambda Python runtime constant (which encodes the major
1414# Python version we standardise on across every Lambda in the project)
1415# against the latest stable Python release on endoflife.date.
1416#
1417# This is informational drift — Lambda may not ship support for a brand-
1418# new Python release for several months — but the signal is useful so
1419# the operator knows when to start planning a runtime bump. It also
1420# complements the CDK-enum check above: that check answers "what does
1421# aws-cdk-lib expose?", this one answers "what has python.org actually
1422# shipped?".
1423# ---------------------------------------------------------------------------
142426echo ""
142526echo "=== Checking Python release ==="
142626
142752PYTHON_RELEASE_RESULTS="$(mktemp)"
142826PYTHON_RELEASE_SKIP_REASON=""
1429
1430# Re-read in case the CDK section was skipped and never set the var.
143129LAMBDA_RT_CURRENT="${LAMBDA_RT_CURRENT:-$(extract_constant_value LAMBDA_PYTHON_RUNTIME)}"
143252LATEST_PYTHON="$(get_latest_python_release)"
1433
143426if [ -z "$LATEST_PYTHON" ]; then
14352 PYTHON_RELEASE_SKIP_REASON="endoflife.date query failed (network or schema change)."
14362 echo " $PYTHON_RELEASE_SKIP_REASON"
143724elif [ -z "$LAMBDA_RT_CURRENT" ]; then
14381 PYTHON_RELEASE_SKIP_REASON="Could not parse LAMBDA_PYTHON_RUNTIME for the Python release comparison."
14391 echo " $PYTHON_RELEASE_SKIP_REASON"
1440else
144169 cur_v="$(echo "$LAMBDA_RT_CURRENT" | sed -E 's/^PYTHON_([0-9]+)_([0-9]+)$/\1.\2/')"
144223 if [ "$cur_v" != "$LATEST_PYTHON" ] \
144310 && [ "$(compare_semver "$cur_v" "$LATEST_PYTHON")" = "newer" ]; then
14444 echo " - python (LAMBDA_PYTHON_RUNTIME): ${cur_v} -> ${LATEST_PYTHON}"
14454 echo "python|${cur_v}|${LATEST_PYTHON}" >> "$PYTHON_RELEASE_RESULTS"
1446 fi
1447fi
1448
144978PYTHON_RELEASE_COUNT="$(wc -l < "$PYTHON_RELEASE_RESULTS" 2>/dev/null | tr -d ' ')"
145026[ -z "$PYTHON_RELEASE_COUNT" ] && PYTHON_RELEASE_COUNT=0
1451
1452# ---------------------------------------------------------------------------
1453# Ruby release
1454#
1455# Ruby is a CI-only toolchain: bashcov (see the Gemfile) measures which lines
1456# of each shell script the BATS suite executes, and the unit:bats:shell job
1457# gets its interpreter from .ruby-version via ruby/setup-ruby. Dependabot
1458# watches the *gems* in Gemfile.lock but has nothing to say about the
1459# interpreter series, so — exactly like .python-version — the pin is compared
1460# against the newest supported series here. Informational, not a failure: a new
1461# Ruby series is a deliberate move, not a security patch.
1462# ---------------------------------------------------------------------------
146326echo ""
146426echo "=== Checking Ruby release ==="
146526
146652RUBY_RELEASE_RESULTS="$(mktemp)"
146726RUBY_RELEASE_SKIP_REASON=""
1468
146952RUBY_PIN_CURRENT="$(read_ruby_version_pin .ruby-version)"
147052LATEST_RUBY="$(get_latest_ruby_release)"
1471
147226if [ -z "$LATEST_RUBY" ]; then
14732 RUBY_RELEASE_SKIP_REASON="endoflife.date query failed (network or schema change)."
14742 echo " $RUBY_RELEASE_SKIP_REASON"
147524elif [ -z "$RUBY_PIN_CURRENT" ]; then
14761 RUBY_RELEASE_SKIP_REASON="Could not parse .ruby-version for the Ruby release comparison."
14771 echo " $RUBY_RELEASE_SKIP_REASON"
1478else
147923 if [ "$RUBY_PIN_CURRENT" != "$LATEST_RUBY" ] \
148044 && [ "$(compare_semver "$RUBY_PIN_CURRENT" "$LATEST_RUBY")" = "newer" ]; then
14814 echo " - ruby (.ruby-version): ${RUBY_PIN_CURRENT} -> ${LATEST_RUBY}"
14824 echo "ruby|${RUBY_PIN_CURRENT}|${LATEST_RUBY}" >> "$RUBY_RELEASE_RESULTS"
1483 else
148419 echo " .ruby-version pins ${RUBY_PIN_CURRENT}; newest supported series is ${LATEST_RUBY}."
1485 fi
1486fi
1487
148878RUBY_RELEASE_COUNT="$(wc -l < "$RUBY_RELEASE_RESULTS" 2>/dev/null | tr -d ' ')"
148926[ -z "$RUBY_RELEASE_COUNT" ] && RUBY_RELEASE_COUNT=0
1490
1491# ---------------------------------------------------------------------------
1492# Runner images
1493#
1494# `runs-on: ubuntu-latest` is not a version pin, so Dependabot has nothing to
1495# bump and the platform underneath every job changes only when GitHub moves the
1496# label. Two opposite mistakes are possible: staying on an explicitly pinned
1497# image long after a newer one is generally available (and eventually on one
1498# upstream has marked deprecated, which is a removal notice), or chasing an
1499# image that is still in preview.
1500#
1501# check_runner_images.py reports those separately: `--format rows` is drift to
1502# act on (newer GA, or deprecated), `--format notes` is context (a newer image
1503# exists but is preview, so the current pin is deliberate). It exits 2 without
1504# printing anything when the upstream catalog cannot be read, which is treated
1505# as a skip here rather than as an all-clear.
1506# ---------------------------------------------------------------------------
150726echo ""
150826echo "=== Checking runner images ==="
150926
151052RUNNER_IMAGE_RESULTS="$(mktemp)"
151152RUNNER_IMAGE_NOTES="$(mktemp)"
151226RUNNER_IMAGE_SKIP_REASON=""
1513
151426if python3 .github/scripts/check_runner_images.py --format rows > "$RUNNER_IMAGE_RESULTS" 2>/dev/null; then
151524 python3 .github/scripts/check_runner_images.py --format notes > "$RUNNER_IMAGE_NOTES" 2>/dev/null || true
151650 while IFS='|' read -r label current recommended; do
15171 [ -z "$label" ] && continue
15181 echo " - runner ${label}: ${current} -> ${recommended}"
1519 done < "$RUNNER_IMAGE_RESULTS"
152096 while IFS='|' read -r label current recommended; do
152124 [ -z "$label" ] && continue
152224 echo " ${label} pins ${current}; ${recommended} exists but is still in preview."
1523 done < "$RUNNER_IMAGE_NOTES"
152424 if [ ! -s "$RUNNER_IMAGE_RESULTS" ]; then
152523 echo " every runner label in use is the newest generally-available image."
1526 fi
1527else
15282 RUNNER_IMAGE_SKIP_REASON="Could not read the actions/runner-images catalog (network or upstream README change)."
15292 echo " $RUNNER_IMAGE_SKIP_REASON"
15302 : > "$RUNNER_IMAGE_RESULTS"
1531fi
1532
153378RUNNER_IMAGE_COUNT="$(wc -l < "$RUNNER_IMAGE_RESULTS" 2>/dev/null | tr -d ' ')"
153426[ -z "$RUNNER_IMAGE_COUNT" ] && RUNNER_IMAGE_COUNT=0
1535
1536# ---------------------------------------------------------------------------
1537# CI tooling pins (public endpoints — no AWS creds)
1538#
1539# The workflows install their own pinned tooling — Trivy (cve-scan.yml /
1540# security.yml), actionlint (lint.yml), Helm + kubectl (deps-scan.yml), and
1541# kubeconform, Calico, and Metrics Server (integration-tests.yml) — from plain
1542# ``*_VERSION`` env strings. The integration workflow also pins kind + its node
1543# image on the ``helm/kind-action`` step. None are ``uses:`` refs or Dockerfile
1544# ``FROM`` lines, so Dependabot never sees them. Compare each against upstream.
1545# ---------------------------------------------------------------------------
154626echo ""
154726echo "=== Checking CI tooling pins ==="
154826
154952CI_TOOLING_RESULTS="$(mktemp)"
1550# The kind-action lockstep check below files its finding under Version
1551# Consistency, so that section's results file has to exist already; it is
1552# created here, once, and the consistency section appends to it.
155352CONSISTENCY_RESULTS="$(mktemp)"
1554
1555# check_github_tool <display-name> <current-pin> <owner/repo> <ref-url>
1556# Records drift when the pinned semver is behind the latest GitHub Release.
1557check_github_tool() {
1558182 local name="$1" current="$2" repo="$3" url="$4" latest=""
1559182 if [ -z "$current" ]; then
15607 mark_scan_incomplete "Could not parse the committed version pin for ${name}."
15617 return 0
1562 fi
1563350 latest="$(get_latest_github_release_tag "$repo")"
1564175 if [ -z "$latest" ]; then
15657 mark_scan_incomplete "GitHub release lookup failed for ${name} (${repo})."
15667 return 0
1567 fi
1568168 if [ "$current" != "$latest" ] \
156970 && [ "$(compare_semver "$current" "$latest")" = "newer" ]; then
157032 echo " - ${name}: ${current} -> ${latest}"
157132 echo "${name}|${current}|${latest}|${url}" >> "$CI_TOOLING_RESULTS"
1572 fi
1573}
1574
1575# Trivy (aquasecurity/trivy) — the version-input default of the
1576# install-trivy composite action, the single pin every caller inherits.
157778TRIVY_PIN="$(extract_install_trivy_pin .github/actions/install-trivy/action.yml | head -1)"
157826check_github_tool "Trivy (install-trivy action default)" "$TRIVY_PIN" "aquasecurity/trivy" \
1579 "https://github.com/aquasecurity/trivy/releases"
1580
1581# actionlint (rhysd/actionlint) — lint.yml downloads this release archive.
158278ACTIONLINT_PIN="$(extract_workflow_env_pin ACTIONLINT_VERSION | head -1)"
158326check_github_tool "actionlint (ACTIONLINT_VERSION)" "$ACTIONLINT_PIN" "rhysd/actionlint" \
1584 "https://github.com/rhysd/actionlint/releases"
1585
1586# Helm (helm/helm) — the authenticated RUN-line pin in
1587# lambda/helm-installer/Dockerfile, the single source every CI job derives
1588# its HELM_VERSION/HELM_SHA256 from at runtime.
1589104HELM_PIN="$(extract_helm_installer_pins lambda/helm-installer/Dockerfile | awk -F'|' '$1=="HELM_VERSION"{print $2}' | head -1)"
159026check_github_tool "Helm (helm-installer Dockerfile)" "$HELM_PIN" "helm/helm" \
1591 "https://github.com/helm/helm/releases"
1592
1593# kubeconform (yannh/kubeconform) — KUBECONFORM_VERSION the
1594# integration:k8s:manifest-schema job installs to schema-validate the K8s
1595# manifests. Plain env pin Dependabot doesn't watch; a stale kubeconform
1596# silently validates against outdated Kubernetes schemas.
159778KUBECONFORM_PIN="$(extract_workflow_env_pin KUBECONFORM_VERSION | head -1)"
159826check_github_tool "kubeconform (KUBECONFORM_VERSION)" "$KUBECONFORM_PIN" "yannh/kubeconform" \
1599 "https://github.com/yannh/kubeconform/releases"
1600
1601# Metrics Server (kubernetes-sigs/metrics-server) — kind installs this pin so
1602# the inference proxy HPA must reach ScalingActive, mirroring the EKS managed
1603# add-on contract used in production.
160478METRICS_SERVER_PIN="$(extract_workflow_env_pin METRICS_SERVER_VERSION | head -1)"
160526check_github_tool \
1606 "Metrics Server (METRICS_SERVER_VERSION)" \
1607 "$METRICS_SERVER_PIN" \
1608 "kubernetes-sigs/metrics-server" \
1609 "https://github.com/kubernetes-sigs/metrics-server/releases"
161026
1611# Calico (projectcalico/calico) — kind installs the authenticated release
1612# manifest so NetworkPolicy behavior is exercised by the E2E job.
161378CALICO_PIN="$(extract_workflow_env_pin CALICO_VERSION | head -1)"
161426check_github_tool "Calico (CALICO_VERSION)" "$CALICO_PIN" "projectcalico/calico" \
1615 "https://github.com/projectcalico/calico/releases"
1616
1617# kind (kubernetes-sigs/kind) — the kind binary on the kind-action step.
1618# Both kind-action steps (cluster-e2e and examples-smoke) must pin the same
1619# kind binary + node image; extract_kind_pins de-duplicates identical pins,
1620# so >1 value per key means the two jobs drifted apart.
162152for kind_key in kind kind-node; do
1622156 kind_vals="$(extract_kind_pins .github/workflows/integration-tests.yml \
1623 | awk -F'|' -v k="$kind_key" '$1==k{print $2}')"
1624208 kind_distinct="$(printf '%s\n' "$kind_vals" | sed '/^$/d' | grep -c .)"
162552 if [ "$kind_distinct" -gt 1 ]; then
16268 kind_list="$(printf '%s\n' "$kind_vals" | sed '/^$/d' | paste -sd',' -)"
16272 echo " - ${kind_key} pins disagree across kind-action steps: ${kind_list}"
16282 echo "${kind_key} (across kind-action steps)|${kind_list}" >> "$CONSISTENCY_RESULTS"
1629 fi
1630done
1631104KIND_PIN="$(extract_kind_pins .github/workflows/integration-tests.yml | awk -F'|' '$1=="kind"{print $2}' | head -1)"
163226check_github_tool "kind" "$KIND_PIN" "kubernetes-sigs/kind" \
1633 "https://github.com/kubernetes-sigs/kind/releases"
1634
1635# kubectl — the authenticated RUN-line pin in lambda/helm-installer/
1636# Dockerfile (the single source the CI jobs derive from); compared against
1637# the stable release for its own minor line (dl.k8s.io), the same source
1638# the Dockerfile.dev kubectl pin uses.
1639104KUBECTL_WF_PIN="$(extract_helm_installer_pins lambda/helm-installer/Dockerfile | awk -F'|' '$1=="KUBECTL_VERSION"{print $2}' | head -1)"
164026if [ -n "$KUBECTL_WF_PIN" ]; then
164175 kubectl_minor="$(echo "${KUBECTL_WF_PIN#v}" | cut -d. -f1-2)"
164275 if ! kubectl_latest="$(curl -fsSL --max-time 15 \
1643 "https://dl.k8s.io/release/stable-${kubectl_minor}.txt" 2>/dev/null | tr -d '[:space:]')" \
164424 || [ -z "$kubectl_latest" ]; then
16451 mark_scan_incomplete "kubectl stable-version lookup failed for minor ${kubectl_minor}."
164624 elif [ "$KUBECTL_WF_PIN" != "$kubectl_latest" ] \
164710 && [ "$(compare_semver "$KUBECTL_WF_PIN" "$kubectl_latest")" = "newer" ]; then
16485 echo " - kubectl (helm-installer Dockerfile): ${KUBECTL_WF_PIN} -> ${kubectl_latest}"
16495 echo "kubectl (helm-installer Dockerfile)|${KUBECTL_WF_PIN}|${kubectl_latest}|https://kubernetes.io/releases/" >> "$CI_TOOLING_RESULTS"
1650 fi
1651else
16521 mark_scan_incomplete "Could not parse the kubectl pin from lambda/helm-installer/Dockerfile."
1653fi
1654
1655# kind node image (kindest/node) — report a newer PATCH within the pinned K8s
1656# minor only. Jumping minors is governed by the kind release, not free drift,
1657# so scoping to the same minor avoids false "upgrade" noise.
1658104KIND_NODE_PIN="$(extract_kind_pins .github/workflows/integration-tests.yml | awk -F'|' '$1=="kind-node"{print $2}' | head -1)"
165926if [ -n "$KIND_NODE_PIN" ]; then
166025 node_tag="${KIND_NODE_PIN##*:}"
166175 node_minor="$(echo "${node_tag#v}" | cut -d. -f1-2)"
1662172 if ! node_latest="$(skopeo list-tags "docker://docker.io/kindest/node" 2>/dev/null \
1663 | jq -r '.Tags[]' 2>/dev/null \
1664 | grep -E "^v?${node_minor}\.[0-9]+$" \
1665 | sort -V | tail -1)" || [ -z "$node_latest" ]; then
16663 mark_scan_incomplete "Container registry lookup failed for kindest/node minor ${node_minor}."
166722 elif [ "$node_tag" != "$node_latest" ] \
16688 && [ "$(compare_semver "$node_tag" "$node_latest")" = "newer" ]; then
16694 echo " - kind node image (kindest/node): ${node_tag} -> ${node_latest}"
16704 echo "kind node image (kindest/node)|${node_tag}|${node_latest}|https://hub.docker.com/r/kindest/node/tags" >> "$CI_TOOLING_RESULTS"
1671 fi
1672else
16731 mark_scan_incomplete "Could not parse the kind node-image pin from integration-tests.yml."
1674fi
1675
167678CI_TOOLING_COUNT="$(wc -l < "$CI_TOOLING_RESULTS" 2>/dev/null | tr -d ' ')"
167726[ -z "$CI_TOOLING_COUNT" ] && CI_TOOLING_COUNT=0
1678
1679# ---------------------------------------------------------------------------
1680# Version consistency (no network)
1681#
1682# Some versions are pinned in more than one place and must move together.
1683# The other sections answer "is this pin behind upstream?"; this one answers
1684# "do the copies of this pin agree with each other?" — a class of drift that
1685# otherwise only surfaces when a formatter/linter behaves differently in CI
1686# than it does locally.
1687#
1688# - ruff: pyproject (dev install) vs the pre-commit hook vs the prebuilt-
1689# binary ruff-action step in lint.yml.
1690# - python-version across the workflows vs the project's canonical Python
1691# (the LAMBDA_PYTHON_RUNTIME the Lambdas ship on).
1692# - Node major across LAMBDA_NODEJS_RUNTIME, .nvmrc, every package engine,
1693# and Dockerfile.dev; npm across every packageManager and Dockerfile.dev.
1694# - AWS CDK CLI across the locked root npm graph and Dockerfile.dev.
1695# - every repository-owned package.json has a lockfile, exact direct pins,
1696# and a matching npm entry in Dependabot.
1697# - the same tool env pin (HELM_VERSION/KUBECTL_VERSION/CALICO_*)
1698# resolving to different values in different workflow files.
1699# - every [build-system] requires entry in pyproject.toml is an exact
1700# ``==`` pin (the drift itself reports through the Python surface).
1701# - every per-Lambda requirements.txt pin agrees with the version the
1702# repository resolves centrally (pyproject, then the lock for
1703# transitives) — the copies that ship to production.
1704# - no digest-pinned image carries two different digests under one tag,
1705# which means an upstream re-push was only half applied.
1706# ---------------------------------------------------------------------------
170726echo ""
170826echo "=== Checking version consistency ==="
170926
171052RUFF_PINS="$(extract_ruff_pins pyproject.toml .pre-commit-config.yaml .github/workflows/lint.yml)"
171126if [ -n "$RUFF_PINS" ]; then
1712125 ruff_distinct="$(echo "$RUFF_PINS" | cut -d'|' -f2 | sort -u | grep -c .)"
171325 if [ "$ruff_distinct" -gt 1 ]; then
17144 detail="$(echo "$RUFF_PINS" | awk -F'|' '{printf "%s=%s ", $1, $2}' | sed 's/ *$//')"
17151 echo " - ruff pins disagree: $detail"
17161 echo "ruff (pyproject / pre-commit / lint action)|${detail}" >> "$CONSISTENCY_RESULTS"
1717 fi
1718fi
1719
172078CANON_PY="$(echo "${LAMBDA_RT_CURRENT:-}" | sed -E 's/^PYTHON_([0-9]+)_([0-9]+)$/\1.\2/')"
172129[ -z "$CANON_PY" ] && CANON_PY="$(extract_constant_value LAMBDA_PYTHON_RUNTIME | sed -E 's/^PYTHON_([0-9]+)_([0-9]+)$/\1.\2/')"
172278PY_PINS_UNIQUE="$(extract_python_version_pins "$WORKFLOWS_DIR" .python-version | sort -u)"
172326if [ -n "$PY_PINS_UNIQUE" ]; then
172475 py_distinct="$(echo "$PY_PINS_UNIQUE" | grep -c .)"
172575 py_list="$(echo "$PY_PINS_UNIQUE" | paste -sd',' -)"
172673 if [ "$py_distinct" -gt 1 ] || { [ -n "$CANON_PY" ] && [ "$py_list" != "$CANON_PY" ]; }; then
17271 echo " - python-version pins: ${py_list} (project runtime: ${CANON_PY:-unknown})"
17281 echo "python-version (CI vs runtime)|CI: ${py_list}; runtime: ${CANON_PY:-unknown}" >> "$CONSISTENCY_RESULTS"
1729 fi
1730fi
1731
1732158NPM_PACKAGE_SOURCES="$(
1733 list_npm_package_dirs . | while IFS= read -r package_dir; do
1734 [ -n "$package_dir" ] || continue
1735 if [ "$package_dir" = "." ]; then
1736 echo "package.json"
1737 else
1738 echo "${package_dir}/package.json"
1739 fi
1740 done
1741)"
174227
1743105NODE_PINS="$(extract_node_major_pins . gco/stacks/constants.py .nvmrc Dockerfile.dev | sort -u)"
174452EXPECTED_NODE_SOURCES="$(
1745 {
1746 printf '%s\n' gco/stacks/constants.py .nvmrc Dockerfile.dev
1747 printf '%s\n' "$NPM_PACKAGE_SOURCES"
1748 } | sed '/^$/d' | sort -u
1749)"
1750208NODE_PIN_SOURCES="$(printf '%s\n' "$NODE_PINS" | cut -d'|' -f1 | sed '/^$/d' | sort -u)"
1751104NODE_MISSING="$(comm -23 <(printf '%s\n' "$EXPECTED_NODE_SOURCES") <(printf '%s\n' "$NODE_PIN_SOURCES"))"
1752156node_distinct="$(printf '%s\n' "$NODE_PINS" | cut -d'|' -f2 | sed '/^$/d' | sort -u | grep -c .)"
175349if [ -n "$NODE_MISSING" ] || [ "$node_distinct" -gt 1 ]; then
175412 node_detail="$(printf '%s\n' "$NODE_PINS" | awk -F'|' '{printf "%s=%s ", $1, $2}' | sed 's/ *$//')"
17553 if [ -n "$NODE_MISSING" ]; then
17569 node_missing_list="$(printf '%s\n' "$NODE_MISSING" | paste -sd',' -)"
17573 node_detail="${node_detail}; missing=${node_missing_list}"
1758 fi
17593 echo " - Node.js major pins disagree or are missing: ${node_detail}"
17603 echo "Node.js major (runtime / packages / dev container)|${node_detail}" >> "$CONSISTENCY_RESULTS"
1761fi
1762
176378NPM_PINS="$(extract_npm_version_pins . Dockerfile.dev | sort -u)"
176452EXPECTED_NPM_SOURCES="$(
1765 {
1766 printf '%s\n' Dockerfile.dev
1767 printf '%s\n' "$NPM_PACKAGE_SOURCES"
1768 } | sed '/^$/d' | sort -u
1769)"
1770208NPM_PIN_SOURCES="$(printf '%s\n' "$NPM_PINS" | cut -d'|' -f1 | sed '/^$/d' | sort -u)"
1771104NPM_MISSING="$(comm -23 <(printf '%s\n' "$EXPECTED_NPM_SOURCES") <(printf '%s\n' "$NPM_PIN_SOURCES"))"
1772156npm_distinct="$(printf '%s\n' "$NPM_PINS" | cut -d'|' -f2 | sed '/^$/d' | sort -u | grep -c .)"
177350if [ -n "$NPM_MISSING" ] || [ "$npm_distinct" -gt 1 ]; then
177412 npm_detail="$(printf '%s\n' "$NPM_PINS" | awk -F'|' '{printf "%s=%s ", $1, $2}' | sed 's/ *$//')"
17753 if [ -n "$NPM_MISSING" ]; then
17766 npm_missing_list="$(printf '%s\n' "$NPM_MISSING" | paste -sd',' -)"
17772 npm_detail="${npm_detail}; missing=${npm_missing_list}"
1778 fi
17793 echo " - npm pins disagree or are missing: ${npm_detail}"
17803 echo "npm (packageManager / dev container)|${npm_detail}" >> "$CONSISTENCY_RESULTS"
1781fi
1782
178378CDK_CLI_PINS="$(extract_cdk_cli_pins . Dockerfile.dev | sort -u)"
1784208CDK_CLI_MISSING="$(comm -23 \
1785 <(printf '%s\n' Dockerfile.dev package.json | sort -u) \
1786 <(printf '%s\n' "$CDK_CLI_PINS" | cut -d'|' -f1 | sed '/^$/d' | sort -u))"
1787156cdk_cli_distinct="$(printf '%s\n' "$CDK_CLI_PINS" | cut -d'|' -f2 | sed '/^$/d' | sort -u | grep -c .)"
178849if [ -n "$CDK_CLI_MISSING" ] || [ "$cdk_cli_distinct" -gt 1 ]; then
178912 cdk_detail="$(printf '%s\n' "$CDK_CLI_PINS" | awk -F'|' '{printf "%s=%s ", $1, $2}' | sed 's/ *$//')"
17903 if [ -n "$CDK_CLI_MISSING" ]; then
17919 cdk_missing_list="$(printf '%s\n' "$CDK_CLI_MISSING" | paste -sd',' -)"
17923 cdk_detail="${cdk_detail}; missing=${cdk_missing_list}"
1793 fi
17943 echo " - AWS CDK CLI pins disagree or are missing: ${cdk_detail}"
17953 echo "AWS CDK CLI (package / dev container)|${cdk_detail}" >> "$CONSISTENCY_RESULTS"
1796fi
1797
179852NPM_MANAGEMENT_PROBLEMS="$(check_npm_package_management . .github/dependabot.yml)"
179926if [ -n "$NPM_MANAGEMENT_PROBLEMS" ]; then
18004 while IFS='|' read -r manifest problem; do
18011 [ -n "$manifest" ] || continue
18021 echo " - npm dependency management: ${manifest}: ${problem}"
18031 echo "npm dependency management|${manifest}: ${problem}" >> "$CONSISTENCY_RESULTS"
1804 done <<< "$NPM_MANAGEMENT_PROBLEMS"
1805fi
1806
1807# Every tool pinned in more than one workflow (or more than one job) must
1808# agree. CALICO_* and METRICS_SERVER_* are here because the kind jobs install
1809# them per job — two jobs on different Calico builds would enforce
1810# NetworkPolicy with two different engines, and a version/checksum pair that
1811# disagrees fails the download as what looks like a flake. The PR-time half of
1812# this contract lives in
1813# tests/test_supply_chain_integrity.py::test_repeated_workflow_pins_agree_across_jobs.
1814# (Trivy is absent from this list on purpose: its pin is the install-trivy
1815# composite action's input default, a single declaration that cannot
1816# disagree with itself.)
1817# (Helm and kubectl are absent from this list on purpose: their pins live
1818# only in lambda/helm-installer/Dockerfile — workflows derive their env
1819# copies from it at runtime, so a checked-in workflow declaration that
1820# could disagree no longer exists. The guard below still catches a stray
1821# reintroduced copy.)
1822104for consistency_var in \
1823 CALICO_VERSION CALICO_SHA256 METRICS_SERVER_VERSION METRICS_SERVER_SHA256; do
1824208 cvals="$(extract_workflow_env_pin "$consistency_var")"
1825312 cnum="$(echo "$cvals" | grep -c .)"
1826104 if [ "$cnum" -gt 1 ]; then
18273 clist="$(echo "$cvals" | paste -sd',' -)"
18281 echo " - ${consistency_var} disagrees across workflows: ${clist}"
18291 echo "${consistency_var} (across workflows)|${clist}" >> "$CONSISTENCY_RESULTS"
1830 fi
1831done
1832
1833# helm / kubectl pins hardcoded in lambda/helm-installer/Dockerfile RUN-line
1834# URLs must agree with the workflow env pins (and, for kubectl, with the
1835# Dockerfile.dev ARG). These URL literals were previously invisible to every
1836# check — the integration-tests workflow even carries a comment noting the
1837# Lambda copy "isn't caught by the consistency check". Now it is.
183852INSTALLER_PINS="$(extract_helm_installer_pins lambda/helm-installer/Dockerfile)"
183926if [ -n "$INSTALLER_PINS" ]; then
184050 for tool_var in HELM_VERSION KUBECTL_VERSION; do
1841150 installer_val="$(printf '%s\n' "$INSTALLER_PINS" | awk -F'|' -v v="$tool_var" '$1==v{print $2}')"
184250 [ -n "$installer_val" ] || continue
184350 all_vals="$installer_val"
1844 # Workflows derive their copies from the installer Dockerfile at
1845 # runtime, so a checked-in workflow declaration is itself a finding:
1846 # it would shadow the derive step's GITHUB_ENV export and drift.
1847100 wf_vals="$(extract_workflow_env_pin "$tool_var")"
184850 if [ -n "$wf_vals" ]; then
18491 echo " - ${tool_var} is declared literally in a workflow again (should derive from the installer Dockerfile): ${wf_vals}"
18501 echo "${tool_var} (literal workflow copy reintroduced)|${wf_vals}" >> "$CONSISTENCY_RESULTS"
18512 all_vals="$(printf '%s\n%s' "$all_vals" "$wf_vals")"
1852 fi
185350 if [ "$tool_var" = "KUBECTL_VERSION" ]; then
185475 dev_val="$(extract_dockerfile_pins Dockerfile.dev | awk -F'|' '$1=="KUBECTL_VERSION"{print $2}')"
185573 [ -n "$dev_val" ] && all_vals="$(printf '%s\n%s' "$all_vals" "$dev_val")"
1856 fi
1857250 tool_distinct="$(printf '%s\n' "$all_vals" | sed '/^$/d' | sort -u | grep -c .)"
185850 if [ "$tool_distinct" -gt 1 ]; then
185910 tool_list="$(printf '%s\n' "$all_vals" | sed '/^$/d' | sort -u | paste -sd',' -)"
18602 echo " - ${tool_var} disagrees between helm-installer Dockerfile, workflows, and Dockerfile.dev: ${tool_list}"
18612 echo "${tool_var} (helm-installer Dockerfile / workflows / Dockerfile.dev)|${tool_list}" >> "$CONSISTENCY_RESULTS"
1862 fi
1863 done
1864fi
1865
1866# Build-backend pins must use the same exact ``==`` shape as every other
1867# Python dependency in pyproject.toml, or the version resolved inside
1868# pip's build isolation floats with upstream releases. An empty result is
1869# itself a finding — [build-system] always exists here, so nothing coming
1870# back means the table was removed or the TOML no longer parses, and a
1871# parse break must not silently drop the check.
187227BUILD_SYSTEM_PINS="${BUILD_SYSTEM_PINS:-$(extract_build_system_pins pyproject.toml)}"
187326if [ -z "$BUILD_SYSTEM_PINS" ]; then
18741 echo " - pyproject.toml [build-system] requires is missing or unparseable"
18751 echo "build-system requires (pyproject.toml)|missing or unparseable" >> "$CONSISTENCY_RESULTS"
1876else
1877100 while IFS='|' read -r bs_name bs_version bs_raw; do
187825 [ -n "$bs_raw" ] || continue
187925 if [ -z "$bs_version" ]; then
18801 echo " - build-system requires entry is not an exact ==X.Y.Z pin: ${bs_raw}"
18811 echo "build-system requires (pyproject.toml)|'${bs_raw}' must be an exact ==X.Y.Z pin" >> "$CONSISTENCY_RESULTS"
1882 fi
1883 done <<< "$BUILD_SYSTEM_PINS"
1884fi
1885
1886# Each Lambda is packaged independently and carries its own requirements.txt,
1887# so a library like boto3 is pinned centrally *and* in up to six Lambda copies.
1888# Nothing watched the copies: a bump applied to pyproject.toml and the lock left
1889# them behind silently, and the handlers shipped a boto3 that nothing in CI ever
1890# exercised. This is the one drift surface that reaches production directly, so
1891# the report names the file and the version it must move to. The PR-time half of
1892# this contract lives in
1893# tests/test_integration.py::TestDependencyVersionConsistency::test_lambda_requirements_match_pyproject.
189452LAMBDA_PIN_PROBLEMS="$(check_lambda_requirements_pins . pyproject.toml requirements-lock.txt)"
189526if [ -n "$LAMBDA_PIN_PROBLEMS" ]; then
18968 while IFS='|' read -r lambda_req lambda_problem; do
18972 [ -n "$lambda_req" ] || continue
18982 echo " - Lambda runtime pin: ${lambda_req}: ${lambda_problem}"
18992 echo "Lambda runtime pins|${lambda_req}: ${lambda_problem}" >> "$CONSISTENCY_RESULTS"
1900 done <<< "$LAMBDA_PIN_PROBLEMS"
1901fi
1902
1903# An immutable digest is only immutable where it is written down. When an
1904# upstream tag is re-pushed and the pin is refreshed in one file but restated as
1905# a literal in another, nothing reconciled the two until whichever test held the
1906# stale copy happened to run. Two digests under one tag is unambiguous drift.
190752IMAGE_DIGEST_PROBLEMS="$(check_image_digest_consistency .)"
190826if [ -n "$IMAGE_DIGEST_PROBLEMS" ]; then
19094 while IFS='|' read -r image_repo image_problem; do
19101 [ -n "$image_repo" ] || continue
19111 echo " - image digest: ${image_repo}: ${image_problem}"
19121 echo "Image digest copies|${image_repo}: ${image_problem}" >> "$CONSISTENCY_RESULTS"
1913 done <<< "$IMAGE_DIGEST_PROBLEMS"
1914fi
1915
191678CONSISTENCY_COUNT="$(wc -l < "$CONSISTENCY_RESULTS" 2>/dev/null | tr -d ' ')"
191726[ -z "$CONSISTENCY_COUNT" ] && CONSISTENCY_COUNT=0
1918
1919# ---------------------------------------------------------------------------
1920# Base-image security epochs (no network)
1921#
1922# The service images (Debian) and the helm-installer Lambda (AL2023) pull OS
1923# security patches at build time behind a hand-bumped ``*_SECURITY_EPOCH``
1924# ARG that busts the CI layer cache. Nothing else reminds anyone to move the
1925# date, so a stale epoch silently reuses an old upgrade layer. Trivy's
1926# container scan is the backstop; this flags an epoch older than the window
1927# as the proactive nudge. Only the real Dockerfiles are scanned — the
1928# generated ``*-build`` staging copies are skipped.
1929# ---------------------------------------------------------------------------
193026echo ""
193126echo "=== Checking base-image security epochs ==="
193226
193352EPOCH_RESULTS="$(mktemp)"
193426EPOCH_FILES=(dockerfiles/*-dockerfile Dockerfile.dev lambda/helm-installer/Dockerfile)
193588for df in "${EPOCH_FILES[@]}"; do
193691 [ -f "$df" ] || continue
1937423 extract_security_epochs "$df" | while IFS='|' read -r epoch_arg epoch_date; do
193884 [ -z "$epoch_date" ] && continue
1939168 epoch_age="$(days_since "$epoch_date")"
194084 [ -z "$epoch_age" ] && continue
194184 if [ "$epoch_age" -gt "$SECURITY_EPOCH_STALE_DAYS" ]; then
19425 echo " - ${df} (${epoch_arg}): ${epoch_date} (${epoch_age} days old)"
19435 echo "${df}|${epoch_arg}|${epoch_date}|${epoch_age}" >> "$EPOCH_RESULTS"
1944 fi
1945 done
1946done
1947
194878EPOCH_COUNT="$(wc -l < "$EPOCH_RESULTS" 2>/dev/null | tr -d ' ')"
194926[ -z "$EPOCH_COUNT" ] && EPOCH_COUNT=0
1950
1951# ---------------------------------------------------------------------------
1952# Suppression expiries (no network)
1953#
1954# ``.trivyignore`` / ``.pip-audit-ignore`` / ``.npm-audit-ignore`` entries
1955# carry an ``exp:YYYY-MM-DD`` marker. The CI validators hard-fail a PR on the
1956# day an entry expires; this surfaces entries expiring *soon* so they get
1957# re-evaluated (fixed upstream? extend with a new justification?) before they
1958# break a build.
1959# ---------------------------------------------------------------------------
196026echo ""
196126echo "=== Checking suppression expiries ==="
196226
196352SUPPRESSION_RESULTS="$(mktemp)"
196478for supfile in .github/config/.trivyignore .github/config/.pip-audit-ignore .github/config/.npm-audit-ignore; do
196581 [ -f "$supfile" ] || continue
1966150 supbase="$(basename "$supfile")"
1967377 parse_suppression_expiries "$supfile" | while IFS='|' read -r sup_id sup_date; do
196876 [ -z "$sup_date" ] && continue
1969152 sup_left="$(days_until "$sup_date")"
197076 [ -z "$sup_left" ] && continue
197176 if [ "$sup_left" -le "$SUPPRESSION_EXPIRY_WARN_DAYS" ]; then
19723 echo " - ${supbase}: ${sup_id} expires ${sup_date} (${sup_left} days)"
19733 echo "${supbase}|${sup_id}|${sup_date}|${sup_left}" >> "$SUPPRESSION_RESULTS"
1974 fi
1975 done
1976done
1977
197878SUPPRESSION_COUNT="$(wc -l < "$SUPPRESSION_RESULTS" 2>/dev/null | tr -d ' ')"
197926[ -z "$SUPPRESSION_COUNT" ] && SUPPRESSION_COUNT=0
1980
1981# ---------------------------------------------------------------------------
1982# Lockfile freshness (no network)
1983#
1984# ``requirements-lock.txt`` is compiled from ``pyproject.toml`` with
1985# ``pip-compile --all-extras``. Every direct dependency must use an exact,
1986# concrete pin and is matched to the lock by normalized name, canonical marker
1987# identity, and version. Missing or mismatched records mean the lock is stale;
1988# unrelated transitive pins are ignored.
1989# ---------------------------------------------------------------------------
199026echo ""
199126echo "=== Checking lockfile freshness ==="
199226
199352LOCKFILE_RESULTS="$(mktemp)"
199452LOCKFILE_RAW="$(mktemp)"
199526if check_lockfile_freshness pyproject.toml requirements-lock.txt > "$LOCKFILE_RAW"; then
199652 while IFS='|' read -r lock_name expected_version locked_version; do
19972 [ -z "$lock_name" ] && continue
19982 if [ "$locked_version" = "<missing>" ]; then
19991 echo " - direct dep missing from requirements-lock.txt: ${lock_name}==${expected_version}"
2000 else
20011 echo " - direct dep version mismatch: ${lock_name}==${expected_version} (lock has ${locked_version})"
2002 fi
20032 echo "${lock_name}|${expected_version}|${locked_version}" >> "$LOCKFILE_RESULTS"
2004 done < "$LOCKFILE_RAW"
2005else
20062 mark_scan_incomplete "Lockfile freshness validation failed; inspect its error above."
2007fi
200826rm -f "$LOCKFILE_RAW"
2009
201078LOCKFILE_COUNT="$(wc -l < "$LOCKFILE_RESULTS" 2>/dev/null | tr -d ' ')"
201126[ -z "$LOCKFILE_COUNT" ] && LOCKFILE_COUNT=0
2012
2013# ---------------------------------------------------------------------------
2014# Accelerator catalog and Karpenter NodePools
2015#
2016# The deterministic check always runs and validates the reviewed catalog
2017# against every NodePool plus the exact cdk.json capacity-history watch list.
2018# With AWS credentials, the monthly job also compares the catalog against the
2019# union of NVIDIA GPU / AWS Neuron types returned across enabled commercial
2020# Regions. Ordinary policy/catalog drift joins the rolling dependency issue;
2021# execution or parser failures become one operational finding, never a
2022# false-clean result.
2023# ---------------------------------------------------------------------------
202426echo ""
202526echo "=== Checking accelerator catalog and Karpenter NodePools ==="
202626
202752ACCELERATOR_OFFLINE_REPORT="$(mktemp)"
202852ACCELERATOR_ONLINE_REPORT="$(mktemp)"
202952ACCELERATOR_ONLINE_SUMMARY="$(mktemp)"
203052ACCELERATOR_OFFLINE_ERROR="$(mktemp)"
203152ACCELERATOR_ONLINE_ERROR="$(mktemp)"
203226ACCELERATOR_OFFLINE_COUNT=0
203326ACCELERATOR_ONLINE_COUNT=0
203426ACCELERATOR_SKIP_REASON=""
203526ACCELERATOR_SUMMARY_SKIP_REASON=""
2036
2037write_accelerator_operational_report() {
20385 local report_path="$1" title="$2" detail="$3" error_path="$4"
2039 {
20405 echo "## ${title}"
20415 echo ""
20425 echo "**Status: OPERATIONAL ERROR.**"
20435 echo ""
20445 echo "### Accelerator maintenance check could not complete"
20455 echo ""
20465 echo "- **Why:** ${detail}"
20475 echo "- **Recommended change:** Re-run the command locally, repair the tool or credentials, and do not treat this scan as current until it succeeds."
20485 if [ -s "$error_path" ]; then
20492 echo "- **Tool output:**"
20502 sed 's/^/ /' "$error_path"
2051 fi
2052 } > "$report_path"
2053}
2054
2055record_accelerator_operational_error() {
20565 local report_path="$1" title="$2" detail="$3" error_path="$4"
20575 write_accelerator_operational_report "$report_path" "$title" "$detail" "$error_path"
20585 mark_scan_incomplete "${title}: ${detail}"
2059}
2060
206126python3 scripts/accelerator_catalog.py validate \
2062 --format markdown \
2063 --output "$ACCELERATOR_OFFLINE_REPORT" \
2064 2>"$ACCELERATOR_OFFLINE_ERROR"
206526ACCELERATOR_OFFLINE_STATUS=$?
206626if [ "$ACCELERATOR_OFFLINE_STATUS" -eq 0 ]; then
206723 echo " Offline NodePool/watch-list policy is current."
20683elif [ "$ACCELERATOR_OFFLINE_STATUS" -eq 1 ]; then
20695 ACCELERATOR_OFFLINE_COUNT="$(grep -c '^### ' "$ACCELERATOR_OFFLINE_REPORT" || true)"
20702 if ! [[ "$ACCELERATOR_OFFLINE_COUNT" =~ ^[1-9][0-9]*$ ]]; then
20711 ACCELERATOR_OFFLINE_COUNT=1
20721 record_accelerator_operational_error \
2073 "$ACCELERATOR_OFFLINE_REPORT" \
2074 "Offline accelerator catalog validation" \
2075 "The validator reported drift but emitted no parseable actionable findings." \
2076 "$ACCELERATOR_OFFLINE_ERROR"
2077 else
20781 echo " Found ${ACCELERATOR_OFFLINE_COUNT} offline accelerator policy finding(s)."
2079 fi
2080else
20811 ACCELERATOR_OFFLINE_COUNT=1
20821 record_accelerator_operational_error \
2083 "$ACCELERATOR_OFFLINE_REPORT" \
2084 "Offline accelerator catalog validation" \
2085 "The deterministic validator exited with status ${ACCELERATOR_OFFLINE_STATUS}." \
2086 "$ACCELERATOR_OFFLINE_ERROR"
20871 echo " Offline accelerator validator failed operationally."
2088fi
2089
209026if ! aws sts get-caller-identity >/dev/null 2>&1; then
20913 ACCELERATOR_SKIP_REASON="No AWS credentials available for the online EC2 catalog check (needs ec2:DescribeRegions and ec2:DescribeInstanceTypes); offline policy validation still ran."
20923 echo " $ACCELERATOR_SKIP_REASON"
2093else
209423 python3 scripts/accelerator_catalog.py check-online \
2095 --report "$ACCELERATOR_ONLINE_REPORT" \
2096 --json-summary \
2097 >"$ACCELERATOR_ONLINE_SUMMARY" \
2098 2>"$ACCELERATOR_ONLINE_ERROR"
209923 ACCELERATOR_ONLINE_STATUS=$?
210026 if [ "$ACCELERATOR_ONLINE_STATUS" -eq 0 ] || [ "$ACCELERATOR_ONLINE_STATUS" -eq 1 ]; then
210144 if ACCELERATOR_ONLINE_COUNT="$(parse_accelerator_drift_count "$ACCELERATOR_ONLINE_SUMMARY")"; then
210240 if { [ "$ACCELERATOR_ONLINE_STATUS" -eq 0 ] && [ "$ACCELERATOR_ONLINE_COUNT" -ne 0 ]; } \
210322 || { [ "$ACCELERATOR_ONLINE_STATUS" -eq 1 ] && [ "$ACCELERATOR_ONLINE_COUNT" -eq 0 ]; }; then
21041 ACCELERATOR_ONLINE_COUNT=1
21051 record_accelerator_operational_error \
2106 "$ACCELERATOR_ONLINE_REPORT" \
2107 "Online EC2 accelerator catalog drift" \
2108 "The command exit status disagreed with its JSON drift summary." \
2109 "$ACCELERATOR_ONLINE_ERROR"
21101 echo " Online accelerator scan returned an inconsistent result."
211120 elif [ "$ACCELERATOR_ONLINE_COUNT" -eq 0 ]; then
211218 echo " Live EC2 accelerator catalog is current."
2113 else
21142 echo " Found ${ACCELERATOR_ONLINE_COUNT} live EC2 catalog drift finding(s)."
2115 fi
2116 else
21171 ACCELERATOR_ONLINE_COUNT=1
21181 record_accelerator_operational_error \
2119 "$ACCELERATOR_ONLINE_REPORT" \
2120 "Online EC2 accelerator catalog drift" \
2121 "The online scanner emitted a missing or malformed JSON summary." \
2122 "$ACCELERATOR_ONLINE_ERROR"
21231 echo " Online accelerator scan summary could not be parsed."
2124 fi
2125 else
21261 ACCELERATOR_ONLINE_COUNT=1
21271 record_accelerator_operational_error \
2128 "$ACCELERATOR_ONLINE_REPORT" \
2129 "Online EC2 accelerator catalog drift" \
2130 "The online scanner exited with status ${ACCELERATOR_ONLINE_STATUS}." \
2131 "$ACCELERATOR_ONLINE_ERROR"
21321 echo " Online accelerator scanner failed operationally."
2133 fi
2134fi
2135
213626ACCELERATOR_COUNT=$((ACCELERATOR_OFFLINE_COUNT + ACCELERATOR_ONLINE_COUNT))
213729if [ -n "$ACCELERATOR_SKIP_REASON" ] && [ "$ACCELERATOR_COUNT" -eq 0 ]; then
21383 ACCELERATOR_SUMMARY_SKIP_REASON="$ACCELERATOR_SKIP_REASON"
2139fi
2140
2141# ---------------------------------------------------------------------------
2142# Summary + Markdown report
2143# ---------------------------------------------------------------------------
214426echo ""
214526echo "=== Summary ==="
214626echo "Python packages outdated: $PYTHON_COUNT"
214726echo "Docker images outdated: $DOCKER_COUNT"
214826echo "Helm charts outdated: $HELM_COUNT"
214926if [ -n "$ADDON_SKIP_REASON" ]; then
21506 echo "EKS add-ons outdated: (skipped)"
2151else
215220 echo "EKS add-ons outdated: $ADDON_COUNT"
2153fi
215426if [ -n "$EKS_K8S_SKIP_REASON" ]; then
21556 echo "EKS Kubernetes version: (skipped)"
2156else
215720 echo "EKS Kubernetes version: $EKS_K8S_COUNT"
2158fi
215926if [ -n "$AURORA_SKIP_REASON" ]; then
21605 echo "Aurora PostgreSQL: (skipped)"
2161else
216221 echo "Aurora PostgreSQL: $AURORA_COUNT"
2163fi
216426if [ -n "$EMR_SKIP_REASON" ]; then
21657 echo "EMR Serverless release: (skipped)"
2166else
216719 echo "EMR Serverless release: $EMR_COUNT"
2168fi
216926if [ -n "$BEDROCK_MODEL_SKIP_REASON" ]; then
21706 echo "Bedrock default model: (skipped)"
2171else
217220 echo "Bedrock default model: $BEDROCK_MODEL_COUNT"
2173fi
217426if [ -n "$ACCELERATOR_SKIP_REASON" ]; then
21753 echo "Accelerator catalog: ${ACCELERATOR_COUNT} (online skipped)"
2176else
217723 echo "Accelerator catalog: $ACCELERATOR_COUNT"
2178fi
217926echo "Dockerfile.dev pins: $DOCKERFILE_COUNT"
218026echo "Pre-commit hooks: $PRECOMMIT_COUNT"
218126if [ -n "$CDK_ENUM_SKIP_REASON" ]; then
21822 echo "CDK enum constants: (skipped)"
2183else
218424 echo "CDK enum constants: $CDK_ENUM_COUNT"
2185fi
218626if [ -n "$PYTHON_RELEASE_SKIP_REASON" ]; then
21873 echo "Python release: (skipped)"
2188else
218923 echo "Python release: $PYTHON_RELEASE_COUNT"
2190fi
219126if [ -n "$RUBY_RELEASE_SKIP_REASON" ]; then
21923 echo "Ruby release: (skipped)"
2193else
219423 echo "Ruby release: $RUBY_RELEASE_COUNT"
2195fi
219626if [ -n "$RUNNER_IMAGE_SKIP_REASON" ]; then
21972 echo "Runner images: (skipped)"
2198else
219924 echo "Runner images: $RUNNER_IMAGE_COUNT"
2200fi
220126if [ -n "$AUTOPILOT_SKIP_REASON" ]; then
22025 echo "GCO autopilot pins: (skipped)"
2203else
220421 echo "GCO autopilot pins: $AUTOPILOT_COUNT"
2205fi
220626echo "CI tooling pins: $CI_TOOLING_COUNT"
220726echo "Version consistency: $CONSISTENCY_COUNT"
220826echo "Base-image epochs: $EPOCH_COUNT"
220926echo "Suppression expiries: $SUPPRESSION_COUNT"
221026echo "Lockfile freshness: $LOCKFILE_COUNT"
221193INCOMPLETE_LOOKUP_COUNT="$(sort -u "$INCOMPLETE_REASONS_FILE" 2>/dev/null | grep -c . || true)"
221226echo "Incomplete lookups: ${INCOMPLETE_LOOKUP_COUNT:-0}"
2213
221426SCAN_COMPLETE=true
221526if ! dependency_scan_is_complete \
2216 "$INCOMPLETE_REASONS_FILE" \
2217 "$ADDON_SKIP_REASON" \
2218 "$EKS_K8S_SKIP_REASON" \
2219 "$AURORA_SKIP_REASON" \
2220 "$EMR_SKIP_REASON" \
2221 "$BEDROCK_MODEL_SKIP_REASON" \
2222 "$ACCELERATOR_SKIP_REASON" \
2223 "$AUTOPILOT_SKIP_REASON" \
2224 "$CDK_ENUM_SKIP_REASON" \
2225 "$PYTHON_RELEASE_SKIP_REASON" \
2226 "$RUBY_RELEASE_SKIP_REASON" \
2227 "$RUNNER_IMAGE_SKIP_REASON"; then
222815 SCAN_COMPLETE=false
2229fi
2230
223170if [ "$PYTHON_COUNT" -eq 0 ] && [ "$NPM_COUNT" -eq 0 ] && [ "$DOCKER_COUNT" -eq 0 ] \
223242 && [ "$HELM_COUNT" -eq 0 ] && [ "$ADDON_COUNT" -eq 0 ] \
223321 && [ "$EKS_K8S_COUNT" -eq 0 ] \
223442 && [ "$AURORA_COUNT" -eq 0 ] && [ "$EMR_COUNT" -eq 0 ] \
223521 && [ "$DOCKERFILE_COUNT" -eq 0 ] \
223621 && [ "$AUTOPILOT_COUNT" -eq 0 ] \
223718 && [ "$PRECOMMIT_COUNT" -eq 0 ] \
223818 && [ "$CDK_ENUM_COUNT" -eq 0 ] \
223918 && [ "$PYTHON_RELEASE_COUNT" -eq 0 ] \
224018 && [ "$RUBY_RELEASE_COUNT" -eq 0 ] \
224118 && [ "$RUNNER_IMAGE_COUNT" -eq 0 ] \
224218 && [ "$BEDROCK_MODEL_COUNT" -eq 0 ] \
224318 && [ "$ACCELERATOR_COUNT" -eq 0 ] \
224414 && [ "$CI_TOOLING_COUNT" -eq 0 ] \
224514 && [ "$CONSISTENCY_COUNT" -eq 0 ] \
224611 && [ "$EPOCH_COUNT" -eq 0 ] \
22477 && [ "$SUPPRESSION_COUNT" -eq 0 ] \
22487 && [ "$LOCKFILE_COUNT" -eq 0 ]; then
22497 echo ""
22507 SKIP_NOTES=""
22517 if [ -n "$ADDON_SKIP_REASON" ]; then
22523 SKIP_NOTES="EKS add-ons skipped: $ADDON_SKIP_REASON"
2253 fi
22547 if [ -n "$EKS_K8S_SKIP_REASON" ]; then
22554 [ -n "$SKIP_NOTES" ] && SKIP_NOTES="$SKIP_NOTES; "
22562 SKIP_NOTES="${SKIP_NOTES}EKS Kubernetes skipped: $EKS_K8S_SKIP_REASON"
2257 fi
22587 if [ -n "$AURORA_SKIP_REASON" ]; then
22594 [ -n "$SKIP_NOTES" ] && SKIP_NOTES="$SKIP_NOTES; "
22602 SKIP_NOTES="${SKIP_NOTES}Aurora engine skipped: $AURORA_SKIP_REASON"
2261 fi
22627 if [ -n "$EMR_SKIP_REASON" ]; then
22634 [ -n "$SKIP_NOTES" ] && SKIP_NOTES="$SKIP_NOTES; "
22642 SKIP_NOTES="${SKIP_NOTES}EMR Serverless skipped: $EMR_SKIP_REASON"
2265 fi
22667 if [ -n "$BEDROCK_MODEL_SKIP_REASON" ]; then
22674 [ -n "$SKIP_NOTES" ] && SKIP_NOTES="$SKIP_NOTES; "
22682 SKIP_NOTES="${SKIP_NOTES}Bedrock model skipped: $BEDROCK_MODEL_SKIP_REASON"
2269 fi
22707 if [ -n "$ACCELERATOR_SKIP_REASON" ]; then
22714 [ -n "$SKIP_NOTES" ] && SKIP_NOTES="$SKIP_NOTES; "
22722 SKIP_NOTES="${SKIP_NOTES}Online accelerator catalog skipped: $ACCELERATOR_SKIP_REASON"
2273 fi
22747 if [ -n "$AUTOPILOT_SKIP_REASON" ]; then
22753 [ -n "$SKIP_NOTES" ] && SKIP_NOTES="$SKIP_NOTES; "
22762 SKIP_NOTES="${SKIP_NOTES}GCO autopilot pins skipped: $AUTOPILOT_SKIP_REASON"
2277 fi
22787 if [ -n "$CDK_ENUM_SKIP_REASON" ]; then
22792 [ -n "$SKIP_NOTES" ] && SKIP_NOTES="$SKIP_NOTES; "
22801 SKIP_NOTES="${SKIP_NOTES}CDK enums skipped: $CDK_ENUM_SKIP_REASON"
2281 fi
22827 if [ -n "$PYTHON_RELEASE_SKIP_REASON" ]; then
22832 [ -n "$SKIP_NOTES" ] && SKIP_NOTES="$SKIP_NOTES; "
22841 SKIP_NOTES="${SKIP_NOTES}Python release skipped: $PYTHON_RELEASE_SKIP_REASON"
2285 fi
22867 if [ -n "$RUBY_RELEASE_SKIP_REASON" ]; then
22872 [ -n "$SKIP_NOTES" ] && SKIP_NOTES="$SKIP_NOTES; "
22881 SKIP_NOTES="${SKIP_NOTES}Ruby release skipped: $RUBY_RELEASE_SKIP_REASON"
2289 fi
22907 if [ -n "$RUNNER_IMAGE_SKIP_REASON" ]; then
22912 [ -n "$SKIP_NOTES" ] && SKIP_NOTES="$SKIP_NOTES; "
22921 SKIP_NOTES="${SKIP_NOTES}Runner images skipped: $RUNNER_IMAGE_SKIP_REASON"
2293 fi
22947 if [ -s "$INCOMPLETE_REASONS_FILE" ]; then
22954 [ -n "$SKIP_NOTES" ] && SKIP_NOTES="$SKIP_NOTES; "
22966 SKIP_NOTES="${SKIP_NOTES}Incomplete checks: $(join_scan_incomplete_reasons)"
2297 fi
22987 if [ "$SCAN_COMPLETE" != true ]; then
22996 STATUS_MESSAGE="No drift was found in completed checks, but the scan is incomplete."
2300 else
23011 STATUS_MESSAGE="All dependencies are up to date."
2302 fi
23037 echo "$STATUS_MESSAGE"
23047 rm -f "$NPM_RESULTS" "$DOCKER_RESULTS" "$HELM_RESULTS" "$ADDON_RESULTS" "$EKS_K8S_RESULTS" "$AURORA_RESULTS" "$EMR_RESULTS" "$DOCKERFILE_RESULTS" "$AUTOPILOT_RESULTS" "$PRECOMMIT_RESULTS" "$CDK_ENUM_RESULTS" "$PYTHON_RELEASE_RESULTS" "$RUBY_RELEASE_RESULTS" "$RUNNER_IMAGE_RESULTS" "$RUNNER_IMAGE_NOTES" "$BEDROCK_MODEL_RESULTS" "$CI_TOOLING_RESULTS" "$CONSISTENCY_RESULTS" "$EPOCH_RESULTS" "$SUPPRESSION_RESULTS" "$LOCKFILE_RESULTS" "$ACCELERATOR_OFFLINE_REPORT" "$ACCELERATOR_ONLINE_REPORT" "$ACCELERATOR_ONLINE_SUMMARY" "$ACCELERATOR_OFFLINE_ERROR" "$ACCELERATOR_ONLINE_ERROR" "$INCOMPLETE_REASONS_FILE"
23057 if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
2306 {
23077 echo "# Dependency Update Report"
23087 echo ""
23097 echo "$STATUS_MESSAGE"
23107 if [ -n "$SKIP_NOTES" ]; then
23116 echo ""
23126 echo "_Incomplete or skipped checks: ${SKIP_NOTES}_"
2313 fi
2314 } >> "$GITHUB_STEP_SUMMARY"
2315 fi
23167 if [ -n "${GITHUB_OUTPUT:-}" ]; then
2317 {
23187 echo "has_drift=false"
23197 echo "scan_complete=$SCAN_COMPLETE"
2320 } >> "$GITHUB_OUTPUT"
2321 fi
23227 exit 0
2323fi
2324
2325# summary_row <title> <count> <skip_reason> <urgency>
2326# Emits one row of the top-of-report TL;DR table. Surfaces with drift link to
2327# their detailed section; skipped surfaces are marked and get no urgency.
2328summary_row() {
2329418 local title="$1" count="$2" skip="$3" urgency="$4"
2330418 local anchor label
2331836 anchor="$(md_anchor "$title")"
2332418 if [ -n "$skip" ]; then
233329 label="skipped"
233429 urgency="—"
2335389 elif [ "$count" -gt 0 ]; then
233686 label="${count} update(s)"
233786 title="[${title}](#${anchor})"
2338303 elif [ "$SCAN_COMPLETE" != true ]; then
2339145 label="no drift found (incomplete scan)"
2340145 urgency="—"
2341 else
2342158 label="up to date"
2343158 urgency="—"
2344 fi
2345418 echo "| ${title} | ${label} | ${urgency} |"
2346}
2347
2348{
234919 echo "# Dependency Update Report"
235019 echo ""
235138 echo "_Generated $(date -u '+%Y-%m-%d %H:%M UTC') by the \`deps-scan\` workflow._"
235219 echo ""
235319 echo "> [!TIP]"
235419 echo "> Reproduce this update list programmatically, on demand: run"
235519 echo "> \`gco deps scan\` from a checkout (add \`--nodepools-only\` for just the"
235619 echo "> accelerator-catalog / NodePool freshness check, \`-o json\` for a"
235719 echo "> machine-readable envelope), or call the \`deps_scan\` MCP tool from an"
235819 echo "> agent session. Surfaces needing AWS credentials or missing host tools"
235919 echo "> are skipped and flagged as incomplete, exactly as in this workflow."
236019 echo ""
236119 if [ "$SCAN_COMPLETE" != true ]; then
23629 echo "> [!WARNING]"
23639 echo "> **Incomplete scan.** Zero-count surfaces are provisional, not confirmed current."
23649 if [ -s "$INCOMPLETE_REASONS_FILE" ]; then
236516 echo "> Recorded failures: $(join_scan_incomplete_reasons)"
2366 else
23671 echo "> One or more credential-dependent or optional checks were skipped; see the workflow log."
2368 fi
23699 echo ""
2370 fi
2371
2372 # ----- TL;DR summary -----
237319 echo "## Summary"
237419 echo ""
237519 echo "| Surface | Status | Urgency |"
237619 echo "|---------|--------|---------|"
237719 summary_row "Python Packages" "$PYTHON_COUNT" "" "routine"
237819 summary_row "npm Packages" "$NPM_COUNT" "" "routine"
237919 summary_row "Docker Images" "$DOCKER_COUNT" "" "routine"
238019 summary_row "Helm Charts" "$HELM_COUNT" "" "routine"
238119 summary_row "EKS Add-ons" "$ADDON_COUNT" "$ADDON_SKIP_REASON" "routine"
238219 summary_row "EKS Kubernetes Version" "$EKS_K8S_COUNT" "$EKS_K8S_SKIP_REASON" "act soon"
238319 summary_row "Aurora PostgreSQL Engine" "$AURORA_COUNT" "$AURORA_SKIP_REASON" "routine"
238419 summary_row "EMR Serverless" "$EMR_COUNT" "$EMR_SKIP_REASON" "routine"
238519 summary_row "Bedrock Default Model" "$BEDROCK_MODEL_COUNT" "$BEDROCK_MODEL_SKIP_REASON" "routine"
238619 summary_row "Accelerator Catalog and NodePools" "$ACCELERATOR_COUNT" "$ACCELERATOR_SUMMARY_SKIP_REASON" "act soon"
238719 summary_row "Dockerfile.dev Pins" "$DOCKERFILE_COUNT" "" "routine"
238819 summary_row "GCO Autopilot Pins" "$AUTOPILOT_COUNT" "$AUTOPILOT_SKIP_REASON" "act soon"
238919 summary_row "Pre-commit Hooks" "$PRECOMMIT_COUNT" "" "routine"
239019 summary_row "CDK Enum Constants" "$CDK_ENUM_COUNT" "$CDK_ENUM_SKIP_REASON" "routine"
239119 summary_row "Python Release" "$PYTHON_RELEASE_COUNT" "$PYTHON_RELEASE_SKIP_REASON" "informational"
239219 summary_row "Ruby Release" "$RUBY_RELEASE_COUNT" "$RUBY_RELEASE_SKIP_REASON" "informational"
239319 summary_row "Runner Images" "$RUNNER_IMAGE_COUNT" "$RUNNER_IMAGE_SKIP_REASON" "act soon"
239419 summary_row "CI Tooling" "$CI_TOOLING_COUNT" "" "act soon"
239519 summary_row "Version Consistency" "$CONSISTENCY_COUNT" "" "routine"
239619 summary_row "Base-image Security Epochs" "$EPOCH_COUNT" "" "act soon"
239719 summary_row "Suppression Expiries" "$SUPPRESSION_COUNT" "" "act soon"
239819 summary_row "Lockfile Freshness" "$LOCKFILE_COUNT" "" "routine"
239919 echo ""
240019 echo "_Urgency is a hint: **act soon** = security or a support/cost deadline;"
240119 echo "**routine** = bump at leisure; **informational** = no action yet. Only"
240219 echo "surfaces with drift have a detailed section below._"
240319 echo ""
2404
240519 if [ "$PYTHON_COUNT" -gt 0 ]; then
24064 echo "## Python Packages"
24074 echo ""
24084 echo "Direct dependencies pinned in \`pyproject.toml\` (transitive-only drift"
24094 echo "is excluded — those versions are controlled by upstream pins and bumping"
24104 echo "them ourselves either no-ops or breaks the resolver)."
24114 echo ""
24124 echo "| Package | Current | Latest | Ref |"
24134 echo "|---------|---------|--------|-----|"
24148 echo "$PYTHON_OUTDATED" | jq -r '.[] | "| \(.name) | \(.version) | \(.latest_version) | [PyPI](https://pypi.org/project/\(.name)/) |"'
24154 echo ""
2416 fi
2417
241819 if [ "$NPM_COUNT" -gt 0 ]; then
24194 echo "## npm Packages"
24204 echo ""
24214 echo "Exact direct pins in every repository-owned \`package.json\` (the root"
24224 echo "tooling graph and \`lambda/inference-streaming-proxy\`), compared against"
24234 echo "each package's \`latest\` dist-tag. Transitives are excluded — those are"
24244 echo "controlled by the lockfiles. Bump the pin, then regenerate the graph's"
24254 echo "\`package-lock.json\` with the pinned npm."
24264 echo ""
24278 npm_disp="$(mktemp)"
242826 while IFS='|' read -r graph pkg cur lat; do
24299 echo "\`${graph}\`|${pkg}|${cur}|${lat}|[npm](https://www.npmjs.com/package/${pkg})"
2430 done < "$NPM_RESULTS" > "$npm_disp"
24314 emit_md_table "Graph|Package|Current|Latest|Ref" "$npm_disp"
24324 rm -f "$npm_disp"
24334 echo ""
2434 fi
2435
243619 if [ "$DOCKER_COUNT" -gt 0 ]; then
24375 echo "## Docker Images"
24385 echo ""
24395 emit_md_table "Image|Current|Latest" "$DOCKER_RESULTS"
24405 echo ""
2441 fi
2442
244319 if [ "$HELM_COUNT" -gt 0 ]; then
24444 echo "## Helm Charts"
24454 echo ""
24468 helm_disp="$(mktemp)"
244750 while IFS='|' read -r cname chart cur lat; do
244821 echo "${cname}|${chart}|${cur}|${lat}|[ArtifactHub](https://artifacthub.io/packages/search?ts_query_web=${chart})"
2449 done < "$HELM_RESULTS" > "$helm_disp"
24504 emit_md_table "Chart|Name|Current|Latest|Ref" "$helm_disp"
24514 rm -f "$helm_disp"
24524 echo ""
2453 fi
2454
245519 if [ "$ADDON_COUNT" -gt 0 ]; then
24564 echo "## EKS Add-ons"
24574 echo ""
24584 emit_md_table "Add-on|Current|Latest" "$ADDON_RESULTS"
24594 echo ""
2460 fi
2461
246219 if [ "$EKS_K8S_COUNT" -gt 0 ]; then
24633 echo "## EKS Kubernetes Version"
24643 echo ""
24653 echo "The Kubernetes minor pinned in \`cdk.json::kubernetes_version\` is behind"
24663 echo "the latest release still in EKS **standard support**. Upgrade before"
24673 echo "standard support ends to avoid the extended-support pricing uplift."
24683 echo ""
24696 eks_disp="$(mktemp)"
247012 while IFS='|' read -r pin cur lat eos; do
24713 echo "\`${pin}\`|${cur}|${lat}|${eos}"
2472 done < "$EKS_K8S_RESULTS" > "$eks_disp"
24733 emit_md_table "Pin|Current|Latest (standard support)|Std support ends" "$eks_disp"
24743 rm -f "$eks_disp"
24753 echo ""
2476 fi
2477
247819 if [ "$AURORA_COUNT" -gt 0 ]; then
24794 echo "## Aurora PostgreSQL Engine"
24804 echo ""
24814 emit_md_table "Engine|Current|Latest" "$AURORA_RESULTS"
24824 echo ""
2483 fi
2484
248519 if [ "$EMR_COUNT" -gt 0 ]; then
24863 echo "## EMR Serverless"
24873 echo ""
24883 emit_md_table "Release|Current|Latest" "$EMR_RESULTS"
24893 echo ""
2490 fi
2491
249219 if [ "$BEDROCK_MODEL_COUNT" -gt 0 ]; then
24933 echo "## Bedrock Default Model"
24943 echo ""
24953 echo "A Bedrock model default configured in \`cdk.json\` is behind a newer"
24963 echo "release in the same model family. For"
24973 echo "\`bedrock.mission_default_model_id\` and \`bedrock.codex_default_model_id\`,"
24983 echo "update the value and re-capture the matching scaffold fixture"
24993 echo "(\`scripts/capture_scaffold_fixtures.py\`). For"
25003 echo "\`bedrock.capacity_advisor_default_model_id\` and"
25013 echo "\`bedrock.claude_code_default_model_id\`, updating the value is enough."
25023 echo "For the embedding keys — \`bedrock.embedding_model_id\` (Mission memory)"
25033 echo "and \`vector_store.embedding_model_id\` (workload RAG corpus) — stored"
25043 echo "vectors are only comparable to vectors from the same model: plan to"
25053 echo "re-embed (for the vector store, re-ingest the corpus) or segregate"
25063 echo "existing data before adopting the newer model."
25073 echo ""
25083 emit_md_table "Configuration key|Current|Latest" "$BEDROCK_MODEL_RESULTS"
25093 echo ""
2510 fi
2511
251219 if [ "$ACCELERATOR_COUNT" -gt 0 ]; then
25135 echo "## Accelerator Catalog and NodePools"
25145 echo ""
25155 echo "The offline check keeps reviewed lifecycle/generation policy, Karpenter"
25165 echo "NodePools, \`historical.watch_instance_types\`, and the Spot Placement"
25175 echo "Score instance pools synchronized. The online check compares the catalog"
25185 echo "with EC2 across enabled commercial Regions."
25195 echo "Follow each recommended change; review family metadata before refreshing"
25205 echo "the checked-in catalog."
25215 echo ""
25225 if [ -s "$ACCELERATOR_OFFLINE_REPORT" ]; then
25233 sed -E 's/^### /#### /; s/^## /### /' "$ACCELERATOR_OFFLINE_REPORT"
25243 echo ""
2525 fi
25265 if [ -s "$ACCELERATOR_ONLINE_REPORT" ]; then
25275 sed -E 's/^### /#### /; s/^## /### /' "$ACCELERATOR_ONLINE_REPORT"
25285 echo ""
2529 fi
2530 fi
2531
253219 if [ "$DOCKERFILE_COUNT" -gt 0 ]; then
25334 echo "## Dockerfile.dev Pins"
25344 echo ""
25354 echo "Tooling versions pinned as build-time ARGs in \`Dockerfile.dev\`."
25364 echo ""
25378 dockerfile_disp="$(mktemp)"
253872 while IFS='|' read -r pin cur lat; do
253932 echo "\`${pin}\`|${cur}|${lat}"
2540 done < "$DOCKERFILE_RESULTS" > "$dockerfile_disp"
25414 emit_md_table "Pin|Current|Latest" "$dockerfile_disp"
25424 rm -f "$dockerfile_disp"
25434 echo ""
2544 fi
2545
254619 if [ "$AUTOPILOT_COUNT" -gt 0 ]; then
25477 echo "## GCO Autopilot Pins"
25487 echo ""
25497 echo "\`gco autopilot\`'s dependency surfaces in \`cli/autopilot.py\`: the"
25507 echo "pinned \`CLAUDE_CODE_VERSION\` and \`CODEX_VERSION\` agent CLIs"
25517 echo "(each compared against its npm \`latest\` dist-tag) and the launch-time"
25527 echo "companion MCP servers"
25537 echo "(reported when a package is missing, deprecated, or yanked on its"
25547 echo "registry — an unhealthy companion breaks every new session, so treat"
25557 echo "it like the removals documented in \`gco_mcp/README.md\`). When"
25567 echo "changing the companion set, update \`cli/autopilot.py\` and the"
25577 echo "\`gco_mcp/README.md\` tables together; \`tests/test_cli_autopilot.py\`"
25587 echo "enforces the lockstep."
25597 echo ""
256014 autopilot_disp="$(mktemp)"
256142 while IFS='|' read -r surface cur stat url; do
256214 echo "${surface}|\`${cur}\`|${stat}|[registry](${url})"
2563 done < "$AUTOPILOT_RESULTS" > "$autopilot_disp"
25647 emit_md_table "Surface|Current|Latest / status|Ref" "$autopilot_disp"
25657 rm -f "$autopilot_disp"
25667 echo ""
2567 fi
2568
256919 if [ "$PRECOMMIT_COUNT" -gt 0 ]; then
25705 echo "## Pre-commit Hooks"
25715 echo ""
25725 echo "Hook \`rev:\` pins in \`.pre-commit-config.yaml\` are behind the latest"
25735 echo "tag published by their upstream repos. Bump in \`.pre-commit-config.yaml\`,"
25745 echo "then run \`pre-commit autoupdate\` locally (or edit by hand) and verify"
25755 echo "the hooks still pass."
25765 echo ""
257710 precommit_disp="$(mktemp)"
257834 while IFS='|' read -r repo cur lat; do
257912 echo "${repo}|\`${cur}\`|\`${lat}\`|[releases](${repo}/releases)"
2580 done < "$PRECOMMIT_RESULTS" > "$precommit_disp"
25815 emit_md_table "Repo|Current|Latest|Ref" "$precommit_disp"
25825 rm -f "$precommit_disp"
25835 echo ""
2584 fi
2585
258619 if [ "$CDK_ENUM_COUNT" -gt 0 ]; then
25873 echo "## CDK Enum Constants"
25883 echo ""
25893 echo "Enum-name constants in \`gco/stacks/constants.py\` are behind the highest"
25903 echo "enum member exposed by the installed \`aws-cdk-lib\`. Update the constant"
25913 echo "in \`constants.py\` (and any related deployment notes) so new stacks"
25923 echo "construct the latest CDK enum."
25933 echo ""
25943 emit_md_table "Constant|CDK enum class|Current|Latest" "$CDK_ENUM_RESULTS" code
25953 echo ""
2596 fi
2597
259819 if [ "$PYTHON_RELEASE_COUNT" -gt 0 ]; then
25994 echo "## Python Release"
26004 echo ""
26014 echo "A newer stable Python release is available on python.org than the version"
26024 echo "encoded by \`LAMBDA_PYTHON_RUNTIME\`. AWS Lambda may lag the upstream"
26034 echo "release by a few months — wait for the matching \`Runtime.PYTHON_X_Y\`"
26044 echo "enum to appear in \`aws-cdk-lib\` (tracked by the **CDK Enum Constants**"
26054 echo "section above) before bumping. See <https://www.python.org/downloads/>."
26064 echo ""
26074 emit_md_table "Surface|Current|Latest" "$PYTHON_RELEASE_RESULTS"
26084 echo ""
2609 fi
2610
261119 if [ "$RUNNER_IMAGE_COUNT" -gt 0 ]; then
26121 echo "## Runner Images"
26131 echo ""
26141 echo "A \`runs-on:\` label below is behind a newer **generally-available** image,"
26151 echo "or names one upstream has deprecated (a deprecated image is a removal"
26161 echo "notice with a date attached). Images still in *preview* are deliberately"
26171 echo "not listed here — see the notes under the skipped/collapsed section for"
26181 echo "those, since moving to a preview trades a stable CI platform for an"
26191 echo "unannounced one. Update the \`runs-on:\` value in the named workflow job."
26201 echo "See <https://github.com/actions/runner-images#available-images>."
26211 echo ""
26221 emit_md_table "Label|Current image|Recommended" "$RUNNER_IMAGE_RESULTS"
26231 echo ""
2624 fi
2625
262619 if [ "$RUBY_RELEASE_COUNT" -gt 0 ]; then
26274 echo "## Ruby Release"
26284 echo ""
26294 echo "A newer stable Ruby series is supported upstream than the one pinned in"
26304 echo "\`.ruby-version\`. Ruby is CI-only — \`bundle exec bashcov\` measures shell"
26314 echo "coverage in the \`unit:bats:shell\` job — so bumping is low risk, but it is"
26324 echo "still a deliberate move: check that \`ruby/setup-ruby\` publishes a prebuilt"
26334 echo "binary for the new series and that \`bashcov\` supports it, then update"
26344 echo "\`.ruby-version\` and re-resolve \`Gemfile.lock\`."
26354 echo "See <https://endoflife.date/ruby>."
26364 echo ""
26374 emit_md_table "Surface|Current|Latest" "$RUBY_RELEASE_RESULTS"
26384 echo ""
2639 fi
2640
2641 # ----- New coverage surfaces -----
2642
264319 if [ "$CI_TOOLING_COUNT" -gt 0 ]; then
26445 echo "## CI Tooling"
26455 echo ""
26465 echo "Tool versions the workflows install by hand — not \`uses:\` refs or"
26475 echo "Dockerfile \`FROM\` lines, so Dependabot doesn't watch them. A stale"
26485 echo "**Trivy** in particular means the CVE scan silently misses newer"
26495 echo "detections; bump the \`*_VERSION\` env / kind-action inputs in lockstep."
26505 echo ""
265110 ci_disp="$(mktemp)"
265292 while IFS='|' read -r name cur lat url; do
265341 echo "${name}|\`${cur}\`|\`${lat}\`|[releases](${url})"
2654 done < "$CI_TOOLING_RESULTS" > "$ci_disp"
26555 emit_md_table "Tool|Current|Latest|Ref" "$ci_disp"
26565 rm -f "$ci_disp"
26575 echo ""
2658 fi
2659
266019 if [ "$CONSISTENCY_COUNT" -gt 0 ]; then
26614 echo "## Version Consistency"
26624 echo ""
26634 echo "These versions and dependency-management surfaces must move together."
26644 echo "The rows below identify missing coverage or disagreement across runtime,"
26654 echo "package, dev-container, pre-commit, and CI pins."
26664 echo ""
26674 emit_md_table "What|Pinned values" "$CONSISTENCY_RESULTS"
26684 echo ""
2669 fi
2670
267119 if [ "$EPOCH_COUNT" -gt 0 ]; then
26725 echo "## Base-image Security Epochs"
26735 echo ""
26745 echo "The build-time \`*_SECURITY_EPOCH\` ARGs bust the CI layer cache so a"
26755 echo "fresh OS-security-upgrade layer is built. An epoch older than"
26765 echo "${SECURITY_EPOCH_STALE_DAYS} days may be reusing a stale upgrade layer —"
26775 echo "bump the date to force a rebuild that pulls current patches."
26785 echo ""
267910 epoch_disp="$(mktemp)"
268020 while IFS='|' read -r df arg edate eage; do
26815 echo "\`${df}\`|\`${arg}\`|${edate}|${eage}"
2682 done < "$EPOCH_RESULTS" > "$epoch_disp"
26835 emit_md_table "Dockerfile|ARG|Epoch|Age (days)" "$epoch_disp"
26845 rm -f "$epoch_disp"
26855 echo ""
2686 fi
2687
268819 if [ "$SUPPRESSION_COUNT" -gt 0 ]; then
26893 echo "## Suppression Expiries"
26903 echo ""
26913 echo "\`.trivyignore\` / \`.pip-audit-ignore\` / \`.npm-audit-ignore\` entries"
26923 echo "expiring within ${SUPPRESSION_EXPIRY_WARN_DAYS} days (the CI validator hard-fails a PR on"
26933 echo "the expiry date). Re-evaluate each: drop it if the CVE is fixed upstream,"
26943 echo "or extend with a fresh justification if not."
26953 echo ""
26966 sup_disp="$(mktemp)"
269712 while IFS='|' read -r sbase sid sdate sleft; do
26983 echo "\`${sbase}\`|\`${sid}\`|${sdate}|${sleft}"
2699 done < "$SUPPRESSION_RESULTS" > "$sup_disp"
27003 emit_md_table "File|ID|Expires|Days left" "$sup_disp"
27013 rm -f "$sup_disp"
27023 echo ""
2703 fi
2704
270519 if [ "$LOCKFILE_COUNT" -gt 0 ]; then
27062 echo "## Lockfile Freshness"
27072 echo ""
27082 echo "Direct dependencies whose exact pyproject.toml pin is missing from or"
27092 echo "different in requirements-lock.txt. Regenerate the lock with"
27102 echo "pip-compile --all-extras --strip-extras -o requirements-lock.txt pyproject.toml."
27112 echo ""
27122 emit_md_table "Dependency|Expected|Locked" "$LOCKFILE_RESULTS" code
27132 echo ""
2714 fi
2715
2716 # ----- Skipped checks (collapsed) -----
271719 if [ -n "${ADDON_SKIP_REASON}${EKS_K8S_SKIP_REASON}${AURORA_SKIP_REASON}${EMR_SKIP_REASON}${BEDROCK_MODEL_SKIP_REASON}${ACCELERATOR_SKIP_REASON}${AUTOPILOT_SKIP_REASON}${CDK_ENUM_SKIP_REASON}${PYTHON_RELEASE_SKIP_REASON}${RUBY_RELEASE_SKIP_REASON}${RUNNER_IMAGE_SKIP_REASON}" ] \
271813 || [ -s "$INCOMPLETE_REASONS_FILE" ]; then
27199 echo "<details>"
27209 echo "<summary>Skipped checks</summary>"
27219 echo ""
272212 [ -n "$ADDON_SKIP_REASON" ] && echo "- **EKS Add-ons:** $ADDON_SKIP_REASON"
272313 [ -n "$EKS_K8S_SKIP_REASON" ] && echo "- **EKS Kubernetes Version:** $EKS_K8S_SKIP_REASON"
272412 [ -n "$AURORA_SKIP_REASON" ] && echo "- **Aurora PostgreSQL Engine:** $AURORA_SKIP_REASON"
272514 [ -n "$EMR_SKIP_REASON" ] && echo "- **EMR Serverless:** $EMR_SKIP_REASON"
272613 [ -n "$BEDROCK_MODEL_SKIP_REASON" ] && echo "- **Bedrock Default Model:** $BEDROCK_MODEL_SKIP_REASON"
272710 [ -n "$ACCELERATOR_SKIP_REASON" ] && echo "- **Online Accelerator Catalog:** $ACCELERATOR_SKIP_REASON"
272812 [ -n "$AUTOPILOT_SKIP_REASON" ] && echo "- **GCO Autopilot Pins:** $AUTOPILOT_SKIP_REASON"
272910 [ -n "$CDK_ENUM_SKIP_REASON" ] && echo "- **CDK Enum Constants:** $CDK_ENUM_SKIP_REASON"
273011 [ -n "$PYTHON_RELEASE_SKIP_REASON" ] && echo "- **Python Release:** $PYTHON_RELEASE_SKIP_REASON"
273111 [ -n "$RUBY_RELEASE_SKIP_REASON" ] && echo "- **Ruby Release:** $RUBY_RELEASE_SKIP_REASON"
273210 [ -n "$RUNNER_IMAGE_SKIP_REASON" ] && echo "- **Runner Images:** $RUNNER_IMAGE_SKIP_REASON"
27339 if [ -s "$INCOMPLETE_REASONS_FILE" ]; then
2734196 while IFS= read -r incomplete_reason; do
273590 echo "- **Incomplete lookup or parse:** ${incomplete_reason}"
2736 done < <(sort -u "$INCOMPLETE_REASONS_FILE")
2737 fi
27389 echo ""
27399 echo "</details>"
27409 echo ""
2741 fi
2742
274319 echo "## Action Required"
274419 echo ""
274519 echo "1. Review changelogs for breaking changes (see the per-row **Ref** links)"
274619 echo "2. Follow accelerator findings exactly; review lifecycle/generation metadata"
274719 echo " before running \`python scripts/accelerator_catalog.py refresh\`"
274819 echo "3. Update versions in \`pyproject.toml\`, manifests, \`charts.yaml\`, or the"
274919 echo " pinned \`*_VERSION\` env / ARG values"
275019 echo "4. Regenerate \`requirements-lock.txt\` if Python deps changed"
275119 echo "5. Reconcile any **Version Consistency** rows so every copy of a pin agrees"
275219 echo "6. Run tests locally to verify compatibility, then open a PR"
275319 echo ""
275419 echo "---"
275519 echo "_Automatically created by the \`deps-scan\` workflow._"
2756} > "$REPORT_FILE"
2757
275819rm -f "$NPM_RESULTS" "$DOCKER_RESULTS" "$HELM_RESULTS" "$ADDON_RESULTS" "$EKS_K8S_RESULTS" "$AURORA_RESULTS" "$EMR_RESULTS" "$DOCKERFILE_RESULTS" "$AUTOPILOT_RESULTS" "$PRECOMMIT_RESULTS" "$CDK_ENUM_RESULTS" "$PYTHON_RELEASE_RESULTS" "$RUBY_RELEASE_RESULTS" "$RUNNER_IMAGE_RESULTS" "$RUNNER_IMAGE_NOTES" "$BEDROCK_MODEL_RESULTS" "$CI_TOOLING_RESULTS" "$CONSISTENCY_RESULTS" "$EPOCH_RESULTS" "$SUPPRESSION_RESULTS" "$LOCKFILE_RESULTS" "$ACCELERATOR_OFFLINE_REPORT" "$ACCELERATOR_ONLINE_REPORT" "$ACCELERATOR_ONLINE_SUMMARY" "$ACCELERATOR_OFFLINE_ERROR" "$ACCELERATOR_ONLINE_ERROR" "$INCOMPLETE_REASONS_FILE"
2759
276019if [ -n "${GITHUB_OUTPUT:-}" ]; then
2761 {
276219 echo "has_drift=true"
276319 echo "scan_complete=$SCAN_COMPLETE"
276419 echo "report_path=$REPORT_FILE"
2765 } >> "$GITHUB_OUTPUT"
2766fi
2767
2768# Mirror the report into the workflow run's job summary so results are visible
2769# on the Actions run page even for workflow_dispatch runs and regardless of
2770# whether an issue is opened.
277119if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
277219 cat "$REPORT_FILE" >> "$GITHUB_STEP_SUMMARY"
2773fi
2774
277519echo ""
277619echo "Wrote report to $REPORT_FILE"