← all scripts

demo/lib_demo.sh

488 of 488 statements covered (100.00%).

coveredmissednever traced by Bash (not counted)A line ending in … continues the statement above it and shares its fate.

112#!/usr/bin/env bash
2# ─────────────────────────────────────────────────────────────────────────────
3# Shared library for GCO demo scripts
4# ─────────────────────────────────────────────────────────────────────────────
5# Sourced by live_demo.sh and record_demo.sh. Also sourced by BATS tests
6# so the tests exercise the real functions, not duplicated copies.
7#
8# Usage:
9# source demo/lib_demo.sh
10#
11# shellcheck disable=SC2034 # Variables are used by sourcing scripts
12# ─────────────────────────────────────────────────────────────────────────────
13
14# ── Colors & Formatting ─────────────────────────────────────────────────────
15# Uses tput for portability. Falls back to empty strings when there's no
16# terminal or tput isn't available.
17
1871setup_colors() {
19255 if [ -t 1 ] && command -v tput &>/dev/null && [ "${TERM:-dumb}" != "dumb" ]; then
202 BOLD=$(tput bold)
212 DIM=$(tput dim)
222 RESET=$(tput sgr0)
232 CYAN=$(tput setaf 6)
242 GREEN=$(tput setaf 2)
252 YELLOW=$(tput setaf 3)
262 MAGENTA=$(tput setaf 5)
272 BLUE=$(tput setaf 4)
282 WHITE=$(tput setaf 7)
292 RED=$(tput setaf 1)
302 BG_BLUE=$(tput setab 4)
31 else
321157 BOLD="" DIM="" RESET="" CYAN="" GREEN="" YELLOW=""
33976 MAGENTA="" BLUE="" WHITE="" RED="" BG_BLUE=""
34 fi
35}
36
37# ── Pause Durations ─────────────────────────────────────────────────────────
38# GCO_DEMO_FAST=1 shortens pauses for rehearsal or recording.
39
402setup_pauses() {
4141 PAUSE_SHORT="${GCO_DEMO_FAST:+1}"
4241 PAUSE_SHORT="${PAUSE_SHORT:-3}"
4341 PAUSE_LONG="${GCO_DEMO_FAST:+2}"
4441 PAUSE_LONG="${PAUSE_LONG:-5}"
45}
46
47# ── Display Helpers ──────────────────────────────────────────────────────────
48
491banner() {
50 # Use terminal width if available, otherwise default to 72.
51 # This ensures the banner fills the recording frame nicely.
5271 local width
53142 width=$(tput cols 2>/dev/null || echo "72")
54 # Cap at 120 to avoid absurdly wide banners on ultrawide terminals
55108 if [ "$width" -gt 120 ]; then width=120; fi
5671 local text="$1"
5771 local text_len=${#text}
5871 local pad_left=$(( (width - text_len) / 2 ))
5971 local pad_right=$(( width - text_len - pad_left ))
6071 echo ""
6171 printf "%s%s%s%*s%s\n" "$BG_BLUE" "$WHITE" "$BOLD" "$width" "" "$RESET"
6271 printf "%s%s%s%*s%s%*s%s\n" "$BG_BLUE" "$WHITE" "$BOLD" "$pad_left" "" "$text" "$pad_right" "" "$RESET"
6371 printf "%s%s%s%*s%s\n" "$BG_BLUE" "$WHITE" "$BOLD" "$width" "" "$RESET"
6471 echo ""
65}
66
671section_header() {
68160 local num="$1"
69160 local title="$2"
70160 local color="${3:-$CYAN}"
71 # Build a divider line that fills the terminal width (capped at 120)
72160 local width
73320 width=$(tput cols 2>/dev/null || echo "72")
74281 if [ "$width" -gt 120 ]; then width=120; fi
75160 local divider
76480 divider=$(printf '%*s' "$width" '' | tr ' ' '━')
77160 echo ""
78160 echo "${color}${BOLD}${divider}${RESET}"
79160 echo "${color}${BOLD} [$num] $title${RESET}"
80160 echo "${color}${BOLD}${divider}${RESET}"
81160 echo ""
82}
83
841252narrate() { echo " ${DIM}$1${RESET}"; }
85491highlight() { echo " ${YELLOW}${BOLD}▸ $1${RESET}"; }
86260success() { echo " ${GREEN}${BOLD}✓ $1${RESET}"; }
8756warn() { echo " ${RED}${BOLD}⚠ $1${RESET}"; }
88679spacer() { echo ""; }
89
902feature_status() {
91252 local value="$1"
92252 if [ "$value" = "true" ]; then
9383 echo "${GREEN}enabled${RESET}"
94 else
95169 echo "${DIM}disabled${RESET}"
96 fi
97}
98
99run_cmd() {
100473 echo ""
101473 echo " ${MAGENTA}\$ ${WHITE}${BOLD}$1${RESET}"
102473 echo " ${DIM}────────────────────────────────────────────────────────────${RESET}"
1031441 eval "$1" 2>&1 | sed 's/^/ /'
104473 local exit_code=${PIPESTATUS[0]}
105473 echo " ${DIM}────────────────────────────────────────────────────────────${RESET}"
106473 if [ "$exit_code" -ne 0 ]; then
10719 warn "Command exited with code $exit_code"
108 fi
109473 return "$exit_code"
110}
111
112# wait_for_inference_generation <endpoint> [attempts] [delay_seconds]
113#
114# Kubernetes readiness proves only the pod-local health endpoint. Before the
115# visible demo prompt, send a tiny real completion through the same unpinned
116# global API route narrated by live_demo.sh. This deliberately retries at the
117# workflow level: the generic client never replays POST automatically.
118wait_for_inference_generation() {
11921 local endpoint="$1"
12021 local attempts="${2:-4}"
12121 local delay_seconds="${3:-10}"
12221 local attempt
123
12448 for attempt in $(seq 1 "$attempts"); do
12527 if gco inference invoke "$endpoint" \
126 -p 'Reply with ready.' --max-tokens 1 >/dev/null 2>&1; then
12719 return 0
128 fi
1298 if [ "$attempt" -lt "$attempts" ]; then
1306 narrate "The global inference route is still converging; retrying in ${delay_seconds}s..."
1316 sleep "$delay_seconds"
132 fi
133 done
1342 return 1
135}
136
137# cleanup_inference_endpoint <endpoint>
138#
139# Best-effort fallback for ambiguous deployment failures and abnormal exits.
140# Preserve the caller's original status, but never hide a possible GPU leak.
141cleanup_inference_endpoint() {
14212 local endpoint="$1"
14312 if gco inference delete "$endpoint" -y >/dev/null 2>&1; then
1449 return 0
145 fi
1463 echo "WARNING: inference endpoint '${endpoint}' may still be running." >&2
1473 echo "Run: gco inference delete ${endpoint} -y" >&2
1483 return 1
149}
150
151# report_inference_lifecycle_result <invoke_ok> <delete_ok>
152#
153# Print the full-lifecycle claim only when generation and normal cleanup both
154# succeeded. Callers propagate a nonzero result to the recorder.
155report_inference_lifecycle_result() {
15623 local invoke_ok="$1"
15723 local delete_ok="$2"
15823 if [ "$invoke_ok" -ne 1 ]; then
1593 warn "Inference generation failed; refusing to publish a false-success recording."
1603 return 1
161 fi
16220 if [ "$delete_ok" -ne 1 ]; then
1632 warn "Inference cleanup failed; refusing to publish an incomplete lifecycle recording."
1642 return 1
165 fi
16618 success "Endpoint deployed, invoked, and torn down — full lifecycle."
167}
168
169# report_feature_result <submitted_ok> <feature label> <success claim>
170#
171# The same fail-closed rule as report_inference_lifecycle_result, applied to the
172# optional feature sections. Their commands are individually forgiving —
173# `submit-direct ... || true` keeps a presentation moving, `wait_for_job` always
174# returns 0, and the kubectl reads end in `|| echo '(pod scheduling...)'` — so
175# without this the section printed a green claim about FSx throughput or Valkey
176# caching even when nothing was ever submitted. In a published recording an
177# unearned claim is worse than a missing section.
178#
179# A feature can be absent for two reasons that look identical here: it was named
180# in GCO_DEMO_ENABLE but never deployed, or it is deployed and genuinely broken.
181# Both make the claim false, so both fail.
182#
183# Returns nonzero on failure; guarded recordings propagate that and publish
184# nothing. Live presentations degrade to a warning and keep going.
185report_feature_result() {
18647 local submitted_ok="$1"
18747 local label="$2"
18847 local claim="$3"
18947 if [ "$submitted_ok" -eq 1 ]; then
19032 success "$claim"
19132 return 0
192 fi
19315 warn "${label} did not run: its workload could not be submitted."
19415 narrate "Verify ${label} is actually deployed — a section enabled through"
19515 narrate "GCO_DEMO_ENABLE still needs the matching 'deploy-all --enable'."
19615 if [ "${GCO_DEMO_GUARDED_RECORDING:-}" = "1" ]; then
1978 warn "Refusing to publish a recording that claims an unproven feature."
1988 return 1
199 fi
2007 return 0
201}
202
203pause_for_audience() {
204181 if [ "${GCO_DEMO_NONINTERACTIVE:-}" = "1" ]; then
205178 sleep 1
206178 return
207 fi
2083 echo ""
2093 echo " ${DIM}Press Enter to continue...${RESET}"
2103 read -r
211}
212
213countdown() {
21479 local msg="$1"
21579 local secs="$2"
2161000 for i in $(seq "$secs" -1 1); do
217921 printf "\r %s%s %d...%s" "$DIM" "$msg" "$i" "$RESET"
218921 sleep 1
219 done
22079 printf "\r %s%-60s%s\n" "$DIM" "$msg done." "$RESET"
221}
222
223# wait_for_job <job-name> <namespace> [timeout_seconds]
224#
225# Waits for a Kubernetes Job to reach the ``complete`` condition, showing a
226# live-updating progress indicator until it succeeds, fails, or times out.
227# Designed for the live demo where we want the audience to actually see the
228# job's final logs — a fixed-duration ``sleep`` used to fall short when
229# image pulls or node provisioning pushed completion past the window.
230#
231# Arguments:
232# job-name Name of the ``batch/v1`` Job resource.
233# namespace Namespace containing the job.
234# timeout_seconds Optional wall-clock budget (default: 240). This is a
235# deadline, not a target — if the job finishes sooner
236# we return immediately. The budget deliberately does
237# not shrink in ``GCO_DEMO_FAST=1`` mode: that flag is
238# for narration pauses, not real work.
239#
240# The helper *always* returns 0, even on timeout or failure. Callers are
241# running under ``set -euo pipefail`` and we don't want a slow job to kill
242# the entire recording mid-demo — the next ``kubectl logs`` / ``kubectl
243# get`` call will surface the state naturally. On timeout we print the pod
244# status so the next narration makes sense instead of showing a blank log
245# block.
246wait_for_job() {
24755 local job="$1"
24855 local ns="$2"
24955 local budget="${3:-240}"
250 # NOTE: GCO_DEMO_FAST is for narration pauses, not for real work. Jobs
251 # still need as long as they need. If the caller explicitly passes a
252 # smaller budget via $3, that wins.
253
25455 local start=$SECONDS
25555 local deadline=$((start + budget))
256
257 # First tick: the Job resource itself may not have appeared in the API
258 # yet (submit-direct returns before the apply is persisted across the
259 # control plane on a cold cluster). Spin briefly until it does.
26056 while [ "$SECONDS" -lt "$deadline" ]; do
26156 if kubectl get "job/${job}" -n "$ns" >/dev/null 2>&1; then
26255 break
263 fi
2641 printf "\r %sWaiting for job/%s to register...%s" "$DIM" "$job" "$RESET"
2651 sleep 1
266 done
267
268 # Use kubectl's own wait primitive for the remainder of the budget. It
269 # returns immediately once the condition is met, so this is both faster
270 # than polling and more accurate than a fixed sleep.
27155 local remaining=$((deadline - SECONDS))
27257 if [ "$remaining" -lt 5 ]; then remaining=5; fi
273
27455 printf "\r %sWaiting for job/%s to complete (up to %ds)...%s\n" \
275 "$DIM" "$job" "$remaining" "$RESET"
276
27755 if kubectl wait --for=condition=complete "job/${job}" \
278 -n "$ns" --timeout="${remaining}s" >/dev/null 2>&1; then
27954 local elapsed=$((SECONDS - start))
28054 printf " %s${GREEN}${BOLD}✓${RESET} %sjob/%s completed in %ds%s\n" \
281 "" "$DIM" "$job" "$elapsed" "$RESET"
28254 return 0
283 fi
284
285 # Timed out or job failed — show what the pod is doing so the audience
286 # sees meaningful context before we hit ``kubectl logs`` on a non-ready
287 # pod. We always return 0 so ``set -e`` callers don't die on a slow job.
2881 printf " %s${YELLOW}${BOLD}!${RESET} %sjob/%s still running after %ds — showing latest pod status%s\n" \
289 "" "$DIM" "$job" "$budget" "$RESET"
2901 kubectl get pods -n "$ns" \
2911 -l "job-name=${job}" --no-headers 2>/dev/null | sed 's/^/ /' || true
2921 return 0
293}
294
295# ── Feature Detection ────────────────────────────────────────────────────────
296# Reads cdk.json and sets global variables for each feature flag.
297# Requires jq and CDK_JSON to be set.
298
299# demo_feature_forced <name>
300#
301# True when GCO_DEMO_ENABLE names this feature or chart. The variable carries
302# the same comma-separated names as `gco stacks deploy-all --enable`, which is
303# the whole point: the recorders derive both values from one knob, so a session
304# can never deploy a feature and then skip demonstrating it (or narrate a
305# feature it never deployed).
306#
307# GCO ships every optional add-on disabled in cdk.json because each carries
308# recurring cost. Recording a full-topology demo therefore needs a run-scoped
309# override rather than a committed config change — see docs/CUSTOMIZATION.md
310# (Run-scoped enablement overrides).
311#
312# Names with no demo section (keda, cert_manager, ...) are accepted and simply
313# have no effect here; they still reach the deploy. A typo is caught by the
314# deploy itself, which validates `--enable` against the canonical name sets in
315# gco/enablement_overrides.py before making any AWS call.
31676demo_feature_forced() {
317421 local wanted="$1"
318421 local requested
3191263 requested=$(printf '%s' "${GCO_DEMO_ENABLE:-}" | tr -d '[:space:]')
320421 case ",${requested}," in
32127 *",${wanted},"*) return 0 ;;
322394 *) return 1 ;;
323 esac
324}
325
326# verify_enablement_overrides <repo_root>
327#
328# Validates GCO_DEMO_ENABLE against the canonical name sets in
329# gco/enablement_overrides.py, which is the same authority `gco stacks
330# deploy-all --enable` validates against.
331#
332# The deploy and destroy recorders get this for free because the CLI rejects an
333# unknown --enable name before any AWS call. The live-demo recorder does not:
334# it only *reads* the variable for section detection, so an unnoticed typo
335# would silently skip the very section the operator set out to record — after
336# the deploy already ran. Checking here keeps that failure loud and early.
337#
338# No-op when unset or empty, so unguarded default recordings are unaffected.
339verify_enablement_overrides() {
34041 local repo_root="$1"
34141 local requested="${GCO_DEMO_ENABLE:-}"
34241 if [ -z "$requested" ]; then
34329 return 0
344 fi
345 # Distinguish "cannot check" from "check failed", so a broken interpreter is
346 # not reported to the operator as an invalid feature name.
34712 if ! command -v python3 >/dev/null 2>&1; then
3484 echo "python3 is required to validate GCO_DEMO_ENABLE." >&2
3494 return 2
350 fi
351 # The Python body is deliberately flush-left: it lives inside a quoted
352 # shell string, so any indentation would reach the interpreter verbatim and
353 # raise IndentationError. Errors are reported as one line rather than a
354 # traceback, because this surfaces inside preflight output.
355 (
3568 cd "$repo_root" || exit 1
3578 python3 -c '
358import sys
359
360from gco.enablement_overrides import EnablementOverrideError, route_enablement_overrides
361
362try:
363 route_enablement_overrides(sys.argv[1:])
364except EnablementOverrideError as exc:
365 sys.exit(str(exc))
366' "$requested"
367 )
368}
369
3709detect_features() {
37151 local cdk="${1:-cdk.json}"
372102 VOLCANO_ENABLED=$(jq -r '.context.helm.volcano.enabled // false' "$cdk")
373102 KUEUE_ENABLED=$(jq -r '.context.helm.kueue.enabled // false' "$cdk")
374102 YUNIKORN_ENABLED=$(jq -r '.context.helm.yunikorn.enabled // false' "$cdk")
375102 SLURM_ENABLED=$(jq -r '.context.helm.slurm.enabled // false' "$cdk")
376102 FSX_ENABLED=$(jq -r '.context.fsx_lustre.enabled // false' "$cdk")
377102 VALKEY_ENABLED=$(jq -r '.context.valkey.enabled // false' "$cdk")
378102 AURORA_PGVECTOR_ENABLED=$(jq -r '.context.aurora_pgvector.enabled // false' "$cdk")
379102 VECTOR_STORE_ENABLED=$(jq -r '.context.vector_store.enabled // false' "$cdk")
380
381 # Overrides are one-way, matching the CDK context semantics: they can only
382 # turn a feature on, never off. A feature an operator disabled stays
383 # disabled unless it is named explicitly.
38451 if demo_feature_forced volcano; then VOLCANO_ENABLED=true; fi
38551 if demo_feature_forced kueue; then KUEUE_ENABLED=true; fi
38653 if demo_feature_forced yunikorn; then YUNIKORN_ENABLED=true; fi
38753 if demo_feature_forced slurm; then SLURM_ENABLED=true; fi
38855 if demo_feature_forced fsx_lustre; then FSX_ENABLED=true; fi
38956 if demo_feature_forced valkey; then VALKEY_ENABLED=true; fi
39053 if demo_feature_forced aurora_pgvector; then AURORA_PGVECTOR_ENABLED=true; fi
39155 if demo_feature_forced vector_store; then VECTOR_STORE_ENABLED=true; fi
392}
393
3942detect_region() {
395110 local cdk="${1:-cdk.json}"
396213 REGION="${GCO_DEMO_REGION:-$(jq -r '.context.deployment_regions.regional[0] // "us-east-1"' "$cdk")}"
397}
398
3991detect_endpoint_access() {
40035 local cdk="${1:-cdk.json}"
40170 ENDPOINT_ACCESS=$(jq -r '.context.eks_cluster.endpoint_access // "PRIVATE"' "$cdk")
402}
403
404# ── Section Counter ──────────────────────────────────────────────────────────
405
406# Section counter — can't use $(next_section) because command substitution
407# runs in a subshell and the counter increment is lost. Instead we increment
408# inline and use the variable directly.
409472SECTION=0
410
411# ── ARN Helpers (shared with setup-cluster-access.sh) ────────────────────────
412
413# Checks if an ARN is an assumed-role ARN.
4146is_assumed_role() {
41514 [[ "$1" == *":assumed-role/"* ]]
416}
417
418# Extracts the role name from an assumed-role ARN.
419# Input: arn:aws:sts::123456789012:assumed-role/MyRole/session-name
420# Output: MyRole
4214extract_role_name() {
42218 echo "$1" | sed 's/.*:assumed-role\/\([^\/]*\)\/.*/\1/'
423}
424
425# Reconstructs an IAM role ARN from an assumed-role ARN and account ID.
426# Input: role_name, account_id
427# Output: arn:aws:iam::123456789012:role/MyRole
4282build_role_arn() {
4295 local role_name="$1"
4305 local account_id="$2"
4315 echo "arn:aws:iam::${account_id}:role/${role_name}"
432}
433
434# ── Recording Helpers ────────────────────────────────────────────────────────
435
436# Default font family used when rendering .cast files to GIFs with agg.
437#
438# agg's text renderer (resvg/usvg) is first-family-wins — it does not do
439# per-glyph fallback down the family list like a GUI text engine would. So
440# Menlo is kept first because it covers the characters our scripts emit
441# (box-drawing, arrows, geometric shapes, and the dingbats ✓ ✗ ⚠ ▸). Any
442# codepoint Menlo doesn't have (typically color-emoji pictographs from CDK
443# output like ✨ and ✅, or the information symbol ℹ) would otherwise fall
444# through to ``.LastResort`` and render as a tofu box.
445#
446# We fix that upstream instead of with more font fallbacks: every cast file
447# runs through ``strip_emoji_from_cast`` before ``render_gif``, which maps
448# the known tofu-triggering codepoints to safe monochrome equivalents.
449# After that pass, Menlo covers every character in the cast and agg never
450# needs to fall back.
451#
452# Override via the DEMO_FONT_FAMILY environment variable if you need to
453# skip this substitution and use a font that has real coverage of those
454# codepoints (e.g. a full Unicode monospace font).
455472DEMO_FONT_FAMILY_DEFAULT="Menlo,Monaco,Courier New"
456
457# verify_recording_git_state <repo_root> [allowed_dirty_path ...]
458#
459# When GCO_EXPECTED_GIT_SHA is set, verifies that the checkout is exactly that
460# full commit and rejects every dirty/untracked path except the explicitly
461# supplied recorder outputs. The output allowlist lets a destroy recording
462# follow a deploy recording before the four generated artifacts are committed,
463# while still proving that the source tree matches the CI-green commit.
464verify_recording_git_state() {
46542 local repo_root="$1"
46642 shift
46742 local expected="${GCO_EXPECTED_GIT_SHA:-}"
46842 if [ -z "$expected" ]; then
4691 return 0
470 fi
47181 if [ "${#expected}" -ne 40 ] || [[ "$expected" == *[!0-9a-fA-F]* ]]; then
4722 echo "GCO_EXPECTED_GIT_SHA must be a full 40-character hexadecimal SHA." >&2
4732 return 1
474 fi
475
47639 local actual
47778 if ! actual=$(git -C "$repo_root" rev-parse HEAD 2>/dev/null); then
4782 echo "Unable to resolve git HEAD in $repo_root." >&2
4792 return 1
480 fi
48137 local expected_normalized actual_normalized
482111 expected_normalized=$(printf '%s' "$expected" | tr '[:upper:]' '[:lower:]')
483111 actual_normalized=$(printf '%s' "$actual" | tr '[:upper:]' '[:lower:]')
48437 if [ "$actual_normalized" != "$expected_normalized" ]; then
4851 echo "Git HEAD does not match GCO_EXPECTED_GIT_SHA." >&2
4861 return 1
487 fi
488
48936 local dirty
49072 if ! dirty=$(git -C "$repo_root" status --porcelain=v1 --untracked-files=all); then
4911 echo "Unable to inspect git worktree state in $repo_root." >&2
4921 return 1
493 fi
494
49535 local line path candidate is_allowed rename_source rename_destination
49635 local unexpected=""
497146 while IFS= read -r line; do
49869 [ -n "$line" ] || continue
4997 path="${line:3}"
5007 rename_source=""
5017 rename_destination="$path"
5027 case "$path" in
503 *" -> "*)
5041 rename_source="${path%% -> *}"
5051 rename_destination="${path##* -> }"
506 ;;
507 esac
508 # Porcelain-v1 renders renames as ``source -> destination``. Both
509 # paths must be allowlisted: accepting only the destination would let
510 # a recorder delete or move arbitrary tracked source files.
51114 for path in "$rename_source" "$rename_destination"; do
51220 [ -n "$path" ] || continue
5138 is_allowed=0
51424 for candidate in "$@"; do
51524 if [ "$path" = "$candidate" ]; then
5165 is_allowed=1
5175 break
518 fi
519 done
5208 if [ "$is_allowed" -ne 1 ]; then
5213 unexpected="${unexpected}${unexpected:+, }${path}"
522 fi
523 done
524 done <<< "$dirty"
525
52635 if [ -n "$unexpected" ]; then
5273 echo "Unexpected dirty paths for guarded recording: $unexpected" >&2
5283 return 1
529 fi
530}
531
532# verify_recording_aws_account
533#
534# When GCO_EXPECTED_ACCOUNT_ID is set, resolves the active caller through STS
535# and requires an exact match before a recorder can mutate infrastructure.
536verify_recording_aws_account() {
53736 local expected="${GCO_EXPECTED_ACCOUNT_ID:-}"
53836 if [ -z "$expected" ]; then
5391 return 0
540 fi
54135 if ! [[ "$expected" =~ ^[0-9]{12}$ ]]; then
5421 echo "GCO_EXPECTED_ACCOUNT_ID must contain exactly 12 digits." >&2
5431 return 1
544 fi
545
54634 local actual
54768 if ! actual=$(aws sts get-caller-identity --query Account --output text 2>/dev/null); then
5481 echo "Unable to resolve the active AWS account through STS." >&2
5491 return 1
550 fi
55133 if [ "$actual" != "$expected" ]; then
5521 echo "Active AWS account does not match GCO_EXPECTED_ACCOUNT_ID." >&2
5531 return 1
554 fi
555}
556
557# verify_legacy_live_recording_authorization <repo_root>
558#
559# Fail-closed gate for the three publishable legacy recordings. A caller must
560# explicitly acknowledge live mutations and bind the session to one reviewed
561# commit and one authorized AWS account. The six generated legacy artifacts
562# may be dirty so deploy, live-demo, and destroy can be captured sequentially
563# from the same checkout; every source or Autopilot path must remain clean.
564verify_legacy_live_recording_authorization() {
56541 local repo_root="$1"
56641 if [ "${GCO_RECORDING_LIVE:-}" != "1" ]; then
5674 echo "Set GCO_RECORDING_LIVE=1 to acknowledge live AWS/Kubernetes mutations." >&2
5684 return 1
569 fi
57037 if [ -z "${GCO_EXPECTED_GIT_SHA:-}" ]; then
5711 echo "GCO_EXPECTED_GIT_SHA is required for a live legacy recording." >&2
5721 return 1
573 fi
57436 if [ -z "${GCO_EXPECTED_ACCOUNT_ID:-}" ]; then
5751 echo "GCO_EXPECTED_ACCOUNT_ID is required for a live legacy recording." >&2
5761 return 1
577 fi
57835 verify_recording_git_state "$repo_root" \
579 "demo/deploy.cast" "demo/deploy.gif" \
580 "demo/live_demo.cast" "demo/live_demo.gif" \
5813 "demo/destroy.cast" "demo/destroy.gif" || return 1
58232 verify_recording_aws_account || return 1
583}
584
585# verify_recording_kube_context <cluster-name> <region>
586#
587# Bind the active kubectl context to the EKS cluster resolved through the same
588# AWS identity that passed the account guard. Exact endpoint comparison prevents
589# namespace-wide cleanup from reaching an unrelated but otherwise healthy
590# cluster in another account or context.
591verify_recording_kube_context() {
59245 local cluster_name="$1"
59345 local region="$2"
59445 local expected_endpoint current_endpoint
59590 if ! expected_endpoint=$(aws eks describe-cluster \
596 --name "$cluster_name" \
597 --region "$region" \
598 --query 'cluster.endpoint' \
599 --output text 2>/dev/null); then
6001 echo "Unable to resolve the expected EKS endpoint for recording." >&2
6011 return 1
602 fi
60388 if ! current_endpoint=$(kubectl config view --minify \
604 -o 'jsonpath={.clusters[0].cluster.server}' 2>/dev/null); then
6051 echo "Unable to resolve the active kubectl server." >&2
6061 return 1
607 fi
60843 expected_endpoint="${expected_endpoint%/}"
60943 current_endpoint="${current_endpoint%/}"
61086 if [ -z "$expected_endpoint" ] || [ "$expected_endpoint" = "None" ] || \
611 [ "$current_endpoint" != "$expected_endpoint" ]; then
6125 echo "Active kubectl context does not match the authorized GCO EKS cluster." >&2
6135 return 1
614 fi
615}
616
617# A fixed hard-link beneath Git's common directory serializes all legacy
618# recorders across linked worktrees without dirtying any checkout. Each process
619# prepares a private owner file before atomically linking it into the fixed lock
620# path. Registering both paths first lets handled signals clean up safely before,
621# during, or immediately after acquisition without touching another owner. A
622# SIGKILL can still leave the lock fail-closed for operator inspection.
623472LEGACY_RECORDING_LOCK_FILE=""
624472LEGACY_RECORDING_LOCK_OWNER_FILE=""
625
6264acquire_legacy_recording_lock() {
62752 local repo_root="$1"
62852 local git_common
629104 if ! git_common=$(git -C "$repo_root" rev-parse --git-common-dir 2>/dev/null); then
6301 echo "Unable to resolve the Git common directory for recording lock." >&2
6311 return 1
632 fi
63351 case "$git_common" in
634 /*) ;;
63551 *) git_common="${repo_root}/${git_common}" ;;
636 esac
637152 if ! git_common=$(cd "$git_common" 2>/dev/null && pwd -P); then
6381 echo "Unable to canonicalize the Git common directory for recording lock." >&2
6391 return 1
640 fi
641
64250 local lock_file="${git_common}/gco-legacy-recording.lock"
64350 local owner_file="${lock_file}.owner.${BASHPID:-$$}.${RANDOM}"
64450 LEGACY_RECORDING_LOCK_FILE="$lock_file"
64550 LEGACY_RECORDING_LOCK_OWNER_FILE="$owner_file"
646
647150 if ! (umask 077; set -o noclobber; printf 'pid=%s\nrepo=%s\n' \
648 "$$" "$repo_root" > "$owner_file") 2>/dev/null; then
6493 LEGACY_RECORDING_LOCK_FILE=""
6503 LEGACY_RECORDING_LOCK_OWNER_FILE=""
6513 echo "Unable to create recording lock owner file: ${owner_file}." >&2
6523 return 1
653 fi
65447 if ! ln "$owner_file" "$lock_file" 2>/dev/null; then
6551 rm -f -- "$owner_file" || true
6561 LEGACY_RECORDING_LOCK_FILE=""
6571 LEGACY_RECORDING_LOCK_OWNER_FILE=""
6581 echo "Another legacy demo recorder holds ${lock_file}." >&2
6591 return 1
660 fi
661}
662
6631release_legacy_recording_lock() {
66466 local lock_file="${LEGACY_RECORDING_LOCK_FILE:-}"
66566 local owner_file="${LEGACY_RECORDING_LOCK_OWNER_FILE:-}"
666110 if [ -z "$lock_file" ] || [ -z "$owner_file" ]; then
66722 return 0
668 fi
669
67085 if [ -e "$lock_file" ] && [ -e "$owner_file" ] && \
671 [ "$owner_file" -ef "$lock_file" ]; then
67241 if ! rm -f -- "$lock_file"; then
6734 echo "Unable to release legacy recording lock: ${lock_file}" >&2
6744 return 1
675 fi
676 fi
67780 if [ -e "$owner_file" ] && ! rm -f -- "$owner_file"; then
6781 echo "Unable to remove recording lock owner file: ${owner_file}" >&2
6791 return 1
680 fi
68139 LEGACY_RECORDING_LOCK_FILE=""
68239 LEGACY_RECORDING_LOCK_OWNER_FILE=""
683}
684
685# sanitize_cast <cast_file>
686#
687# Redacts AWS account IDs and temporary/long-lived AWS access-key IDs from an
688# asciinema recording. Account-ID-shaped values become 000000000000 and
689# AKIA/ASIA access-key IDs become REDACTED_AWS_ACCESS_KEY_ID. Operates in place.
690#
691# The account heuristic is intentionally broad: unrelated standalone 12-digit
692# values are also redacted. Over-redaction is safer than allowing an account ID
693# into a committed cast or the GIF rendered from it.
694#
695# Use SKIP_SANITIZE=1 only to debug a local recording. Bypassed artifacts must
696# never be committed or distributed.
6977sanitize_cast() {
69846 local cast_file="$1"
69946 if [ "${SKIP_SANITIZE:-}" = "1" ]; then
7002 return
701 fi
70244 if [ ! -f "$cast_file" ]; then
7031 return
704 fi
705
70643 python3 - "$cast_file" <<'PYEOF'
707import json
708import re
709import sys
710from pathlib import Path
711
712ACCOUNT_ID = re.compile(r"(?<![0-9])[0-9]{12}(?![0-9])")
713ACCESS_KEY_ID = re.compile(r"(?<![A-Z0-9])(?:AKIA|ASIA)[A-Z0-9]{16}(?![A-Z0-9])")
714PATTERNS = (
715 (ACCOUNT_ID, "000000000000"),
716 (ACCESS_KEY_ID, "REDACTED_AWS_ACCESS_KEY_ID"),
717)
718
719
720def redactions(text):
721 edits = [
722 (match.start(), match.end(), replacement)
723 for pattern, replacement in PATTERNS
724 for match in pattern.finditer(text)
725 ]
726 edits.sort(key=lambda edit: (edit[0], edit[1]))
727 return edits
728
729
730def redact_text(text):
731 edits = redactions(text)
732 parts = []
733 cursor = 0
734 for start, end, replacement in edits:
735 if start < cursor:
736 continue
737 parts.extend((text[cursor:start], replacement))
738 cursor = end
739 parts.append(text[cursor:])
740 return "".join(parts)
741
742
743def redact_value(value):
744 if isinstance(value, str):
745 return redact_text(value)
746 if isinstance(value, list):
747 return [redact_value(item) for item in value]
748 if isinstance(value, dict):
749 return {key: redact_value(item) for key, item in value.items()}
750 return value
751
752
753def mapped_offset(position, edits):
754 source_cursor = 0
755 output_cursor = 0
756 for start, end, replacement in edits:
757 if position <= start:
758 return output_cursor + position - source_cursor
759 output_cursor += start - source_cursor
760 if position < end:
761 # Assign a replacement spanning event boundaries to the event in
762 # which the sensitive value began; later fragments become empty.
763 return output_cursor + len(replacement)
764 output_cursor += len(replacement)
765 source_cursor = end
766 return output_cursor + position - source_cursor
767
768
769path = Path(sys.argv[1])
770documents = []
771for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
772 if not line.strip():
773 raise ValueError(f"blank line {line_number} is not valid cast NDJSON")
774 documents.append(json.loads(line))
775
776output_events = []
777for index, document in enumerate(documents):
778 if isinstance(document, list) and len(document) >= 3 and document[1] == "o":
779 if not isinstance(document[2], str):
780 raise ValueError(f"output event {index + 1} has a non-string payload")
781 output_events.append(document)
782 else:
783 documents[index] = redact_value(document)
784
785# Redact each payload first so a complete identifier cannot be hidden by
786# adjacent event content that turns it into one longer alphanumeric run.
787for event in output_events:
788 event[2] = redact_text(event[2])
789
790# Then redact the concatenated rendered terminal so identifiers split by
791# asciinema's event boundaries cannot evade either pattern.
792source = "".join(event[2] for event in output_events)
793edits = redactions(source)
794rendered = redact_text(source)
795offset = 0
796for event in output_events:
797 start = offset
798 offset += len(event[2])
799 event[2] = rendered[mapped_offset(start, edits):mapped_offset(offset, edits)]
800
801serialized = "\n".join(
802 json.dumps(document, ensure_ascii=False, separators=(",", ":"))
803 for document in documents
804)
805path.write_text(serialized + ("\n" if documents else ""), encoding="utf-8")
806PYEOF
80743}
80843
80943# verify_cast_sanitized <cast_file>
81043#
81143# Independently verifies the sanitizer's postcondition without printing the
81243# matched values. The all-zero account placeholder is allowed; every other
81343# standalone 12-digit value and every AKIA/ASIA access-key ID fails closed.
8142verify_cast_sanitized() {
81536 local cast_file="$1"
81636 if [ "${SKIP_SANITIZE:-}" = "1" ]; then
81743 return
81843 fi
81936 if [ ! -f "$cast_file" ]; then
82043 echo "Cannot verify missing cast file: $cast_file" >&2
82143 return 1
82243 fi
82343
82443 python3 - "$cast_file" <<'PYEOF'
825import json
826import re
827import sys
828from pathlib import Path
829
830ACCOUNT_ID = re.compile(r"(?<![0-9])[0-9]{12}(?![0-9])")
831ACCESS_KEY_ID = re.compile(r"(?<![A-Z0-9])(?:AKIA|ASIA)[A-Z0-9]{16}(?![A-Z0-9])")
832
833
834def string_values(value):
835 if isinstance(value, str):
836 yield value
837 elif isinstance(value, list):
838 for item in value:
839 yield from string_values(item)
840 elif isinstance(value, dict):
841 for item in value.values():
842 yield from string_values(item)
843
844
845documents = []
846path = Path(sys.argv[1])
847for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
848 if not line.strip():
849 raise ValueError(f"blank line {line_number} is not valid cast NDJSON")
850 documents.append(json.loads(line))
851
852output_payloads = []
853non_output_strings = []
854for index, document in enumerate(documents):
855 if isinstance(document, list) and len(document) >= 3 and document[1] == "o":
856 if not isinstance(document[2], str):
857 raise ValueError(f"output event {index + 1} has a non-string payload")
858 output_payloads.append(document[2])
859 non_output_strings.extend(string_values(document[:2]))
860 non_output_strings.extend(string_values(document[3:]))
861 else:
862 non_output_strings.extend(string_values(document))
863
864# Verify decoded content, not raw JSON serialization. This independently
865# reconstructs the rendered output stream and catches identifiers split across
866# adjacent output events while still checking header/non-output string fields.
867texts = [*output_payloads, "".join(output_payloads), *non_output_strings]
868account_ids = [
869 match.group(0)
870 for text in texts
871 for match in ACCOUNT_ID.finditer(text)
872 if match.group(0) != "000000000000"
873]
874access_key_ids = [
875 match.group(0)
876 for text in texts
877 for match in ACCESS_KEY_ID.finditer(text)
878]
879if account_ids or access_key_ids:
880 print(
881 "Cast sanitization verification failed: "
882 f"{len(account_ids)} account-ID pattern(s), "
883 f"{len(access_key_ids)} access-key-ID pattern(s) remain.",
884 file=sys.stderr,
885 )
886 raise SystemExit(1)
887PYEOF
88843}
88943
89043# strip_emoji_from_cast <cast_file>
89143#
89243# Rewrites tofu-triggering Unicode codepoints in a .cast file to ASCII or
89343# to monochrome glyphs Menlo can render, so agg never falls back to
89443# ``.LastResort`` during GIF conversion.
89543#
89643# Background: agg uses resvg/usvg, a pure-vector text renderer. When the
89743# first font in the family list can't render a glyph, usvg falls back to
89843# ``.LastResort`` (the system tofu font) rather than iterating the family
89943# list. Color emoji fonts like Apple Color Emoji don't help because they're
90043# bitmap (sbix/COLR) fonts, which usvg cannot use.
90143#
90243# This helper runs in-place with Python 3 for portable Unicode handling.
90343# The substitutions:
90443# ℹ (INFORMATION SOURCE, U+2139) → i Menlo has no glyph
90543# ✅ (WHITE HEAVY CHECK MARK, U+2705) → ✓ Menlo has ✓, not ✅
90643# ✨ (SPARKLES, U+2728) → * Menlo has no glyph
90743# 📦 (PACKAGE, U+1F4E6) → [pkg] Menlo has no glyph
90843# 🚀 (ROCKET, U+1F680) → >> Menlo has no glyph
90943#
91043# Use SKIP_EMOJI_STRIP=1 to bypass (useful when you're confident your font
91143# chain renders everything correctly and don't want the substitutions).
9128strip_emoji_from_cast() {
91348 local cast_file="$1"
91448 if [ "${SKIP_EMOJI_STRIP:-}" = "1" ]; then
9152 return
91643 fi
91746 if [ ! -f "$cast_file" ]; then
9181 return
91943 fi
92043 # Python handles Unicode character substitution cleanly across GNU and
92143 # BSD sed variants, and lets us express the character set as a readable
92243 # translation table rather than cramming UTF-8 byte sequences into a
92343 # fragile sed one-liner.
92445 python3 - "$cast_file" <<'PYEOF'
925import sys
926from pathlib import Path
927
928# Single-character substitutions (str.translate with the ord key).
929SINGLE = {
930 0x2139: "i", # ℹ INFORMATION SOURCE → i
931 0x2705: "\u2713", # ✅ WHITE HEAVY CHECK MARK → ✓ (monochrome check, in Menlo)
932 0x2728: "*", # ✨ SPARKLES → *
933}
934
935# Multi-character substitutions applied after the translate pass.
936MULTI = {
937 "\U0001F4E6": "[pkg]", # 📦 PACKAGE
938 "\U0001F680": ">>", # 🚀 ROCKET
939}
940
941path = Path(sys.argv[1])
942text = path.read_text(encoding="utf-8")
943text = text.translate(SINGLE)
944for src, dst in MULTI.items():
945 text = text.replace(src, dst)
946path.write_text(text, encoding="utf-8")
947PYEOF
948}
949
950# render_gif <cast_file> <gif_file> <speed> <theme> <cols> <rows>
951#
952# Converts an asciinema .cast file to an animated GIF using agg with the
953# shared font-family fallback chain. Centralised here so all three
954# record scripts render consistent-looking output.
955#
956# The DEMO_FONT_FAMILY env var overrides the default fallback list.
9574render_gif() {
95830 local cast_file="$1"
95930 local gif_file="$2"
96030 local speed="$3"
96130 local theme="$4"
96230 local cols="$5"
96330 local rows="$6"
96430 local font_family="${DEMO_FONT_FAMILY:-$DEMO_FONT_FAMILY_DEFAULT}"
965
96630 agg \
967 --speed "$speed" \
968 --theme "$theme" \
969 --font-family "$font_family" \
970 --font-size 14 \
971 --cols "$cols" \
972 --rows "$rows" \
973 "$cast_file" \
974 "$gif_file"
975}
976
977# ── Recording Publication Transaction ───────────────────────────────────────
978#
979# A cast and its GIF cannot be switched with one POSIX rename. These globals
980# describe the narrow interval in which one file may have been switched but the
981# other has not. Recorder EXIT/signal cleanup calls rollback_recording_publication
982# so every handled failure restores the complete previous pair (or removes both
983# newly-created outputs when no previous pair existed).
984472RECORDING_PUBLICATION_IN_PROGRESS=0
985472RECORDING_PUBLICATION_COMPLETE=0
986472RECORDING_PUBLICATION_STAGE_DIR=""
987472RECORDING_PUBLICATION_CAST_FILE=""
988472RECORDING_PUBLICATION_GIF_FILE=""
989472RECORDING_PUBLICATION_CAST_BACKUP=""
990472RECORDING_PUBLICATION_GIF_BACKUP=""
991472RECORDING_PUBLICATION_HAD_CAST=0
992472RECORDING_PUBLICATION_HAD_GIF=0
993
994# recording_publication_restore_file <backup> <final> <restore-staging-path>
995#
996# Copies the preserved artifact to another same-filesystem staging path before
997# renaming it over the final. The original backup remains available if this
998# restoration attempt is interrupted or fails and the EXIT trap retries it.
999recording_publication_restore_file() {
100025 local backup_file="$1"
100125 local final_file="$2"
100225 local restore_file="$3"
1003
100425 rm -f "$restore_file"
100525 if ! cp -p "$backup_file" "$restore_file"; then
10069 return 1
1007 fi
100816 if ! mv -f "$restore_file" "$final_file"; then
10097 return 1
1010 fi
1011}
1012
1013# rollback_recording_publication
1014#
1015# Restores both artifacts represented by the active publication transaction.
1016# Backups are copied rather than consumed so an EXIT cleanup can retry after a
1017# partial restoration failure. The transaction remains active until both final
1018# paths match their pre-publication state.
1019rollback_recording_publication() {
102067 if [ "${RECORDING_PUBLICATION_IN_PROGRESS:-0}" != "1" ]; then
102152 return 0
1022 fi
1023
102415 RECORDING_PUBLICATION_COMPLETE=0
102515 local rollback_status=0
1026
102715 if [ "$RECORDING_PUBLICATION_HAD_CAST" = "1" ]; then
102813 if ! recording_publication_restore_file \
1029 "$RECORDING_PUBLICATION_CAST_BACKUP" \
1030 "$RECORDING_PUBLICATION_CAST_FILE" \
1031 "${RECORDING_PUBLICATION_STAGE_DIR}/.restore-cast"; then
10329 rollback_status=1
1033 fi
10342 elif ! rm -f "$RECORDING_PUBLICATION_CAST_FILE"; then
10351 rollback_status=1
1036 fi
1037
103815 if [ "$RECORDING_PUBLICATION_HAD_GIF" = "1" ]; then
103912 if ! recording_publication_restore_file \
1040 "$RECORDING_PUBLICATION_GIF_BACKUP" \
1041 "$RECORDING_PUBLICATION_GIF_FILE" \
1042 "${RECORDING_PUBLICATION_STAGE_DIR}/.restore-gif"; then
10437 rollback_status=1
1044 fi
10453 elif ! rm -f "$RECORDING_PUBLICATION_GIF_FILE"; then
10461 rollback_status=1
1047 fi
1048
104915 if [ "$rollback_status" -eq 0 ]; then
10504 RECORDING_PUBLICATION_IN_PROGRESS=0
1051 fi
105215 return "$rollback_status"
1053}
1054
1055# publish_recording_artifacts <staged-cast> <staged-gif-or-empty> <final-cast> <final-gif>
1056#
1057# Publishes a prepared cast/GIF pair with rollback across the two individually
1058# atomic same-filesystem renames. An empty staged GIF means the final GIF must
1059# be absent (SKIP_GIF mode). Existing finals are preserved before either final
1060# is changed. On any ordinary command failure this helper rolls both paths back;
1061# recorder EXIT/HUP/INT/TERM handling covers interruption between commands.
10622publish_recording_artifacts() {
106345 local staged_cast="$1"
106445 local staged_gif="$2"
106545 local final_cast="$3"
106645 local final_gif="$4"
1067
106845 if [ "${RECORDING_PUBLICATION_IN_PROGRESS:-0}" = "1" ]; then
10691 echo "A recording publication transaction is already active." >&2
10701 return 1
1071 fi
107244 if [ ! -f "$staged_cast" ]; then
10731 echo "Cannot publish missing staged cast: $staged_cast" >&2
10741 return 1
1075 fi
107673 if [ -n "$staged_gif" ] && [ ! -f "$staged_gif" ]; then
10771 echo "Cannot publish missing staged GIF: $staged_gif" >&2
10781 return 1
1079 fi
108084 if [ -d "$final_cast" ] || [ -d "$final_gif" ]; then
10811 echo "Recording publication destinations must be files." >&2
10821 return 1
1083 fi
1084
108541 local stage_dir
108641 local cast_backup
108741 local gif_backup
108841 local had_cast=0
108941 local had_gif=0
109082 stage_dir=$(dirname "$staged_cast")
109141 cast_backup="${stage_dir}/.previous-cast"
109241 gif_backup="${stage_dir}/.previous-gif"
109341 rm -f "$cast_backup" "$gif_backup" \
1094 "${stage_dir}/.restore-cast" "${stage_dir}/.restore-gif"
1095
1096 # Do not mark the transaction active until both required backups are fully
1097 # copied. An interruption during this phase leaves the final paths intact.
109841 if [ -e "$final_cast" ]; then
109939 if ! cp -p "$final_cast" "$cast_backup"; then
11001 return 1
1101 fi
110238 had_cast=1
1103 fi
110440 if [ -e "$final_gif" ]; then
110537 if ! cp -p "$final_gif" "$gif_backup"; then
11061 return 1
1107 fi
110836 had_gif=1
1109 fi
1110
111139 RECORDING_PUBLICATION_STAGE_DIR="$stage_dir"
111239 RECORDING_PUBLICATION_CAST_FILE="$final_cast"
111339 RECORDING_PUBLICATION_GIF_FILE="$final_gif"
111439 RECORDING_PUBLICATION_CAST_BACKUP="$cast_backup"
111539 RECORDING_PUBLICATION_GIF_BACKUP="$gif_backup"
111639 RECORDING_PUBLICATION_HAD_CAST="$had_cast"
111739 RECORDING_PUBLICATION_HAD_GIF="$had_gif"
111839 RECORDING_PUBLICATION_COMPLETE=0
111939 RECORDING_PUBLICATION_IN_PROGRESS=1
1120
112139 local publication_status
112239 if mv -f "$staged_cast" "$final_cast"; then
112337 :
1124 else
11252 publication_status=$?
11262 if ! rollback_recording_publication; then
11271 echo "Recording publication failed and rollback could not complete." >&2
11281 return 1
1129 fi
11301 return "$publication_status"
1131 fi
1132
113337 if [ -n "$staged_gif" ]; then
113426 if mv -f "$staged_gif" "$final_gif"; then
113518 :
1136 else
11378 publication_status=$?
11388 if ! rollback_recording_publication; then
11396 echo "Recording publication failed and rollback could not complete." >&2
11406 return 1
1141 fi
11422 return "$publication_status"
1143 fi
114411 elif rm -f "$final_gif"; then
11459 :
1146 else
11472 publication_status=$?
11482 if ! rollback_recording_publication; then
11491 echo "Recording publication failed and rollback could not complete." >&2
11501 return 1
1151 fi
11521 return "$publication_status"
1153 fi
1154
1155 # Both final-path operations succeeded. If a handled signal arrives before
1156 # IN_PROGRESS is cleared, the active transaction safely restores both old
1157 # paths; after it is cleared, the new pair is already internally consistent.
115827 RECORDING_PUBLICATION_COMPLETE=1
115927 RECORDING_PUBLICATION_IN_PROGRESS=0
1160}