demo/live_demo.sh537 of 537 statements covered (100.00%).
coveredmissednever traced by Bash (not counted)A line ending in … continues the statement above it and shares its fate.
| 1 | 8 | #!/usr/bin/env bash |
| 2 | # ───────────────────────────────────────────────────────────────────────────── | |
| 3 | # GCO Live Feature Demonstration | |
| 4 | # ───────────────────────────────────────────────────────────────────────────── | |
| 5 | # This script is designed to run in a visible terminal during a live | |
| 6 | # presentation. It walks through GCO's core capabilities automatically, | |
| 7 | # with narrated output and pauses between sections so the audience can | |
| 8 | # follow along. | |
| 9 | # | |
| 10 | # The script reads cdk.json to detect which optional features (schedulers, | |
| 11 | # FSx, Valkey) are enabled and only demos what's actually deployed. | |
| 12 | # | |
| 13 | # Usage: | |
| 14 | # bash demo/live_demo.sh # Standard run | |
| 15 | # GCO_DEMO_REGION=us-west-2 bash demo/live_demo.sh # Override region | |
| 16 | # GCO_DEMO_FAST=1 bash demo/live_demo.sh # Shorter pauses | |
| 17 | # SKIP_COSTS=1 bash demo/live_demo.sh # Skip cost section | |
| 18 | # SKIP_SCHEDULERS=1 bash demo/live_demo.sh # Skip scheduler demos | |
| 19 | # | |
| 20 | # See demo/LIVE_DEMO.md for full documentation and maintenance guide. | |
| 21 | # ───────────────────────────────────────────────────────────────────────────── | |
| 22 | ||
| 23 | # Exit immediately on errors, treat unset variables as errors, and propagate | |
| 24 | # failures through pipes (e.g., "cmd | sed" fails if cmd fails). | |
| 25 | 37 | set -euo pipefail |
| 26 | ||
| 27 | # ── Load Shared Library ────────────────────────────────────────────────────── | |
| 28 | # All helper functions (colors, display, feature detection, ARN helpers) live | |
| 29 | # in lib_demo.sh so they can be shared with record_demo.sh and tested | |
| 30 | # directly by BATS without duplication. | |
| 31 | ||
| 32 | 148 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 33 | # shellcheck source=demo/lib_demo.sh | |
| 34 | 37 | source "${SCRIPT_DIR}/lib_demo.sh" |
| 35 | ||
| 36 | # Initialize colors and pause durations. | |
| 37 | 37 | setup_colors |
| 38 | 37 | setup_pauses |
| 39 | ||
| 40 | # Longer wait for pods to pull images and complete (not affected by FAST mode | |
| 41 | # because pods need real time regardless of presentation speed). | |
| 42 | 37 | WAIT_FOR_POD="${GCO_DEMO_FAST:+15}" |
| 43 | 37 | WAIT_FOR_POD="${WAIT_FOR_POD:-30}" |
| 44 | ||
| 45 | # Once an inference deployment is accepted, keep a child-shell EXIT fallback | |
| 46 | # armed until the normal delete succeeds. This prevents a later demo failure or | |
| 47 | # interruption from silently leaving the endpoint running. | |
| 48 | 37 | INFERENCE_CLEANUP_PENDING=0 |
| 49 | cleanup_demo_inference_on_exit() { | |
| 50 | 8 | local exit_code="$1" |
| 51 | 8 | trap - EXIT |
| 52 | 8 | if [ "${INFERENCE_CLEANUP_PENDING:-0}" = "1" ] && \ |
| 53 | [ -n "${INFERENCE_NAME:-}" ]; then | |
| 54 | 9 | cleanup_inference_endpoint "$INFERENCE_NAME" || true |
| 55 | fi | |
| 56 | 8 | exit "$exit_code" |
| 57 | } | |
| 58 | ||
| 59 | # ── Preflight Validation ───────────────────────────────────────────────────── | |
| 60 | # Before the demo starts, we automatically check every prerequisite. | |
| 61 | # This prevents embarrassing failures mid-presentation. Each check prints | |
| 62 | # a pass/fail/warn line, and at the end we show a summary. If anything | |
| 63 | # critical failed, the presenter can bail out or type "force" to continue. | |
| 64 | ||
| 65 | 37 | CDK_JSON="cdk.json" |
| 66 | ||
| 67 | # Counters for the summary line at the end of preflight. | |
| 68 | 37 | PREFLIGHT_PASS=0 |
| 69 | 37 | PREFLIGHT_FAIL=0 |
| 70 | 37 | PREFLIGHT_WARN=0 |
| 71 | ||
| 72 | # preflight_pass: Green checkmark — this prerequisite is satisfied. | |
| 73 | preflight_pass() { | |
| 74 | 255 | echo " ${GREEN}${BOLD}✓${RESET} $1" |
| 75 | 255 | PREFLIGHT_PASS=$((PREFLIGHT_PASS + 1)) |
| 76 | } | |
| 77 | ||
| 78 | # preflight_fail: Red X — this prerequisite is missing. Second arg is the fix. | |
| 79 | preflight_fail() { | |
| 80 | 9 | echo " ${RED}${BOLD}✗${RESET} $1" |
| 81 | 9 | echo " ${DIM}Fix: $2${RESET}" |
| 82 | 9 | PREFLIGHT_FAIL=$((PREFLIGHT_FAIL + 1)) |
| 83 | } | |
| 84 | ||
| 85 | # preflight_warn: Yellow bang — not ideal but won't block the demo. | |
| 86 | preflight_warn() { | |
| 87 | 43 | echo " ${YELLOW}${BOLD}!${RESET} $1" |
| 88 | 43 | echo " ${DIM}$2${RESET}" |
| 89 | 43 | PREFLIGHT_WARN=$((PREFLIGHT_WARN + 1)) |
| 90 | } | |
| 91 | ||
| 92 | # Clear the screen for a clean start. | |
| 93 | 37 | clear |
| 94 | ||
| 95 | 37 | banner "GCO — Global Capacity Orchestrator on AWS" |
| 96 | ||
| 97 | 37 | echo " ${BOLD}Preflight Check${RESET}" |
| 98 | 37 | narrate "Validating environment before starting the demo..." |
| 99 | 37 | spacer |
| 100 | ||
| 101 | # ── Check 1: cdk.json exists ──────────────────────────────────────────────── | |
| 102 | # cdk.json is the project config file. If it's missing, we're not in the | |
| 103 | # repo root and nothing else will work. | |
| 104 | 37 | if [ -f "$CDK_JSON" ]; then |
| 105 | 36 | preflight_pass "cdk.json found" |
| 106 | else | |
| 107 | 1 | preflight_fail "cdk.json not found" "Run this script from the repo root" |
| 108 | 1 | echo "" |
| 109 | 1 | echo " ${RED}Cannot continue without cdk.json. Exiting.${RESET}" |
| 110 | 1 | exit 1 |
| 111 | fi | |
| 112 | ||
| 113 | # ── Check 2: jq installed ─────────────────────────────────────────────────── | |
| 114 | # jq is used to parse cdk.json and detect which features are enabled. | |
| 115 | 36 | if command -v jq &>/dev/null; then |
| 116 | 70 | preflight_pass "jq installed ($(jq --version 2>&1))" |
| 117 | else | |
| 118 | 1 | preflight_fail "jq not installed" "brew install jq (macOS) or apt install jq (Linux)" |
| 119 | 1 | echo "" |
| 120 | 1 | echo " ${RED}Cannot continue without jq. Exiting.${RESET}" |
| 121 | 1 | exit 1 |
| 122 | fi | |
| 123 | ||
| 124 | # ── Check 3: GCO CLI installed ────────────────────────────────────────────── | |
| 125 | # The gco CLI is the main interface we demo. Without it, there's no demo. | |
| 126 | 35 | if command -v gco &>/dev/null; then |
| 127 | 102 | GCO_VER=$(gco --version 2>&1 | head -1) |
| 128 | 34 | preflight_pass "GCO CLI installed ($GCO_VER)" |
| 129 | else | |
| 130 | 1 | preflight_fail "GCO CLI not installed" "pipx install -e . (from repo root)" |
| 131 | 1 | echo "" |
| 132 | 1 | echo " ${RED}Cannot continue without gco CLI. Exiting.${RESET}" |
| 133 | 1 | exit 1 |
| 134 | fi | |
| 135 | ||
| 136 | # ── Check 4: kubectl installed ────────────────────────────────────────────── | |
| 137 | # kubectl is needed to watch pods, get logs, and interact with the cluster | |
| 138 | # during the scheduler and storage demos. | |
| 139 | 34 | if command -v kubectl &>/dev/null; then |
| 140 | 99 | KUBECTL_VER=$(kubectl version --client -o json 2>/dev/null \ |
| 141 | | jq -r '.clientVersion.gitVersion // "unknown"' 2>/dev/null || echo "unknown") | |
| 142 | 33 | preflight_pass "kubectl installed ($KUBECTL_VER)" |
| 143 | else | |
| 144 | 1 | preflight_fail "kubectl not installed" "https://kubernetes.io/docs/tasks/tools/" |
| 145 | 1 | echo "" |
| 146 | 1 | echo " ${RED}Cannot continue without kubectl. Exiting.${RESET}" |
| 147 | 1 | exit 1 |
| 148 | fi | |
| 149 | ||
| 150 | # ── Read config values needed for remaining checks ────────────────────────── | |
| 151 | # Uses library functions for region and endpoint detection. | |
| 152 | 33 | detect_region "$CDK_JSON" |
| 153 | 33 | detect_endpoint_access "$CDK_JSON" |
| 154 | ||
| 155 | # ── Check 5: Infrastructure deployed ──────────────────────────────────────── | |
| 156 | # Verify that GCO stacks have been deployed. We check the output of | |
| 157 | # "gco stacks list" for known stack name patterns. | |
| 158 | 66 | STACK_CHECK=$(gco stacks list 2>&1 || true) |
| 159 | 66 | if echo "$STACK_CHECK" | grep -qi \ |
| 160 | "gco-.*east\|gco-.*west\|gco-.*eu\|deployed\|CREATE_COMPLETE\|UPDATE_COMPLETE"; then | |
| 161 | 31 | preflight_pass "Infrastructure deployed (stacks detected)" |
| 162 | else | |
| 163 | 2 | preflight_fail "No deployed stacks detected" "gco stacks deploy-all -y" |
| 164 | fi | |
| 165 | ||
| 166 | # ── Check 6: EKS endpoint access mode ─────────────────────────────────────── | |
| 167 | # For the demo, we need kubectl to reach the EKS API server from the | |
| 168 | # presenter's laptop. This requires PUBLIC or PUBLIC_AND_PRIVATE mode. | |
| 169 | # PRIVATE mode means kubectl only works from inside the VPC. | |
| 170 | 66 | if [ "$ENDPOINT_ACCESS" = "PUBLIC_AND_PRIVATE" ] || [ "$ENDPOINT_ACCESS" = "PUBLIC" ]; then |
| 171 | 32 | preflight_pass "EKS endpoint access: $ENDPOINT_ACCESS" |
| 172 | else | |
| 173 | 1 | preflight_warn "EKS endpoint access is $ENDPOINT_ACCESS" \ |
| 174 | "kubectl may not work from this machine. Set to PUBLIC_AND_PRIVATE in cdk.json and redeploy." | |
| 175 | fi | |
| 176 | ||
| 177 | # ── Check 7: kubectl can reach the cluster ────────────────────────────────── | |
| 178 | # Actually try to talk to the cluster. If it fails, we attempt to auto- | |
| 179 | # configure access using the setup script. | |
| 180 | 70 | KUBECTL_TEST=$(kubectl get nodes --request-timeout=5s 2>&1 || true) |
| 181 | 66 | if echo "$KUBECTL_TEST" | grep -qiE "NAME|Ready|STATUS"; then |
| 182 | # Cluster responded and has nodes | |
| 183 | 84 | NODE_COUNT=$(echo "$KUBECTL_TEST" | grep -c "Ready" 2>/dev/null || echo "0") |
| 184 | 28 | preflight_pass "kubectl connected to cluster ($NODE_COUNT node(s) ready)" |
| 185 | 10 | elif echo "$KUBECTL_TEST" | grep -qi "no resources found"; then |
| 186 | # Cluster responded but has zero nodes (normal for scale-to-zero) | |
| 187 | 1 | preflight_pass "kubectl connected to cluster (0 nodes — will scale on demand)" |
| 188 | else | |
| 189 | # Guarded recordings never auto-configure cluster access after the recorder | |
| 190 | # has exported and validated its private kubeconfig snapshot. Repository CLI | |
| 191 | # calls may refresh that disposable copy; the operator's kubeconfig remains | |
| 192 | # untouched. Normal interactive demos retain the convenience auto-setup path. | |
| 193 | 4 | if [ "${GCO_DEMO_GUARDED_RECORDING:-}" = "1" ]; then |
| 194 | 1 | preflight_fail "kubectl cannot reach the pre-authorized cluster" \ |
| 195 | "Restore the validated context before recording; auto-setup is disabled" | |
| 196 | 3 | elif [ -f "./scripts/setup-cluster-access.sh" ]; then |
| 197 | 2 | narrate " Attempting to configure cluster access..." |
| 198 | 2 | bash ./scripts/setup-cluster-access.sh "gco-$REGION" "$REGION" 2>&1 || true |
| 199 | 5 | KUBECTL_RETRY=$(kubectl get nodes --request-timeout=5s 2>&1 || true) |
| 200 | 4 | if echo "$KUBECTL_RETRY" | grep -qiE "NAME|Ready|no resources found"; then |
| 201 | 1 | preflight_pass "kubectl connected (auto-configured via setup-cluster-access.sh)" |
| 202 | else | |
| 203 | 1 | preflight_fail "kubectl cannot reach the cluster" \ |
| 204 | "./scripts/setup-cluster-access.sh gco-$REGION $REGION" | |
| 205 | fi | |
| 206 | else | |
| 207 | 1 | preflight_fail "kubectl cannot reach the cluster" \ |
| 208 | "./scripts/setup-cluster-access.sh gco-$REGION $REGION" | |
| 209 | fi | |
| 210 | fi | |
| 211 | ||
| 212 | # ── Check 8: Terminal width ───────────────────────────────────────────────── | |
| 213 | # The demo output looks best at 120+ columns. Narrower terminals cause | |
| 214 | # wrapping that makes the output harder to read for the audience. | |
| 215 | # Check COLUMNS env var first (set by asciinema), then fall back to tput. | |
| 216 | 33 | TERM_COLS="${COLUMNS:-$(tput cols 2>/dev/null || echo "80")}" |
| 217 | 33 | if [ "$TERM_COLS" -ge 120 ]; then |
| 218 | 23 | preflight_pass "Terminal width: ${TERM_COLS} columns" |
| 219 | 10 | elif [ "$TERM_COLS" -ge 90 ]; then |
| 220 | 9 | preflight_warn "Terminal width: ${TERM_COLS} columns (120+ recommended)" \ |
| 221 | "Widen your terminal for best presentation appearance." | |
| 222 | else | |
| 223 | 1 | preflight_warn "Terminal width: ${TERM_COLS} columns (120+ recommended)" \ |
| 224 | "Output may wrap and look messy. Widen your terminal window." | |
| 225 | fi | |
| 226 | ||
| 227 | # ── Check 9: Color support ────────────────────────────────────────────────── | |
| 228 | # Verify the terminal supports colors. Without colors the demo still works | |
| 229 | # but looks much less polished. | |
| 230 | 34 | if [ -t 1 ] && [ "${TERM:-dumb}" != "dumb" ]; then |
| 231 | 1 | preflight_pass "Terminal supports colors" |
| 232 | else | |
| 233 | 32 | preflight_warn "Terminal may not support colors" \ |
| 234 | "Try: TERM=xterm-256color bash demo/live_demo.sh" | |
| 235 | fi | |
| 236 | ||
| 237 | # ── Read feature flags from cdk.json ──────────────────────────────────────── | |
| 238 | # Uses detect_features() from lib_demo.sh to set the global flag variables. | |
| 239 | 33 | detect_features "$CDK_JSON" |
| 240 | 33 | detect_region "$CDK_JSON" |
| 241 | ||
| 242 | # ── Preflight Summary ─────────────────────────────────────────────────────── | |
| 243 | # Show the pass/fail/warn totals. If anything critical failed, give the | |
| 244 | # presenter a chance to bail out or force-continue. | |
| 245 | ||
| 246 | 33 | spacer |
| 247 | 33 | echo " ${DIM}──────────────────────────────────────────────────────────────${RESET}" |
| 248 | 33 | echo " ${BOLD}Results:${RESET} ${GREEN}${PREFLIGHT_PASS} passed${RESET} ${RED}${PREFLIGHT_FAIL} failed${RESET} ${YELLOW}${PREFLIGHT_WARN} warnings${RESET}" |
| 249 | 33 | echo " ${DIM}──────────────────────────────────────────────────────────────${RESET}" |
| 250 | ||
| 251 | 33 | if [ "$PREFLIGHT_FAIL" -gt 0 ]; then |
| 252 | 5 | spacer |
| 253 | 5 | echo " ${RED}${BOLD}$PREFLIGHT_FAIL check(s) failed. Fix the issues above before demoing.${RESET}" |
| 254 | 5 | if [ "${GCO_DEMO_GUARDED_RECORDING:-}" = "1" ]; then |
| 255 | 1 | echo " ${RED}Guarded recording mode never force-continues preflight failures.${RESET}" |
| 256 | 1 | exit 1 |
| 257 | fi | |
| 258 | 4 | spacer |
| 259 | 4 | echo " ${DIM}Press Enter to exit, or type 'force' to continue anyway:${RESET}" |
| 260 | 4 | if [ "${GCO_DEMO_NONINTERACTIVE:-}" = "1" ]; then |
| 261 | 2 | force_input="force" |
| 262 | else | |
| 263 | 2 | read -r force_input |
| 264 | fi | |
| 265 | 4 | if [ "$force_input" != "force" ]; then |
| 266 | 1 | exit 1 |
| 267 | fi | |
| 268 | 3 | warn "Continuing despite failures — some demo sections may break." |
| 269 | fi | |
| 270 | ||
| 271 | # ── Feature Summary ────────────────────────────────────────────────────────── | |
| 272 | # Show the audience which features are enabled so they know what to expect. | |
| 273 | ||
| 274 | 31 | spacer |
| 275 | 31 | echo " ${BOLD}One API. Every Accelerator. Any Region.${RESET}" |
| 276 | 31 | spacer |
| 277 | 31 | narrate "This live demonstration walks through GCO's core capabilities." |
| 278 | 31 | narrate "The script auto-detects which features are enabled in your deployment." |
| 279 | 31 | spacer |
| 280 | 31 | echo " ${BOLD}Region:${RESET} $REGION" |
| 281 | 62 | echo " ${BOLD}Volcano:${RESET} $(feature_status "$VOLCANO_ENABLED")" |
| 282 | 62 | echo " ${BOLD}Kueue:${RESET} $(feature_status "$KUEUE_ENABLED")" |
| 283 | 62 | echo " ${BOLD}YuniKorn:${RESET} $(feature_status "$YUNIKORN_ENABLED")" |
| 284 | 62 | echo " ${BOLD}Slurm:${RESET} $(feature_status "$SLURM_ENABLED")" |
| 285 | 62 | echo " ${BOLD}FSx Lustre:${RESET} $(feature_status "$FSX_ENABLED")" |
| 286 | 62 | echo " ${BOLD}Valkey:${RESET} $(feature_status "$VALKEY_ENABLED")" |
| 287 | 62 | echo " ${BOLD}Aurora pgvector:${RESET} $(feature_status "$AURORA_PGVECTOR_ENABLED")" |
| 288 | 62 | echo " ${BOLD}Vector store:${RESET} $(feature_status "$VECTOR_STORE_ENABLED")" |
| 289 | 31 | spacer |
| 290 | ||
| 291 | 31 | pause_for_audience |
| 292 | ||
| 293 | # ── Pre-Demo Cleanup ───────────────────────────────────────────────────────── | |
| 294 | # Delete leftover jobs from previous demo runs and wait for their pods to | |
| 295 | # disappear. Volcano's webhook rejects updates to existing jobs, so we need | |
| 296 | # a clean slate. Stale `Terminating` pods also count against the GPU/memory | |
| 297 | # resource quota until they're fully gone — skipping the wait makes the | |
| 298 | # next Kueue or Volcano submit fail with a quota error. Runs silently. | |
| 299 | 31 | narrate "Cleaning up any leftover jobs from previous runs..." |
| 300 | 31 | if [ "${GCO_DEMO_GUARDED_RECORDING:-}" = "1" ]; then |
| 301 | 30 | recording_project=$(jq -r '.context.project_name // "gco"' "$CDK_JSON") |
| 302 | 15 | detect_region "$CDK_JSON" |
| 303 | 15 | verify_recording_kube_context \ |
| 304 | "${recording_project}-${REGION}" "$REGION" | |
| 305 | fi | |
| 306 | 30 | kubectl delete jobs --all -n gco-jobs --ignore-not-found=true >/dev/null 2>&1 || true |
| 307 | 30 | kubectl delete vcjob --all -n gco-jobs --ignore-not-found=true >/dev/null 2>&1 || true |
| 308 | 31 | gco inference delete demo-llm -y >/dev/null 2>&1 || true |
| 309 | ||
| 310 | # Wait up to 30s for the job-owned pods to actually disappear. Without this, | |
| 311 | # a fresh submit can hit "forbidden: exceeded quota" because the old pod's | |
| 312 | # GPU/CPU/memory requests are still reserved during the termination window. | |
| 313 | # | |
| 314 | # Note on the pipeline shape: ``grep -c`` still prints ``0`` to stdout when | |
| 315 | # there are no matches, but exits 1. Under ``set -euo pipefail`` that would | |
| 316 | # kill the script silently right after "Cleaning up any leftover jobs". | |
| 317 | # Running the substitution inside ``|| true`` neutralizes the exit code | |
| 318 | # without doubling-up the output the way ``|| echo 0`` would. | |
| 319 | 61 | for _ in $(seq 1 30); do |
| 320 | 123 | LEFTOVER=$( |
| 321 | { kubectl get pods -n gco-jobs --no-headers 2>/dev/null \ | |
| 322 | | grep -cEv '^(gco-|slinky-)'; } || true | |
| 323 | ) | |
| 324 | 31 | if [ "${LEFTOVER:-0}" -eq 0 ]; then |
| 325 | 30 | break |
| 326 | fi | |
| 327 | 1 | sleep 1 |
| 328 | done | |
| 329 | 30 | success "Cleanup complete." |
| 330 | 30 | spacer |
| 331 | ||
| 332 | # ── Pre-Deploy Inference (background) ──────────────────────────────────────── | |
| 333 | # Deploy the inference endpoint now so the GPU node provisions while we demo | |
| 334 | # costs, capacity, schedulers, and storage. By the time we reach the inference | |
| 335 | # section, the model should be loaded and ready to serve. | |
| 336 | 30 | if [ "${SKIP_INFERENCE:-}" != "1" ]; then |
| 337 | 28 | INFERENCE_NAME="demo-llm" |
| 338 | # Wait for any leftover pods from previous runs to fully terminate | |
| 339 | 28 | narrate "Waiting for previous inference pods to terminate..." |
| 340 | 190 | for _ in $(seq 1 20); do |
| 341 | 324 | OLD_PODS=$(kubectl get pods -n gco-inference -l app="$INFERENCE_NAME" --no-headers 2>/dev/null || true) |
| 342 | 162 | if [ -z "$OLD_PODS" ]; then |
| 343 | 21 | break |
| 344 | fi | |
| 345 | # Force-delete stuck Terminating pods after a few attempts | |
| 346 | 282 | if echo "$OLD_PODS" | grep -q "Terminating"; then |
| 347 | 1 | kubectl delete pods -n gco-inference -l app="$INFERENCE_NAME" --force --grace-period=0 >/dev/null 2>&1 || true |
| 348 | fi | |
| 349 | 141 | sleep 3 |
| 350 | done | |
| 351 | 28 | narrate "Pre-deploying inference endpoint (GPU will provision in background)..." |
| 352 | # Retry deploy in case the previous endpoint hasn't been fully cleaned up yet. | |
| 353 | 28 | DEPLOY_OUTPUT="" |
| 354 | 28 | INFERENCE_DEPLOYED=false |
| 355 | 60 | for deploy_attempt in $(seq 1 5); do |
| 356 | 64 | if DEPLOY_OUTPUT=$(gco inference deploy "$INFERENCE_NAME" -i vllm/vllm-openai:v0.29.0 \ |
| 357 | --gpu-count 1 --replicas 1 -r "$REGION" \ | |
| 358 | --extra-args '--model' --extra-args 'facebook/opt-125m' 2>&1) && \ | |
| 359 | 54 | echo "$DEPLOY_OUTPUT" | grep -qi "registered\|success"; then |
| 360 | 27 | INFERENCE_DEPLOYED=true |
| 361 | 27 | break |
| 362 | fi | |
| 363 | 5 | if [ "$deploy_attempt" -lt 5 ]; then |
| 364 | 4 | sleep 5 |
| 365 | fi | |
| 366 | done | |
| 367 | 28 | if [ "$INFERENCE_DEPLOYED" != "true" ]; then |
| 368 | 1 | if [ -n "$DEPLOY_OUTPUT" ]; then |
| 369 | 1 | printf ' %s\n' "${DEPLOY_OUTPUT//$'\n'/$'\n '}" |
| 370 | 1 | fi |
| 371 | 1 | warn "Inference deployment was not accepted after 5 attempts." |
| 372 | 1 | cleanup_inference_endpoint "$INFERENCE_NAME" || true |
| 373 | 1 | exit 1 |
| 374 | 1 | fi |
| 375 | 27 | INFERENCE_CLEANUP_PENDING=1 |
| 376 | 27 | trap 'cleanup_demo_inference_on_exit "$?"' EXIT |
| 377 | 27 | success "Inference endpoint queued for deployment." |
| 378 | 27 | spacer |
| 379 | fi | |
| 380 | ||
| 381 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 382 | # SECTION: Fleet Overview | |
| 383 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 384 | # One aggregate document replaces four separate cost/status calls in the | |
| 385 | # recording: stack state, queue/jobs, capacity, inference, policy agreement, | |
| 386 | # and the optional 30-day Cost Explorer view. | |
| 387 | ||
| 388 | 29 | if [ "${SKIP_COSTS:-}" != "1" ]; then |
| 389 | ||
| 390 | 54 | SECTION=$((SECTION + 1)); section_header "$SECTION" "FLEET OVERVIEW — Status, Cost, and Policy" "$GREEN" |
| 391 | ||
| 392 | 27 | narrate "Start with one fleet-wide answer: what is deployed, what is queued," |
| 393 | 27 | narrate "where capacity exists, whether policy agrees, and what it costs." |
| 394 | 27 | spacer |
| 395 | ||
| 396 | 27 | highlight "Aggregate status across every configured region" |
| 397 | 27 | run_cmd "gco status --with-costs --with-policy" |
| 398 | 27 | sleep "$PAUSE_SHORT" |
| 399 | ||
| 400 | 27 | success "One command joins the control plane without hiding unavailable sections." |
| 401 | 27 | narrate "The base fleet document is also available through the MCP server." |
| 402 | 27 | narrate "Policy comparison is CLI-only; the CLI can also emit strict JSON." |
| 403 | ||
| 404 | 27 | pause_for_audience |
| 405 | ||
| 406 | fi # SKIP_COSTS | |
| 407 | ||
| 408 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 409 | # SECTION: Capacity Discovery | |
| 410 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 411 | # Shows how GCO finds GPU capacity across regions and routes jobs to where | |
| 412 | # resources are actually available. | |
| 413 | ||
| 414 | 29 | if [ "${SKIP_CAPACITY:-}" != "1" ]; then |
| 415 | ||
| 416 | 54 | SECTION=$((SECTION + 1)); section_header "$SECTION" "CAPACITY DISCOVERY — Find GPUs Across Regions" "$GREEN" |
| 417 | ||
| 418 | 27 | narrate "GPU availability varies by region and changes constantly." |
| 419 | 27 | narrate "GCO checks Spot Placement Scores and instance availability" |
| 420 | 27 | narrate "across all configured regions to find where GPUs are right now." |
| 421 | 27 | spacer |
| 422 | ||
| 423 | 27 | highlight "Check GPU availability in a specific region" |
| 424 | 27 | run_cmd "gco capacity check --instance-type g4dn.xlarge --region $REGION" || true |
| 425 | 27 | sleep "$PAUSE_SHORT" |
| 426 | ||
| 427 | 27 | highlight "Find the best region for GPU workloads" |
| 428 | 27 | run_cmd "gco capacity recommend-region --gpu" || true |
| 429 | 27 | sleep "$PAUSE_SHORT" |
| 430 | ||
| 431 | 27 | highlight "Submit a job with automatic region selection" |
| 432 | 27 | narrate "The CLI analyzes capacity across all regions, picks the best one," |
| 433 | 27 | narrate "and places the job on that region's SQS queue automatically." |
| 434 | 27 | run_cmd "gco jobs submit-sqs examples/simple-job.yaml --auto-region" || true |
| 435 | 27 | sleep "$PAUSE_SHORT" |
| 436 | ||
| 437 | 27 | highlight "Check the SQS queue status across all regions" |
| 438 | 27 | run_cmd "gco jobs queue-status --all-regions" || true |
| 439 | 27 | sleep "$PAUSE_SHORT" |
| 440 | ||
| 441 | 27 | success "Capacity-aware job placement without manual region selection." |
| 442 | ||
| 443 | 27 | pause_for_audience |
| 444 | ||
| 445 | fi # SKIP_CAPACITY | |
| 446 | ||
| 447 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 448 | # SECTION: Schedulers | |
| 449 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 450 | # GCO supports multiple Kubernetes schedulers simultaneously. Each one is | |
| 451 | # opt-in via cdk.json. We only demo the ones that are actually enabled. | |
| 452 | # This section is skipped entirely if SKIP_SCHEDULERS=1. | |
| 453 | ||
| 454 | 29 | if [ "${SKIP_SCHEDULERS:-}" != "1" ]; then |
| 455 | ||
| 456 | # Track how many schedulers we demo for the summary at the end. | |
| 457 | 27 | SCHEDULER_COUNT=0 |
| 458 | ||
| 459 | # ── Volcano ────────────────────────────────────────────────────────────────── | |
| 460 | # Volcano is a CNCF batch scheduler for Kubernetes. Its main feature is | |
| 461 | # "gang scheduling" — all pods in a distributed training job must be | |
| 462 | # schedulable at the same time, or none of them start. This prevents | |
| 463 | # deadlocks in multi-node training. | |
| 464 | ||
| 465 | 27 | if [ "$VOLCANO_ENABLED" = "true" ]; then |
| 466 | ||
| 467 | 18 | SECTION=$((SECTION + 1)); section_header "$SECTION" "VOLCANO — Gang Scheduling for Distributed Training" "$MAGENTA" |
| 468 | ||
| 469 | 9 | narrate "Volcano is a Kubernetes-native batch scheduler built for AI/ML." |
| 470 | 9 | narrate "Its killer feature: gang scheduling — all pods in a distributed" |
| 471 | 9 | narrate "training job start together, or none of them start at all." |
| 472 | 9 | narrate "This prevents deadlocks where half the workers are waiting forever." |
| 473 | 9 | spacer |
| 474 | ||
| 475 | 9 | highlight "Submitting a Volcano gang-scheduled job (1 master + 2 workers)" |
| 476 | 10 | run_cmd "gco jobs submit-direct examples/volcano-gang-job.yaml -r $REGION -n gco-jobs" || true |
| 477 | 9 | sleep "$PAUSE_SHORT" |
| 478 | ||
| 479 | 9 | narrate "Volcano ensures all 3 pods are co-scheduled atomically." |
| 480 | 9 | narrate "Let's watch them come up together..." |
| 481 | 9 | spacer |
| 482 | ||
| 483 | 9 | highlight "Checking job status" |
| 484 | 9 | countdown "Waiting for pods to schedule" "$WAIT_FOR_POD" |
| 485 | 9 | run_cmd "kubectl get pods -n gco-jobs -l volcano.sh/job-name=distributed-training --no-headers 2>/dev/null || echo ' (pods not yet visible — node provisioning in progress)'" |
| 486 | ||
| 487 | 9 | highlight "Volcano job status" |
| 488 | 9 | run_cmd "kubectl get vcjob -n gco-jobs --no-headers 2>/dev/null || echo ' (checking Volcano job status...)'" |
| 489 | ||
| 490 | 9 | success "Gang scheduling ensures distributed training jobs don't deadlock." |
| 491 | # Release resource-quota reservations held by this job's pods so the next | |
| 492 | # scheduler section doesn't hit "exceeded quota" on submit. | |
| 493 | 9 | kubectl delete vcjob distributed-training -n gco-jobs --ignore-not-found=true >/dev/null 2>&1 || true |
| 494 | 9 | SCHEDULER_COUNT=$((SCHEDULER_COUNT + 1)) |
| 495 | ||
| 496 | 9 | pause_for_audience |
| 497 | ||
| 498 | fi # VOLCANO | |
| 499 | ||
| 500 | # ── Kueue ──────────────────────────────────────────────────────────────────── | |
| 501 | # Kueue is the Kubernetes-native job queueing system (SIG Scheduling). | |
| 502 | # It manages resource quotas per team/namespace and holds jobs in a queue | |
| 503 | # until the cluster has enough resources to run them. | |
| 504 | ||
| 505 | 27 | if [ "$KUEUE_ENABLED" = "true" ]; then |
| 506 | ||
| 507 | 18 | SECTION=$((SECTION + 1)); section_header "$SECTION" "KUEUE — Quota-Based Job Queueing" "$MAGENTA" |
| 508 | ||
| 509 | 9 | narrate "Kueue is the Kubernetes-native job queueing system." |
| 510 | 9 | narrate "It manages resource quotas, fair-sharing between teams, and" |
| 511 | 9 | narrate "holds jobs in a queue until cluster resources are available." |
| 512 | 9 | narrate "Think of it as a resource-aware admission controller for batch jobs." |
| 513 | 9 | spacer |
| 514 | ||
| 515 | 9 | highlight "Submitting a Kueue-managed job" |
| 516 | 10 | run_cmd "gco jobs submit-direct examples/kueue-job.yaml -r $REGION -n gco-jobs" || true |
| 517 | 9 | sleep "$PAUSE_SHORT" |
| 518 | ||
| 519 | 9 | highlight "Checking Kueue queue status" |
| 520 | 9 | run_cmd "kubectl get clusterqueue --no-headers 2>/dev/null || echo ' (ClusterQueue not yet created — will be created by the manifest)'" |
| 521 | 9 | run_cmd "kubectl get localqueue -n gco-jobs --no-headers 2>/dev/null || echo ' (LocalQueue not yet created)'" |
| 522 | ||
| 523 | 9 | countdown "Waiting for workload admission" "$PAUSE_SHORT" |
| 524 | ||
| 525 | 9 | highlight "Kueue workloads (jobs waiting or admitted)" |
| 526 | 9 | run_cmd "kubectl get workloads -n gco-jobs --no-headers 2>/dev/null || echo ' (no workloads yet)'" |
| 527 | ||
| 528 | 9 | success "Kueue prevents resource overcommit and enforces team quotas." |
| 529 | # Release resource-quota reservations from these jobs before the next section. | |
| 530 | 9 | kubectl delete job kueue-sample-job kueue-gpu-job -n gco-jobs --ignore-not-found=true >/dev/null 2>&1 || true |
| 531 | 9 | SCHEDULER_COUNT=$((SCHEDULER_COUNT + 1)) |
| 532 | ||
| 533 | 9 | pause_for_audience |
| 534 | ||
| 535 | fi # KUEUE | |
| 536 | ||
| 537 | # ── YuniKorn ───────────────────────────────────────────────────────────────── | |
| 538 | # Apache YuniKorn provides hierarchical queues and fair-sharing for | |
| 539 | # multi-tenant clusters. Teams get guaranteed resource shares with the | |
| 540 | # ability to borrow unused capacity from other teams. | |
| 541 | ||
| 542 | 27 | if [ "$YUNIKORN_ENABLED" = "true" ]; then |
| 543 | ||
| 544 | 18 | SECTION=$((SECTION + 1)); section_header "$SECTION" "YUNIKORN — App-Aware Fair Scheduling" "$MAGENTA" |
| 545 | ||
| 546 | 9 | narrate "Apache YuniKorn brings hierarchical queues and fair-sharing" |
| 547 | 9 | narrate "to Kubernetes. It's designed for multi-tenant clusters where" |
| 548 | 9 | narrate "multiple teams compete for GPU resources." |
| 549 | 9 | narrate "YuniKorn also supports gang scheduling and preemption." |
| 550 | 9 | spacer |
| 551 | ||
| 552 | 9 | YUNIKORN_SUBMITTED=0 |
| 553 | 9 | highlight "Submitting a YuniKorn-scheduled job" |
| 554 | 9 | if run_cmd "gco jobs submit-direct examples/yunikorn-job.yaml -r $REGION -n gco-jobs"; then |
| 555 | 7 | YUNIKORN_SUBMITTED=1 |
| 556 | fi | |
| 557 | 9 | sleep "$PAUSE_SHORT" |
| 558 | ||
| 559 | 9 | highlight "Checking YuniKorn pod scheduling" |
| 560 | 9 | countdown "Waiting for YuniKorn to place pods" "$PAUSE_SHORT" |
| 561 | 9 | run_cmd "kubectl get pods -n gco-jobs -l app=yunikorn-demo --no-headers 2>/dev/null || echo ' (pods scheduling...)'" |
| 562 | ||
| 563 | 9 | if ! report_feature_result "$YUNIKORN_SUBMITTED" "YuniKorn" \ |
| 564 | "YuniKorn provides enterprise-grade multi-tenant scheduling."; then | |
| 565 | 1 | exit 1 |
| 566 | fi | |
| 567 | # Release resource-quota reservations from these jobs before the next section. | |
| 568 | 8 | kubectl delete job yunikorn-sample-job yunikorn-gpu-job yunikorn-gang-job -n gco-jobs --ignore-not-found=true >/dev/null 2>&1 || true |
| 569 | 8 | SCHEDULER_COUNT=$((SCHEDULER_COUNT + 1)) |
| 570 | ||
| 571 | 8 | pause_for_audience |
| 572 | ||
| 573 | fi # YUNIKORN | |
| 574 | ||
| 575 | # ── Slurm ──────────────────────────────────────────────────────────────────── | |
| 576 | # The Slinky Slurm Operator runs a full Slurm cluster inside Kubernetes. | |
| 577 | # This lets teams with existing HPC workflows (sbatch scripts, etc.) run | |
| 578 | # them on GCO without modification. | |
| 579 | ||
| 580 | 26 | if [ "$SLURM_ENABLED" = "true" ]; then |
| 581 | ||
| 582 | 16 | SECTION=$((SECTION + 1)); section_header "$SECTION" "SLURM — HPC Batch Scheduling on Kubernetes" "$MAGENTA" |
| 583 | ||
| 584 | 8 | narrate "For teams coming from traditional HPC, GCO includes the Slinky" |
| 585 | 8 | narrate "Slurm Operator. It runs a full Slurm cluster inside Kubernetes," |
| 586 | 8 | narrate "so existing sbatch scripts and workflows work unchanged." |
| 587 | 8 | narrate "This bridges the gap between HPC and cloud-native." |
| 588 | 8 | spacer |
| 589 | ||
| 590 | 8 | SLURM_SUBMITTED=0 |
| 591 | 8 | highlight "Submitting a Slurm batch job via Kubernetes" |
| 592 | 8 | if run_cmd "gco jobs submit-direct examples/slurm-cluster-job.yaml -r $REGION -n gco-jobs"; then |
| 593 | 6 | SLURM_SUBMITTED=1 |
| 594 | fi | |
| 595 | 8 | sleep "$PAUSE_SHORT" |
| 596 | ||
| 597 | 8 | highlight "Checking Slurm job pod" |
| 598 | 8 | wait_for_job "slurm-test" "gco-jobs" |
| 599 | 8 | run_cmd "kubectl get pods -n gco-jobs -l job-name=slurm-test --no-headers 2>/dev/null || echo ' (Slurm job pod starting...)'" |
| 600 | ||
| 601 | 8 | highlight "Tailing Slurm job logs" |
| 602 | 8 | run_cmd "kubectl logs job/slurm-test -n gco-jobs --all-containers=true --tail=20 2>/dev/null || kubectl logs -n gco-jobs -l job-name=slurm-test --all-containers=true --tail=20 2>/dev/null || echo ' (no logs yet)'" |
| 603 | ||
| 604 | 8 | if ! report_feature_result "$SLURM_SUBMITTED" "Slurm" \ |
| 605 | "Existing HPC workflows run on Kubernetes without modification."; then | |
| 606 | 1 | exit 1 |
| 607 | fi | |
| 608 | # Release resource-quota reservations so FSx / Valkey / EFS sections don't | |
| 609 | # hit quota errors. slurm-test itself goes away quickly; we also clean up | |
| 610 | # any Slurm-operator-owned workload pods that were spawned for this job. | |
| 611 | 7 | kubectl delete job slurm-test -n gco-jobs --ignore-not-found=true >/dev/null 2>&1 || true |
| 612 | 7 | SCHEDULER_COUNT=$((SCHEDULER_COUNT + 1)) |
| 613 | ||
| 614 | 7 | pause_for_audience |
| 615 | ||
| 616 | fi # SLURM | |
| 617 | ||
| 618 | # ── Scheduler Summary ──────────────────────────────────────────────────────── | |
| 619 | ||
| 620 | 25 | if [ "$SCHEDULER_COUNT" -gt 0 ]; then |
| 621 | 7 | spacer |
| 622 | 7 | echo " ${GREEN}${BOLD}Demonstrated $SCHEDULER_COUNT scheduler(s) — plus KEDA running the SQS queue processor.${RESET}" |
| 623 | 7 | narrate "GCO supports 5 schedulers simultaneously — pick the right" |
| 624 | 7 | narrate "tool for each workload type, all on the same cluster." |
| 625 | 7 | spacer |
| 626 | fi | |
| 627 | ||
| 628 | fi # SKIP_SCHEDULERS | |
| 629 | ||
| 630 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 631 | # SECTION: FSx for Lustre | |
| 632 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 633 | # FSx for Lustre is a high-performance parallel file system. It provides | |
| 634 | # hundreds of GB/s of throughput with sub-millisecond latency — critical | |
| 635 | # for ML training on large datasets. This section only runs if FSx is | |
| 636 | # enabled in cdk.json. | |
| 637 | ||
| 638 | 27 | if [ "$FSX_ENABLED" = "true" ]; then |
| 639 | ||
| 640 | 16 | SECTION=$((SECTION + 1)); section_header "$SECTION" "FSx FOR LUSTRE — High-Performance Scratch Storage" "$BLUE" |
| 641 | ||
| 642 | 8 | narrate "ML training on large datasets needs serious I/O throughput." |
| 643 | 8 | narrate "FSx for Lustre provides hundreds of GB/s of throughput with" |
| 644 | 8 | narrate "sub-millisecond latency — purpose-built for HPC and ML." |
| 645 | 8 | narrate "GCO provisions it automatically and mounts it into every cluster." |
| 646 | 8 | spacer |
| 647 | ||
| 648 | 8 | FSX_SUBMITTED=0 |
| 649 | 8 | highlight "Submitting a job that exercises FSx Lustre storage" |
| 650 | 8 | if run_cmd "gco jobs submit-direct examples/fsx-lustre-job.yaml -r $REGION -n gco-jobs"; then |
| 651 | 6 | FSX_SUBMITTED=1 |
| 652 | fi | |
| 653 | ||
| 654 | 8 | highlight "Watching the FSx job" |
| 655 | 8 | wait_for_job "fsx-lustre-example" "gco-jobs" |
| 656 | 8 | run_cmd "kubectl get pods -n gco-jobs -l example=fsx-lustre --no-headers 2>/dev/null || echo ' (pod scheduling...)'" |
| 657 | ||
| 658 | 8 | spacer |
| 659 | 8 | narrate "The job writes 10 MB of simulated training data, saves a checkpoint," |
| 660 | 8 | narrate "reads it all back, and reports throughput numbers." |
| 661 | 8 | spacer |
| 662 | ||
| 663 | 8 | highlight "Checking job logs for I/O performance" |
| 664 | 8 | run_cmd "kubectl logs job/fsx-lustre-example -n gco-jobs --all-containers=true --tail=30 2>/dev/null || kubectl logs -n gco-jobs -l example=fsx-lustre --all-containers=true --tail=30 2>/dev/null || echo ' (no logs yet)'" |
| 665 | ||
| 666 | 8 | if ! report_feature_result "$FSX_SUBMITTED" "FSx for Lustre" \ |
| 667 | "FSx for Lustre: sub-millisecond latency, hundreds of GB/s throughput."; then | |
| 668 | 1 | exit 1 |
| 669 | fi | |
| 670 | # Release resource-quota reservations before Valkey/inference/EFS sections. | |
| 671 | 7 | kubectl delete job fsx-lustre-example -n gco-jobs --ignore-not-found=true >/dev/null 2>&1 || true |
| 672 | 7 | narrate "Compare: EFS tops out around 10 GB/s. For large-scale training," |
| 673 | 7 | narrate "FSx is the difference between hours and minutes." |
| 674 | ||
| 675 | 7 | pause_for_audience |
| 676 | ||
| 677 | fi # FSX | |
| 678 | ||
| 679 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 680 | # SECTION: Valkey Cache | |
| 681 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 682 | # Valkey is the open-source successor to Redis. GCO deploys it as a | |
| 683 | # serverless cache in each region. Common uses: prompt caching (saves | |
| 684 | # 30-50% on inference costs), feature stores, and session state. | |
| 685 | ||
| 686 | 26 | if [ "$VALKEY_ENABLED" = "true" ]; then |
| 687 | ||
| 688 | 16 | SECTION=$((SECTION + 1)); section_header "$SECTION" "VALKEY — Serverless In-Memory Cache" "$BLUE" |
| 689 | ||
| 690 | 8 | narrate "Valkey (the open-source Redis successor) runs as a serverless" |
| 691 | 8 | narrate "cache in each region. Use it for prompt caching, feature stores," |
| 692 | 8 | narrate "session state, or any low-latency K/V access from your jobs." |
| 693 | 8 | narrate "The endpoint is injected automatically — no config needed in manifests." |
| 694 | 8 | spacer |
| 695 | ||
| 696 | 8 | VALKEY_SUBMITTED=0 |
| 697 | 8 | highlight "Submitting a job that exercises the Valkey cache" |
| 698 | 8 | if run_cmd "gco jobs submit-direct examples/valkey-cache-job.yaml -r $REGION -n gco-jobs"; then |
| 699 | 6 | VALKEY_SUBMITTED=1 |
| 700 | fi | |
| 701 | ||
| 702 | 8 | highlight "Watching the Valkey job" |
| 703 | 8 | wait_for_job "valkey-cache-example" "gco-jobs" |
| 704 | 8 | run_cmd "kubectl get pods -n gco-jobs -l app=valkey-cache-example --no-headers 2>/dev/null || echo ' (pod scheduling...)'" |
| 705 | ||
| 706 | 8 | highlight "Valkey job output" |
| 707 | 8 | run_cmd "kubectl logs job/valkey-cache-example -n gco-jobs --all-containers=true --tail=20 2>/dev/null || kubectl logs -n gco-jobs -l app=valkey-cache-example --all-containers=true --tail=20 2>/dev/null || echo ' (no logs yet)'" |
| 708 | ||
| 709 | 8 | if ! report_feature_result "$VALKEY_SUBMITTED" "Valkey" \ |
| 710 | "Serverless Valkey: zero management, auto-scaling, per-region."; then | |
| 711 | 1 | exit 1 |
| 712 | fi | |
| 713 | # Release resource-quota reservations before the inference/EFS sections. | |
| 714 | 7 | kubectl delete job valkey-cache-example -n gco-jobs --ignore-not-found=true >/dev/null 2>&1 || true |
| 715 | 7 | narrate "Prompt caching alone can cut inference costs by 30-50%." |
| 716 | ||
| 717 | 7 | pause_for_audience |
| 718 | ||
| 719 | fi # VALKEY | |
| 720 | ||
| 721 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 722 | # SECTION: Aurora pgvector | |
| 723 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 724 | # Aurora Serverless v2 with pgvector provides a fully managed vector database | |
| 725 | # for RAG, semantic search, and embedding storage. This section only runs if | |
| 726 | # Aurora pgvector is enabled in cdk.json. | |
| 727 | ||
| 728 | 25 | if [ "$AURORA_PGVECTOR_ENABLED" = "true" ]; then |
| 729 | ||
| 730 | 12 | SECTION=$((SECTION + 1)); section_header "$SECTION" "AURORA PGVECTOR — Serverless Vector Database" "$BLUE" |
| 731 | ||
| 732 | 6 | narrate "For RAG, semantic search, and embedding storage, GCO can deploy" |
| 733 | 6 | narrate "Aurora Serverless v2 with pgvector in each region. It auto-scales" |
| 734 | 6 | narrate "capacity and requires no instance management." |
| 735 | 6 | narrate "Credentials are in Secrets Manager — pods discover them via ConfigMap." |
| 736 | 6 | spacer |
| 737 | ||
| 738 | 6 | AURORA_SUBMITTED=0 |
| 739 | 6 | highlight "Submitting a job that exercises Aurora pgvector" |
| 740 | 6 | if run_cmd "gco jobs submit-direct examples/aurora-pgvector-job.yaml -r $REGION -n gco-jobs"; then |
| 741 | 4 | AURORA_SUBMITTED=1 |
| 742 | fi | |
| 743 | ||
| 744 | 6 | highlight "Watching the Aurora pgvector job" |
| 745 | 6 | wait_for_job "aurora-pgvector-example" "gco-jobs" |
| 746 | 6 | run_cmd "kubectl get pods -n gco-jobs -l app=aurora-pgvector-example --no-headers 2>/dev/null || echo ' (pod scheduling...)'" |
| 747 | ||
| 748 | 6 | highlight "Aurora pgvector job output" |
| 749 | 6 | run_cmd "kubectl logs job/aurora-pgvector-example -n gco-jobs --all-containers=true --tail=20 2>/dev/null || kubectl logs -n gco-jobs -l app=aurora-pgvector-example --all-containers=true --tail=20 2>/dev/null || echo ' (no logs yet)'" |
| 750 | ||
| 751 | 6 | if ! report_feature_result "$AURORA_SUBMITTED" "Aurora pgvector" \ |
| 752 | "Serverless Aurora pgvector: vector search with zero management."; then | |
| 753 | 1 | exit 1 |
| 754 | fi | |
| 755 | # Release resource-quota reservations before the next section. | |
| 756 | 5 | kubectl delete job aurora-pgvector-example -n gco-jobs --ignore-not-found=true >/dev/null 2>&1 || true |
| 757 | 5 | narrate "pgvector supports HNSW and IVFFlat indexes for fast similarity search." |
| 758 | ||
| 759 | 5 | pause_for_audience |
| 760 | ||
| 761 | fi # AURORA_PGVECTOR | |
| 762 | ||
| 763 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 764 | # SECTION: Globally Replicated Vector Store | |
| 765 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 766 | # The complement to Aurora pgvector rather than a competitor: pgvector gives | |
| 767 | # SQL (joins, range predicates, transactions), this gives a DynamoDB global | |
| 768 | # table whose vector index and data replicate to every deployment region, so | |
| 769 | # every cluster reads its own local replica. Ingestion is S3-triggered — drop | |
| 770 | # a document on the cluster-shared bucket and a Lambda chunks, embeds, and | |
| 771 | # writes it. This section only runs if the vector store is enabled. | |
| 772 | ||
| 773 | 24 | if [ "$VECTOR_STORE_ENABLED" = "true" ]; then |
| 774 | ||
| 775 | 10 | SECTION=$((SECTION + 1)); section_header "$SECTION" "VECTOR STORE — Globally Replicated Semantic Search" "$BLUE" |
| 776 | ||
| 777 | 5 | narrate "RAG workloads need their corpus close to the accelerators using it." |
| 778 | 5 | narrate "GCO can provision a DynamoDB global table with a vector index whose" |
| 779 | 5 | narrate "definition AND data replicate to every deployment region, so a job" |
| 780 | 5 | narrate "in any region searches a local replica — no cross-region hop." |
| 781 | 5 | spacer |
| 782 | ||
| 783 | 5 | highlight "Vector store, replicas, and index state" |
| 784 | 5 | run_cmd "gco vector status --output table" || true |
| 785 | ||
| 786 | 5 | spacer |
| 787 | 5 | narrate "Ingestion is just an upload: the S3 event invokes a Lambda that" |
| 788 | 5 | narrate "chunks each document, embeds it with Amazon Bedrock Titan, and" |
| 789 | 5 | narrate "writes the vectors. Let's seed it with GCO's own documentation." |
| 790 | 5 | spacer |
| 791 | ||
| 792 | 5 | VECTOR_INGESTED=0 |
| 793 | 5 | highlight "Ingesting the checkout's docs/*.md as a demo corpus" |
| 794 | 5 | if run_cmd "gco vector ingest --demo --wait --output table"; then |
| 795 | 3 | VECTOR_INGESTED=1 |
| 796 | fi | |
| 797 | ||
| 798 | 5 | VECTOR_SEARCHED=0 |
| 799 | 5 | if [ "$VECTOR_INGESTED" -eq 1 ]; then |
| 800 | 3 | spacer |
| 801 | 3 | narrate "Now a semantic query — not a keyword grep. The store returns the" |
| 802 | 3 | narrate "passages closest in embedding space, with their similarity scores." |
| 803 | 3 | spacer |
| 804 | ||
| 805 | 3 | highlight "Semantic search: \"how does capacity history work?\"" |
| 806 | 3 | if run_cmd "gco vector search 'how does capacity history work?' --top-k 5 --output table"; then |
| 807 | 2 | VECTOR_SEARCHED=1 |
| 808 | fi | |
| 809 | ||
| 810 | 3 | highlight "The same query against the ${REGION} replica (local read)" |
| 811 | 4 | run_cmd "gco vector search 'how does capacity history work?' --top-k 3 --region $REGION --output table" || true |
| 812 | fi | |
| 813 | ||
| 814 | # The claim is "globally replicated semantic search", so both halves must hold: | |
| 815 | # a corpus that actually ingested, and a query that actually returned matches. | |
| 816 | 5 | VECTOR_PROVEN=0 |
| 817 | 8 | if [ "$VECTOR_INGESTED" -eq 1 ] && [ "$VECTOR_SEARCHED" -eq 1 ]; then |
| 818 | 2 | VECTOR_PROVEN=1 |
| 819 | fi | |
| 820 | 5 | if ! report_feature_result "$VECTOR_PROVEN" "Vector store" \ |
| 821 | "Globally replicated vector search — ingest once, query in every region."; then | |
| 822 | 2 | exit 1 |
| 823 | fi | |
| 824 | 3 | narrate "Complementary to Aurora pgvector, not a replacement: pgvector brings" |
| 825 | 3 | narrate "SQL joins and transactions, this brings managed global replication." |
| 826 | ||
| 827 | 3 | pause_for_audience |
| 828 | ||
| 829 | fi # VECTOR_STORE | |
| 830 | ||
| 831 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 832 | # SECTION: EFS Shared Storage | |
| 833 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 834 | # EFS (Elastic File System) is always deployed — it's the default shared | |
| 835 | # storage for job outputs. This section always runs because EFS is a core | |
| 836 | # feature, not optional. | |
| 837 | ||
| 838 | 44 | SECTION=$((SECTION + 1)); section_header "$SECTION" "EFS — Persistent Shared Storage" "$BLUE" |
| 839 | ||
| 840 | 22 | narrate "When a Kubernetes pod terminates, its local data vanishes." |
| 841 | 22 | narrate "GCO mounts Amazon EFS into every cluster so job outputs," |
| 842 | 22 | narrate "model checkpoints, and training artifacts persist beyond pod lifetime." |
| 843 | 22 | narrate "You can download results even after the job is long gone." |
| 844 | 22 | spacer |
| 845 | ||
| 846 | 22 | highlight "Submitting a job that writes results to shared EFS storage" |
| 847 | 23 | run_cmd "gco jobs submit-direct examples/efs-output-job.yaml -r $REGION -n gco-jobs" || true |
| 848 | ||
| 849 | 22 | highlight "Watching the EFS job" |
| 850 | 22 | wait_for_job "efs-output-example" "gco-jobs" |
| 851 | 22 | run_cmd "kubectl get pods -n gco-jobs -l example=efs-output --no-headers 2>/dev/null || echo ' (pod scheduling...)'" |
| 852 | ||
| 853 | 22 | highlight "Job logs — results written to /outputs on EFS" |
| 854 | 22 | run_cmd "kubectl logs job/efs-output-example -n gco-jobs --all-containers=true --tail=15 2>/dev/null || kubectl logs -n gco-jobs -l example=efs-output --all-containers=true --tail=15 2>/dev/null || echo ' (no logs yet)'" |
| 855 | ||
| 856 | 22 | spacer |
| 857 | 22 | narrate "The pod is gone, but the data lives on. Let's prove it." |
| 858 | 22 | spacer |
| 859 | ||
| 860 | 22 | highlight "Listing files on shared EFS storage" |
| 861 | 22 | run_cmd "gco files ls -r $REGION" || true |
| 862 | ||
| 863 | 22 | highlight "Downloading results to local machine" |
| 864 | 22 | run_cmd "gco files download efs-output-example /tmp/gco-demo-results -r $REGION && cat /tmp/gco-demo-results/results.json" |
| 865 | ||
| 866 | 22 | success "Persistent storage that survives pod termination." |
| 867 | 22 | narrate "Critical for ML checkpoints, training artifacts, and audit trails." |
| 868 | ||
| 869 | 22 | pause_for_audience |
| 870 | ||
| 871 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 872 | # SECTION: Inference Endpoint | |
| 873 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 874 | # The inference endpoint was deployed at the start of the demo (before costs) | |
| 875 | # so the GPU node could provision in the background. By now it should be | |
| 876 | # ready. We just need to wait for readiness, invoke it, and clean up. | |
| 877 | # Placed at the end to give the GPU node maximum time to provision. | |
| 878 | # Skippable with SKIP_INFERENCE=1 (useful if no GPU quota is available). | |
| 879 | ||
| 880 | 22 | if [ "${SKIP_INFERENCE:-}" != "1" ]; then |
| 881 | ||
| 882 | 40 | SECTION=$((SECTION + 1)); section_header "$SECTION" "INFERENCE — Live LLM on GCO" "$CYAN" |
| 883 | ||
| 884 | 20 | narrate "GCO isn't just for batch jobs — it also manages multi-region" |
| 885 | 20 | narrate "inference endpoints. We deployed a vLLM endpoint at the start" |
| 886 | 20 | narrate "of this demo so the GPU could provision while we covered other" |
| 887 | 20 | narrate "features. Let's see if it's ready." |
| 888 | 20 | spacer |
| 889 | ||
| 890 | 20 | highlight "Checking if the inference endpoint is ready" |
| 891 | 20 | narrate "The endpoint was deployed with a single command at the start." |
| 892 | 20 | narrate "EKS Auto Mode provisioned a GPU node, pulled the vLLM image," |
| 893 | 20 | narrate "and loaded the facebook/opt-125m model — all automatically." |
| 894 | ||
| 895 | # Poll for the endpoint to become ready. Since we deployed it at the start, | |
| 896 | # it's had several minutes to provision. We still poll in case it's not | |
| 897 | # quite ready yet. Ignore Terminating pods from previous runs. | |
| 898 | 20 | INFERENCE_READY=false |
| 899 | 91 | for attempt in $(seq 1 50); do |
| 900 | 214 | POD_STATUS=$(kubectl get pods -n gco-inference -l app="$INFERENCE_NAME" --no-headers 2>/dev/null \ |
| 901 | | grep -v "Terminating" || true) | |
| 902 | 142 | if echo "$POD_STATUS" | grep -q "1/1.*Running"; then |
| 903 | 19 | INFERENCE_READY=true |
| 904 | 19 | break |
| 905 | fi | |
| 906 | ||
| 907 | 52 | if [ -n "$POD_STATUS" ]; then |
| 908 | 51 | echo " ${DIM}$POD_STATUS${RESET}" |
| 909 | else | |
| 910 | 2 | STATUS_OUTPUT=$(gco inference status "$INFERENCE_NAME" 2>&1 || true) |
| 911 | 3 | echo "$STATUS_OUTPUT" | head -5 | sed 's/^/ /' |
| 912 | fi | |
| 913 | ||
| 914 | 52 | if [ "$attempt" -lt 50 ]; then |
| 915 | 51 | countdown "Waiting for pod to be ready (attempt $attempt/50)" 15 |
| 916 | fi | |
| 917 | done | |
| 918 | ||
| 919 | 20 | INFERENCE_INVOKE_OK=0 |
| 920 | 20 | if [ "$INFERENCE_READY" = "true" ]; then |
| 921 | 19 | success "Inference pod is Kubernetes-ready." |
| 922 | 19 | narrate "Kubernetes readiness is local; validating one real generation through the global route." |
| 923 | 19 | if wait_for_inference_generation "$INFERENCE_NAME" 4 10; then |
| 924 | 18 | success "End-to-end global inference route is ready." |
| 925 | 18 | spacer |
| 926 | ||
| 927 | 18 | highlight "Sending a prompt to the endpoint" |
| 928 | 18 | narrate "The shared inference route is already registered on the internal ALB." |
| 929 | 18 | narrate "Requests traverse API Gateway → Global Accelerator → ALB → authenticated proxy → vLLM." |
| 930 | 18 | narrate "API Gateway validates SigV4; the private backend hop uses private-root TLS plus a request-bound HMAC." |
| 931 | 18 | if run_cmd "gco inference invoke $INFERENCE_NAME -p 'The benefits of GPU orchestration for ML workloads are: 1)' --max-tokens 80"; then |
| 932 | 18 | INFERENCE_INVOKE_OK=1 |
| 933 | 18 | sleep "$PAUSE_LONG" |
| 934 | 18 | success "Live LLM response from a GPU that didn't exist minutes ago." |
| 935 | fi | |
| 936 | else | |
| 937 | 1 | warn "The end-to-end inference route did not become ready after 4 attempts." |
| 938 | fi | |
| 939 | else | |
| 940 | 1 | warn "Endpoint did not become Kubernetes-ready within the bounded wait." |
| 941 | fi | |
| 942 | ||
| 943 | 20 | spacer |
| 944 | 20 | highlight "Cleaning up the inference endpoint" |
| 945 | 20 | narrate "This deletes the endpoint Deployment and internal Service. The GPU node" |
| 946 | 20 | narrate "scales back to zero automatically once the pod is gone." |
| 947 | 20 | INFERENCE_DELETE_OK=0 |
| 948 | 20 | if run_cmd "gco inference delete $INFERENCE_NAME -y"; then |
| 949 | 19 | INFERENCE_DELETE_OK=1 |
| 950 | 19 | INFERENCE_CLEANUP_PENDING=0 |
| 951 | 19 | trap - EXIT |
| 952 | fi | |
| 953 | ||
| 954 | 20 | if ! report_inference_lifecycle_result "$INFERENCE_INVOKE_OK" "$INFERENCE_DELETE_OK"; then |
| 955 | 3 | exit 1 |
| 956 | fi | |
| 957 | ||
| 958 | 17 | pause_for_audience |
| 959 | ||
| 960 | fi # SKIP_INFERENCE | |
| 961 | ||
| 962 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 963 | # SECTION: Wrap-up | |
| 964 | # ═════════════════════════════════════════════════════════════════════════════ | |
| 965 | # Summary of everything we covered, plus an optional cleanup step. | |
| 966 | ||
| 967 | 19 | banner "Demo Complete" |
| 968 | ||
| 969 | 19 | echo " ${BOLD}What we covered:${RESET}" |
| 970 | 19 | spacer |
| 971 | 19 | echo " ${GREEN}✓${RESET} Fleet status, cost visibility, and policy agreement" |
| 972 | 19 | if [ "${SKIP_CAPACITY:-}" != "1" ]; then |
| 973 | 17 | echo " ${GREEN}✓${RESET} Capacity discovery and auto-region job placement" |
| 974 | fi | |
| 975 | ||
| 976 | # Only show scheduler items if we didn't skip that section. | |
| 977 | 19 | if [ "${SKIP_SCHEDULERS:-}" != "1" ]; then |
| 978 | 17 | if [ "$VOLCANO_ENABLED" = "true" ]; then |
| 979 | 2 | echo " ${GREEN}✓${RESET} Volcano gang scheduling for distributed training" |
| 980 | fi | |
| 981 | 17 | if [ "$KUEUE_ENABLED" = "true" ]; then |
| 982 | 2 | echo " ${GREEN}✓${RESET} Kueue quota-based job queueing" |
| 983 | fi | |
| 984 | 17 | if [ "$YUNIKORN_ENABLED" = "true" ]; then |
| 985 | 2 | echo " ${GREEN}✓${RESET} YuniKorn app-aware fair scheduling" |
| 986 | fi | |
| 987 | 17 | if [ "$SLURM_ENABLED" = "true" ]; then |
| 988 | 2 | echo " ${GREEN}✓${RESET} Slurm HPC batch scheduling on Kubernetes" |
| 989 | fi | |
| 990 | fi | |
| 991 | ||
| 992 | 19 | if [ "$FSX_ENABLED" = "true" ]; then |
| 993 | 3 | echo " ${GREEN}✓${RESET} FSx for Lustre high-performance storage" |
| 994 | fi | |
| 995 | 19 | if [ "$VALKEY_ENABLED" = "true" ]; then |
| 996 | 4 | echo " ${GREEN}✓${RESET} Valkey serverless in-memory cache" |
| 997 | fi | |
| 998 | 19 | if [ "$AURORA_PGVECTOR_ENABLED" = "true" ]; then |
| 999 | 3 | echo " ${GREEN}✓${RESET} Aurora pgvector serverless vector database" |
| 1000 | fi | |
| 1001 | 19 | if [ "$VECTOR_STORE_ENABLED" = "true" ]; then |
| 1002 | 3 | echo " ${GREEN}✓${RESET} Globally replicated vector store with semantic search" |
| 1003 | fi | |
| 1004 | 19 | echo " ${GREEN}✓${RESET} EFS persistent shared storage" |
| 1005 | 19 | if [ "${SKIP_INFERENCE:-}" != "1" ]; then |
| 1006 | 17 | echo " ${GREEN}✓${RESET} Inference endpoint deploy, invoke, and teardown" |
| 1007 | fi | |
| 1008 | ||
| 1009 | 19 | spacer |
| 1010 | 19 | echo " ${BOLD}All of this runs on a single platform, deployed with one command:${RESET}" |
| 1011 | 19 | echo " ${CYAN}gco stacks deploy-all -y${RESET}" |
| 1012 | 19 | spacer |
| 1013 | 19 | echo " ${DIM}Repository: https://github.com/aws-solutions-library-samples/global-capacity-orchestrator-on-aws${RESET}" |
| 1014 | 19 | spacer |
| 1015 | ||
| 1016 | # ── Cleanup Prompt ─────────────────────────────────────────────────────────── | |
| 1017 | # Offer to delete all the demo jobs we just created. This keeps the cluster | |
| 1018 | # clean for the next demo run. | |
| 1019 | ||
| 1020 | 19 | echo " ${YELLOW}${BOLD}Clean up demo jobs?${RESET} ${DIM}(y/N)${RESET}" |
| 1021 | 19 | if [ "${GCO_DEMO_NONINTERACTIVE:-}" = "1" ]; then |
| 1022 | 18 | cleanup="n" |
| 1023 | else | |
| 1024 | 1 | read -r cleanup |
| 1025 | fi | |
| 1026 | 19 | case "$cleanup" in |
| 1027 | y|Y) | |
| 1028 | 1 | narrate "Cleaning up demo jobs..." |
| 1029 | # Delete jobs by label (covers Volcano, FSx, EFS jobs with project=gco label) | |
| 1030 | 1 | kubectl delete job -n gco-jobs -l project=gco --ignore-not-found=true 2>/dev/null || true |
| 1031 | # Delete specific jobs that may not have the project label | |
| 1032 | 1 | kubectl delete job -n gco-jobs efs-output-example --ignore-not-found=true 2>/dev/null || true |
| 1033 | 1 | kubectl delete job -n gco-jobs valkey-cache-example --ignore-not-found=true 2>/dev/null || true |
| 1034 | 1 | kubectl delete job -n gco-jobs aurora-pgvector-example --ignore-not-found=true 2>/dev/null || true |
| 1035 | 1 | kubectl delete job -n gco-jobs kueue-sample-job --ignore-not-found=true 2>/dev/null || true |
| 1036 | 1 | kubectl delete job -n gco-jobs kueue-gpu-job --ignore-not-found=true 2>/dev/null || true |
| 1037 | 1 | kubectl delete job -n gco-jobs yunikorn-sample-job --ignore-not-found=true 2>/dev/null || true |
| 1038 | 1 | kubectl delete job -n gco-jobs yunikorn-gpu-job --ignore-not-found=true 2>/dev/null || true |
| 1039 | 1 | kubectl delete job -n gco-jobs yunikorn-gang-job --ignore-not-found=true 2>/dev/null || true |
| 1040 | 1 | kubectl delete job -n gco-jobs slurm-test --ignore-not-found=true 2>/dev/null || true |
| 1041 | # Volcano uses a custom resource type (vcjob), not a standard Job | |
| 1042 | 1 | kubectl delete vcjob -n gco-jobs distributed-training --ignore-not-found=true 2>/dev/null || true |
| 1043 | # Clean up any downloaded files | |
| 1044 | 1 | rm -rf /tmp/gco-demo-results 2>/dev/null || true |
| 1045 | 1 | success "Demo jobs cleaned up." |
| 1046 | ;; | |
| 1047 | *) | |
| 1048 | 18 | narrate "Skipping cleanup. Remove jobs manually with:" |
| 1049 | 18 | echo " ${CYAN}kubectl delete jobs --all -n gco-jobs${RESET}" |
| 1050 | ;; | |
| 1051 | esac | |
| 1052 | ||
| 1053 | 19 | spacer |
| 1054 | 19 | echo " ${DIM}Thanks for watching.${RESET}" |
| 1055 | 19 | spacer |