Coverage for cli / commands / stacks_cmd.py: 100.00%
940 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-09-14 22:07 +0000
1"""Stack deployment and management commands."""
3import re
4import sys
5from collections.abc import Mapping
6from typing import Any
8import click
10from ..config import GCOConfig, _load_cdk_json
11from ..output import confirm, get_output_formatter, interactive_echo
13pass_config = click.make_pass_decorator(GCOConfig, ensure=True)
15_ENABLE_OPTION_HELP = (
16 "Force-enable an off-by-default feature or Helm chart for this run only "
17 "(repeatable, or comma-separated). Threads the request through CDK context "
18 "rather than rewriting cdk.json, so the committed opt-in defaults survive."
19)
22def _validate_enable(
23 ctx: click.Context, param: click.Parameter, value: tuple[str, ...]
24) -> dict[str, str]:
25 """Resolve ``--enable`` names into CDK context pairs at parse time.
27 Validating in the callback means a typo fails before any AWS call and
28 before the command's own ``try`` block, so an unknown name reports as a
29 bad parameter instead of masquerading as a deployment failure.
30 """
31 from gco.enablement_overrides import EnablementOverrideError, route_enablement_overrides
33 try:
34 return route_enablement_overrides(value)
35 except EnablementOverrideError as exc:
36 raise click.BadParameter(str(exc)) from exc
39def _apply_enable_overrides(formatter: Any, manager: Any, enable: Mapping[str, str]) -> None:
40 """Register run-scoped enablement context and disclose it before mutating AWS.
42 The context rides every CDK invocation of the command so ``list``, ``synth``,
43 and the lifecycle call all evaluate the same app.
45 On ``destroy`` this is about synth symmetry, not about what gets deleted:
46 ``cdk destroy`` issues a CloudFormation ``DeleteStack``, which removes
47 whatever the *deployed* template contains, and none of the override names
48 gates a whole stack. The hazard runs the other way — see the ``--enable``
49 warning in docs/CUSTOMIZATION.md: re-deploying *without* the overrides
50 synthesizes a template that no longer declares the forced-on resources, and
51 CloudFormation deletes them as an ordinary stack update.
52 """
53 if not enable:
54 return
55 manager.set_extra_cdk_context(dict(enable))
56 for key, value in sorted(enable.items()):
57 formatter.print_info(f"Run-scoped override: {key}={value}")
60@click.group()
61@pass_config
62def stacks(config: Any) -> None:
63 """Deploy and manage GCO CDK stacks."""
64 pass
67@stacks.command("list")
68@click.option(
69 "--refresh",
70 is_flag=True,
71 help="Compatibility flag; stack discovery already runs live",
72)
73@pass_config
74def list_stacks(config: Any, refresh: Any) -> None:
75 """List stacks synthesized by the local CDK app."""
76 from ..stacks import get_stack_manager
78 formatter = get_output_formatter(config)
80 try:
81 manager = get_stack_manager(config)
82 if refresh:
83 formatter.print_info(
84 "Stack discovery runs live on every invocation; --refresh is retained "
85 "for compatibility."
86 )
87 local_stacks = manager.list_stacks()
89 formatter.print_info("Available CDK stacks:")
90 for stack in local_stacks:
91 print(f" - {stack}")
93 except Exception as e:
94 formatter.print_error(f"Failed to list stacks: {e}")
95 sys.exit(1)
98@stacks.command("synth")
99@click.argument("stack_name", required=False)
100@click.option("--quiet", "-q", is_flag=True, default=True, help="Quiet output")
101@pass_config
102def synth_stack(config: Any, stack_name: Any, quiet: Any) -> None:
103 """Synthesize CloudFormation templates."""
104 from ..stacks import get_stack_manager
106 formatter = get_output_formatter(config)
108 try:
109 manager = get_stack_manager(config)
110 output = manager.synth(stack_name, quiet=quiet)
111 if output:
112 print(output)
113 formatter.print_success("CDK synthesis completed")
114 except Exception as e:
115 formatter.print_error(f"CDK synth failed: {e}")
116 sys.exit(1)
119@stacks.command("diff")
120@click.argument("stack_name", required=False)
121@pass_config
122def diff_stack(config: Any, stack_name: Any) -> None:
123 """Show differences between deployed and local stacks."""
124 from ..stacks import get_stack_manager
126 formatter = get_output_formatter(config)
128 try:
129 manager = get_stack_manager(config)
130 diff_output = manager.diff(stack_name)
131 if diff_output:
132 print(diff_output)
133 else:
134 formatter.print_success("No differences found")
135 except Exception as e:
136 formatter.print_error(f"CDK diff failed: {e}")
137 sys.exit(1)
140def _print_cluster_access_hint(formatter: Any, config: Any, stack_name: str) -> None:
141 """Point at the cluster-access commands after a regional deploy.
143 Reaching the cluster API needs an access entry (authn/authz) on top of
144 endpoint reachability, and neither is discoverable from the deploy
145 output alone — the misdirection that stretched the original outage's
146 diagnosis. Printed only for base regional stacks (the ones that own an
147 EKS cluster).
148 """
149 prefix = f"{config.project_name}-"
150 if not stack_name.startswith(prefix):
151 return
152 suffix = stack_name[len(prefix) :]
153 if (
154 not suffix
155 or suffix in ("global", "api-gateway", "monitoring")
156 or suffix.startswith("regional-api")
157 ):
158 return
159 formatter.print_info(
160 f"kubectl access to {stack_name}: run 'gco stacks access -r {suffix}' to create "
161 "your EKS access entry (required even over a tunnel). Private endpoint? "
162 "'gco cluster tunnel --via-ssm auto' reaches it over SSM; 'gco cluster doctor' "
163 "diagnoses reachability, authentication, and authorization separately."
164 )
167@stacks.command("deploy")
168@click.argument("stack_name")
169@click.option("--yes", "-y", is_flag=True, help="Skip approval prompts")
170@click.option("--outputs-file", "-o", help="Write outputs to file")
171@click.option("--tag", "-t", multiple=True, help="Add tags (key=value)")
172@click.option(
173 "--enable",
174 "enable",
175 multiple=True,
176 metavar="NAME[,NAME...]",
177 callback=_validate_enable,
178 help=_ENABLE_OPTION_HELP,
179)
180@pass_config
181def deploy_stack(
182 config: Any,
183 stack_name: Any,
184 yes: Any,
185 outputs_file: Any,
186 tag: Any,
187 enable: Mapping[str, str],
188) -> None:
189 """Deploy a single CDK stack to AWS.
191 For deploying all stacks in the correct order, use 'deploy-all'.
193 Examples:
194 gco stacks deploy gco-us-east-1
195 gco stacks deploy gco-global -y
196 gco stacks deploy gco-us-east-1 -t Environment=prod
197 gco stacks deploy gco-us-east-1 -y --enable fsx_lustre,valkey
199 --enable is scoped to this one stack, and no override name is confined to a
200 single stack (vector_store's table lives in the global stack; FSx, Valkey,
201 and Aurora add monitoring-stack dashboard widgets). A single-stack override
202 therefore deploys a partially wired feature without complaining. Prefer
203 'deploy-all --enable' unless you specifically want that.
204 """
205 from ..stacks import get_stack_manager
207 formatter = get_output_formatter(config)
209 # Parse tags
210 tags = {}
211 for t in tag:
212 if "=" in t:
213 k, v = t.split("=", 1)
214 tags[k] = v
216 try:
217 manager = get_stack_manager(config)
218 _apply_enable_overrides(formatter, manager, enable)
220 formatter.print_info(f"Deploying {stack_name}...")
222 success = manager.deploy(
223 stack_name=stack_name,
224 require_approval=not yes,
225 outputs_file=outputs_file,
226 tags=tags if tags else None,
227 )
229 if success:
230 formatter.print_success("Deployment completed successfully")
231 _print_cluster_access_hint(formatter, config, str(stack_name))
232 else:
233 formatter.print_error("Deployment failed")
234 sys.exit(1)
236 except Exception as e:
237 formatter.print_error(f"Deployment failed: {e}")
238 sys.exit(1)
241@stacks.command("destroy")
242@click.argument("stack_name")
243@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
244@click.option(
245 "--retain-volumes",
246 is_flag=True,
247 help="Report the cluster's orphaned EBS volumes instead of deleting them",
248)
249@click.option(
250 "--enable",
251 "enable",
252 multiple=True,
253 metavar="NAME[,NAME...]",
254 callback=_validate_enable,
255 help=_ENABLE_OPTION_HELP,
256)
257@pass_config
258def destroy_stack(
259 config: Any,
260 stack_name: Any,
261 yes: Any,
262 retain_volumes: Any,
263 enable: Mapping[str, str],
264) -> None:
265 """Destroy a single CDK stack.
267 For destroying all stacks in the correct order, use 'destroy-all'.
269 For a regional stack this also deletes the EBS volumes the cluster's CSI
270 driver provisioned for in-cluster PVCs (Prometheus, Grafana, Alertmanager,
271 MLflow). Deleting an EKS cluster does not delete them, so they would
272 otherwise remain billable forever with nothing able to reattach them. Pass
273 --retain-volumes to list them instead of deleting them.
275 Examples:
276 gco stacks destroy gco-us-east-1
277 gco stacks destroy gco-us-east-1 -y
278 gco stacks destroy gco-us-east-1 -y --retain-volumes
279 gco stacks destroy gco-us-east-1 -y --enable fsx_lustre,valkey
280 """
281 from ..stacks import get_stack_manager
283 formatter = get_output_formatter(config)
285 if not yes:
286 confirm(f"Are you sure you want to destroy {stack_name}?", abort=True)
288 try:
289 manager = get_stack_manager(config)
290 _apply_enable_overrides(formatter, manager, enable)
292 formatter.print_info(f"Destroying {stack_name}...")
294 success = manager.destroy(
295 stack_name=stack_name,
296 force=yes,
297 )
299 if success:
300 formatter.print_success(f"Stack {stack_name} destroyed successfully")
301 else:
302 formatter.print_error("Destroy failed")
303 sys.exit(1)
305 # Unlike destroy-all, this path has no orchestrated cleanup barrier, so
306 # the cluster's dynamically provisioned volumes are swept here. Runs only
307 # after a reported success, and the sweep itself re-proves the cluster is
308 # absent before touching anything (#268).
309 manager.cleanup_cluster_volumes(stack_name, retain=retain_volumes)
311 except Exception as e:
312 formatter.print_error(f"Destroy failed: {e}")
313 sys.exit(1)
316@stacks.command("deploy-all")
317@click.option("--yes", "-y", is_flag=True, help="Skip approval prompts")
318@click.option("--outputs-file", "-o", help="Write outputs to file")
319@click.option("--tag", "-t", multiple=True, help="Add tags (key=value)")
320@click.option("--parallel", "-p", is_flag=True, help="Deploy regional stacks in parallel")
321@click.option("--max-workers", "-w", default=4, help="Max parallel deployments (default: 4)")
322@click.option(
323 "--enable",
324 "enable",
325 multiple=True,
326 metavar="NAME[,NAME...]",
327 callback=_validate_enable,
328 help=_ENABLE_OPTION_HELP,
329)
330@pass_config
331def deploy_all_orchestrated(
332 config: Any,
333 yes: Any,
334 outputs_file: Any,
335 tag: Any,
336 parallel: Any,
337 max_workers: Any,
338 enable: Mapping[str, str],
339) -> None:
340 """Deploy all stacks in the correct order.
342 Deploys in three phases:
343 1. Global stacks (gco-global, gco-api-gateway)
344 2. Regional stacks (gco-us-east-1, etc.) - can be parallelized
345 3. Monitoring stack (gco-monitoring) - depends on regional stacks
347 Use --parallel to deploy regional stacks concurrently, which can
348 significantly reduce total deployment time when deploying to
349 multiple regions.
351 Examples:
352 gco stacks deploy-all -y
353 gco stacks deploy-all -y --parallel
354 gco stacks deploy-all -y -p --max-workers 8
355 gco stacks deploy-all -y -t Environment=prod
356 gco stacks deploy-all -y --enable fsx_lustre,valkey,aurora_pgvector,slurm,yunikorn
357 """
358 from ..stacks import get_stack_manager
360 formatter = get_output_formatter(config)
362 # Parse tags
363 tags = {}
364 for t in tag:
365 if "=" in t:
366 k, v = t.split("=", 1)
367 tags[k] = v
369 try:
370 manager = get_stack_manager(config)
371 _apply_enable_overrides(formatter, manager, enable)
372 stacks = manager.list_stacks()
374 formatter.print_info(f"Found {len(stacks)} stacks to deploy")
375 if parallel:
376 formatter.print_info(f"Parallel mode enabled (max workers: {max_workers})")
378 def on_start(stack_name: str) -> None:
379 formatter.print_info(f"Deploying {stack_name}...")
381 def on_complete(stack_name: str, success: bool) -> None:
382 if success:
383 formatter.print_success(f" ✓ {stack_name} deployed")
384 else:
385 formatter.print_error(f" ✗ {stack_name} failed")
387 success, successful, failed = manager.deploy_orchestrated(
388 require_approval=not yes,
389 outputs_file=outputs_file,
390 tags=tags if tags else None,
391 on_stack_start=on_start,
392 on_stack_complete=on_complete,
393 parallel=parallel,
394 max_workers=max_workers,
395 )
397 formatter.print_info("")
398 formatter.print_info(f"Deployed: {len(successful)}/{len(stacks)} stacks")
400 if success:
401 formatter.print_success("All stacks deployed successfully")
402 else:
403 formatter.print_error(f"Deployment failed. Failed stacks: {', '.join(failed)}")
404 sys.exit(1)
406 except Exception as e:
407 formatter.print_error(f"Deployment failed: {e}")
408 sys.exit(1)
411@stacks.command("destroy-all")
412@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
413@click.option("--parallel", "-p", is_flag=True, help="Destroy regional stacks in parallel")
414@click.option("--max-workers", "-w", default=4, help="Max parallel destructions (default: 4)")
415@click.option(
416 "--retain-volumes",
417 is_flag=True,
418 help="Report each cluster's orphaned EBS volumes instead of deleting them",
419)
420@click.option(
421 "--enable",
422 "enable",
423 multiple=True,
424 metavar="NAME[,NAME...]",
425 callback=_validate_enable,
426 help=_ENABLE_OPTION_HELP,
427)
428@pass_config
429def destroy_all_orchestrated(
430 config: Any,
431 yes: Any,
432 parallel: Any,
433 max_workers: Any,
434 retain_volumes: Any,
435 enable: Mapping[str, str],
436) -> None:
437 """Destroy all stacks in the correct order.
439 Destroys in four dependency phases:
440 1. Monitoring stack (<project>-monitoring)
441 2. Regional API bridges (<project>-regional-api-<region>)
442 3. Base regional stacks (<project>-<region>) - can be parallelized
443 4. Global stacks (<project>-api-gateway, <project>-global)
445 Automatically retries up to 3 times (with 30s waits) if any stacks fail,
446 which handles transient issues like orphaned resources during teardown.
448 Once every regional stack is gone this deletes the EBS volumes their
449 clusters' CSI drivers provisioned for in-cluster PVCs, which CloudFormation
450 does not own and deleting an EKS cluster does not remove. Pass
451 --retain-volumes to list them instead of deleting them.
453 After a fully successful teardown this also purges the runtime
454 /{project}/traffic-dial SSM parameters (controller state and manual
455 overrides), which are written outside CloudFormation.
457 Use --parallel to destroy regional stacks concurrently, which can
458 significantly reduce total teardown time when destroying multiple
459 regional stacks.
461 Examples:
462 gco stacks destroy-all -y
463 gco stacks destroy-all -y --parallel
464 gco stacks destroy-all -y -p --max-workers 8
465 gco stacks destroy-all -y --enable fsx_lustre,valkey,aurora_pgvector,slurm,yunikorn
466 """
467 import time
469 from ..stacks import get_stack_destroy_order, get_stack_manager
471 formatter = get_output_formatter(config)
472 # Retry up to 3 times total. CloudFormation stack deletions can fail
473 # transiently — e.g., EKS leaves behind a cluster security group that
474 # blocks VPC deletion, but it gets cleaned up async. A 30-second wait
475 # between attempts is usually enough for the orphaned resources to clear.
476 max_attempts = 3
478 try:
479 manager = get_stack_manager(config)
480 _apply_enable_overrides(formatter, manager, enable)
481 stacks = manager.list_stacks()
482 ordered = get_stack_destroy_order(
483 stacks,
484 project_name=config.project_name,
485 )
487 if not yes:
488 formatter.print_warning("This will destroy ALL GCO stacks:")
489 for stack in ordered:
490 if config.output_format == "table":
491 formatter.print_info(f" - {stack}")
492 else:
493 interactive_echo(f" - {stack}")
494 confirm("\nAre you sure you want to destroy all stacks?", abort=True)
496 total_stacks = len(stacks)
498 for attempt in range(1, max_attempts + 1):
499 if attempt > 1:
500 # Inspect each regional VPC for resources that block teardown
501 # (the EKS cluster security group EKS leaves behind, plus any
502 # lingering ENIs from ELB / Global Accelerator), clear what's
503 # safe to remove, and report what the next attempt is waiting
504 # on. The service-managed ENIs drain asynchronously, which is
505 # what the 30s wait is for.
506 formatter.print_info(
507 "Inspecting VPCs for resources that can block teardown "
508 "(orphaned ENIs, EKS security groups)..."
509 )
510 manager.cleanup_orphaned_network_interfaces()
511 formatter.print_warning(
512 f"Attempt {attempt}/{max_attempts}: waiting 30 seconds before retrying..."
513 )
514 time.sleep(30)
516 formatter.print_info(f"Destroying {len(stacks)} stacks...")
517 if parallel:
518 formatter.print_info(f"Parallel mode enabled (max workers: {max_workers})")
520 def on_start(stack_name: str) -> None:
521 formatter.print_info(f"Destroying {stack_name}...")
523 def on_complete(stack_name: str, success: bool) -> None:
524 if success:
525 formatter.print_success(f" ✓ {stack_name} destroyed")
526 else:
527 formatter.print_error(f" ✗ {stack_name} failed")
529 success, successful, failed = manager.destroy_orchestrated(
530 force=True,
531 on_stack_start=on_start,
532 on_stack_complete=on_complete,
533 parallel=parallel,
534 max_workers=max_workers,
535 retain_volumes=retain_volumes,
536 )
538 if success:
539 break
541 if attempt < max_attempts:
542 formatter.print_warning(f"{len(failed)} stack(s) failed: {', '.join(failed)}")
544 formatter.print_info("")
545 formatter.print_info(f"Destroyed: {total_stacks - len(failed)}/{total_stacks} stacks")
547 if success:
548 formatter.print_success("All stacks destroyed successfully")
549 else:
550 formatter.print_error(f"Some stacks failed to destroy: {', '.join(failed)}")
551 sys.exit(1)
553 except Exception as e:
554 formatter.print_error(f"Destroy failed: {e}")
555 sys.exit(1)
558@stacks.command("bootstrap")
559@click.option("--account", "-a", help="AWS account ID")
560@click.option("--region", "-r", required=True, help="AWS region")
561@pass_config
562def bootstrap_cdk(config: Any, account: Any, region: Any) -> None:
563 """Bootstrap CDK in an AWS account/region.
565 This is required before deploying stacks to a new account/region.
567 Example:
568 gco stacks bootstrap --region us-east-1
569 gco stacks bootstrap -a 123456789012 -r eu-west-1
570 """
571 from ..stacks import get_stack_manager
573 formatter = get_output_formatter(config)
575 try:
576 manager = get_stack_manager(config)
577 formatter.print_info(f"Bootstrapping CDK in {region}...")
579 success = manager.bootstrap(account=account, region=region)
581 if success:
582 formatter.print_success(f"CDK bootstrapped in {region}")
583 else:
584 formatter.print_error("Bootstrap failed")
585 sys.exit(1)
587 except Exception as e:
588 formatter.print_error(f"Bootstrap failed: {e}")
589 sys.exit(1)
592def _print_eks_endpoint_drift(formatter: Any, config: Any, stack_name: str, region: str) -> None:
593 """Report configured-vs-live EKS endpoint drift for a regional stack.
595 An endpoint flip that was configured (``gco stacks eks endpoint set``)
596 but not deployed — or applied out-of-band and never written back to
597 cdk.json — must be visible in ``gco stacks status`` rather than
598 silently diverging. Best-effort: any probe or config-read failure
599 skips the drift report, never the status output.
600 """
601 if stack_name != f"{config.project_name}-{region}":
602 return # Only base regional stacks own an EKS cluster.
603 try:
604 from ..cluster_doctor import endpoint_drift
605 from ..kubectl_helpers import describe_cluster_access
606 from ..stacks import get_eks_cluster_config
608 eks_config = get_eks_cluster_config()
609 live = describe_cluster_access(stack_name, region)
610 drift = endpoint_drift(
611 str(eks_config.get("endpoint_access", "PRIVATE")),
612 [str(cidr) for cidr in eks_config.get("public_access_cidrs") or []],
613 live,
614 )
615 except Exception:
616 return
617 if drift:
618 formatter.print_warning(
619 f"Config drift: {drift}. Run 'gco stacks deploy {stack_name} -y' to "
620 "converge the endpoint, or update cdk.json to match what is deployed."
621 )
624@stacks.command("status")
625@click.argument("stack_name")
626@click.option("--region", "-r", required=True, help="AWS region")
627@pass_config
628def stack_status(config: Any, stack_name: Any, region: Any) -> None:
629 """Get detailed status of a deployed stack.
631 For a base regional stack this also compares the configured EKS endpoint
632 access (cdk.json eks_cluster) against the live cluster and reports any
633 drift.
634 """
635 from ..stacks import get_stack_manager
637 formatter = get_output_formatter(config)
639 try:
640 manager = get_stack_manager(config)
641 status = manager.get_stack_status(stack_name, region)
643 if status:
644 formatter.print(status.to_dict())
645 _print_eks_endpoint_drift(formatter, config, str(stack_name), str(region))
646 else:
647 formatter.print_error(f"Stack {stack_name} not found in {region}")
648 sys.exit(1)
650 except Exception as e:
651 formatter.print_error(f"Failed to get stack status: {e}")
652 sys.exit(1)
655@stacks.command("outputs")
656@click.argument("stack_name")
657@click.option("--region", "-r", required=True, help="AWS region")
658@pass_config
659def stack_outputs(config: Any, stack_name: Any, region: Any) -> None:
660 """Get outputs from a deployed stack."""
661 from ..stacks import get_stack_manager
663 formatter = get_output_formatter(config)
665 try:
666 manager = get_stack_manager(config)
667 outputs = manager.get_outputs(stack_name, region)
669 if outputs:
670 formatter.print(outputs)
671 else:
672 formatter.print_warning(f"No outputs found for {stack_name}")
674 except Exception as e:
675 formatter.print_error(f"Failed to get outputs: {e}")
676 sys.exit(1)
679@stacks.command("access")
680@click.option("--cluster", "-c", help="Cluster name (default: <project_name>-<region>)")
681@click.option("--region", "-r", help="AWS region (default: first deployment region)")
682@pass_config
683def setup_access(config: Any, cluster: Any, region: Any) -> None:
684 """Configure kubectl access to a GCO EKS cluster.
686 Updates kubeconfig, creates an EKS access entry for your IAM principal,
687 and associates the cluster admin policy. Handles assumed roles automatically.
689 Examples:
690 gco stacks access
691 gco stacks access -r us-west-2
692 gco stacks access -c my-cluster -r eu-west-1
693 """
694 import subprocess
696 from .._image_uri import aws_partition
697 from ..config import _load_cdk_json
699 formatter = get_output_formatter(config)
701 # Determine region
702 if not region:
703 cdk_regions = _load_cdk_json()
704 if cdk_regions and "regional" in cdk_regions:
705 region = cdk_regions["regional"][0]
706 else:
707 region = config.default_region or "us-east-1"
709 partition = aws_partition(str(region))
711 # Determine cluster name
712 if not cluster:
713 cluster = f"{config.project_name}-{region}"
715 formatter.print_info(f"Setting up access to cluster: {cluster} in region: {region}")
717 # Cluster endpoint access mode — warn early if the API server is
718 # private-only, since every kubectl call from outside the VPC will
719 # fail. We still try every step so the access entry + policy
720 # association land (those use the EKS control plane via boto3,
721 # which doesn't go through the cluster endpoint), but the verify
722 # step at the end will hit a connection timeout from the laptop.
723 private_endpoint_only = False
724 public_cidrs: list[str] = []
725 try:
726 endpoint_check = subprocess.run(
727 [
728 "aws",
729 "eks",
730 "describe-cluster",
731 "--name",
732 cluster,
733 "--region",
734 region,
735 "--query",
736 # Explicit ``+`` rather than implicit string concatenation
737 # so static analysers don't flag the multi-line literal as
738 # a possibly-missing comma between two list elements. The
739 # value is one JMESPath expression passed as a single
740 # ``--query`` argument.
741 "cluster.resourcesVpcConfig.{public:endpointPublicAccess,"
742 + "private:endpointPrivateAccess,publicCidrs:publicAccessCidrs}",
743 "--output",
744 "json",
745 ],
746 check=True,
747 capture_output=True,
748 text=True,
749 )
750 import json
752 endpoint_cfg = json.loads(endpoint_check.stdout or "{}")
753 is_public = bool(endpoint_cfg.get("public"))
754 public_cidrs = endpoint_cfg.get("publicCidrs") or []
755 if not is_public:
756 private_endpoint_only = True
757 formatter.print_warning(
758 f"Cluster {cluster!r} has endpointPublicAccess=false — kubectl from "
759 "outside the VPC will not be able to reach the API server. The access "
760 "entry and policy association below still apply, but the verify step "
761 "at the end will time out from this host."
762 )
763 formatter.print_warning(
764 "To enable kubectl from your laptop or CI runner, set "
765 '``eks_cluster.endpoint_access`` to ``"PUBLIC_AND_PRIVATE"`` in '
766 "``cdk.json`` and redeploy the regional stack: ``gco stacks deploy "
767 f"{config.project_name}-{region} -y``."
768 )
769 elif public_cidrs:
770 # Public access is on but restricted to a CIDR allowlist — the
771 # caller's IP may or may not be in it.
772 formatter.print_info(
773 "Cluster API endpoint is public+private with a CIDR allowlist; "
774 f"verify your egress IP is covered by one of: {', '.join(public_cidrs)}"
775 )
776 except (subprocess.CalledProcessError, FileNotFoundError) as exc:
777 # Don't block setup if describe-cluster fails — the access steps
778 # below may still succeed (e.g. for a brand new cluster the caller
779 # already has permission to update).
780 formatter.print_info(f"Could not determine endpoint access mode: {exc}")
782 try:
783 # Step 1: Update kubeconfig
784 formatter.print_info("Updating kubeconfig...")
785 subprocess.run(
786 ["aws", "eks", "update-kubeconfig", "--name", cluster, "--region", region],
787 check=True,
788 capture_output=True,
789 text=True,
790 )
792 # Step 2: Get IAM principal
793 formatter.print_info("Getting your IAM principal...")
794 result = subprocess.run(
795 ["aws", "sts", "get-caller-identity", "--query", "Arn", "--output", "text"],
796 check=True,
797 capture_output=True,
798 text=True,
799 )
800 principal_arn = result.stdout.strip()
801 formatter.print_info(f"Principal: {principal_arn}")
803 # Handle assumed roles — extract the role ARN from the assumed-role ARN
804 if ":assumed-role/" in principal_arn:
805 import re
807 role_name = re.search(r":assumed-role/([^/]+)/", principal_arn)
808 if role_name:
809 account_result = subprocess.run(
810 [
811 "aws",
812 "sts",
813 "get-caller-identity",
814 "--query",
815 "Account",
816 "--output",
817 "text",
818 ],
819 check=True,
820 capture_output=True,
821 text=True,
822 )
823 account_id = account_result.stdout.strip()
824 principal_arn = f"arn:{partition}:iam::{account_id}:role/{role_name.group(1)}"
825 formatter.print_info(f"Using role ARN: {principal_arn}")
827 # Step 3: Create access entry
828 formatter.print_info("Creating EKS access entry...")
829 try:
830 subprocess.run(
831 [
832 "aws",
833 "eks",
834 "create-access-entry",
835 "--cluster-name",
836 cluster,
837 "--region",
838 region,
839 "--principal-arn",
840 principal_arn,
841 ],
842 check=True,
843 capture_output=True,
844 text=True,
845 )
846 except subprocess.CalledProcessError:
847 formatter.print_info("Access entry may already exist")
849 # Step 4: Associate admin policy
850 formatter.print_info("Associating cluster admin policy...")
851 try:
852 subprocess.run(
853 [
854 "aws",
855 "eks",
856 "associate-access-policy",
857 "--cluster-name",
858 cluster,
859 "--region",
860 region,
861 "--principal-arn",
862 principal_arn,
863 "--policy-arn",
864 f"arn:{partition}:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy",
865 "--access-scope",
866 "type=cluster",
867 ],
868 check=True,
869 capture_output=True,
870 text=True,
871 )
872 except subprocess.CalledProcessError:
873 formatter.print_info("Policy may already be associated")
875 # Step 5: Verify access
876 formatter.print_info("Waiting for permissions to propagate...")
877 import time
879 time.sleep(10)
881 result = subprocess.run(
882 ["kubectl", "get", "nodes", "--request-timeout=10s"],
883 capture_output=True,
884 text=True,
885 )
886 if result.returncode == 0:
887 node_count = len(
888 [line for line in result.stdout.strip().split("\n")[1:] if line.strip()]
889 )
890 print(result.stdout)
891 formatter.print_info(f"Access configured successfully. {node_count} node(s) ready.")
892 elif private_endpoint_only:
893 # Don't double-warn — we already explained this above. Just
894 # restate the fix so the operator doesn't have to scroll up.
895 formatter.print_warning(
896 "kubectl could not reach the API server, as expected for a "
897 "private-only cluster from outside the VPC. The IAM access entry "
898 "and admin policy association above did succeed, so kubectl will "
899 "work from inside the VPC (e.g. SSM Session Manager into a node) "
900 "or after redeploying with endpoint_access=PUBLIC_AND_PRIVATE."
901 )
902 else:
903 stderr = (result.stderr or "").strip()
904 # When the laptop's egress IP isn't in the CIDR allowlist, AWS
905 # returns the API server endpoint but kubectl times out at the
906 # TLS handshake. Surface the same actionable hint as the
907 # private-only case.
908 looks_like_network_block = (
909 "i/o timeout" in stderr
910 or "no route to host" in stderr
911 or "connection refused" in stderr
912 or "dial tcp" in stderr
913 )
914 if looks_like_network_block:
915 formatter.print_warning(
916 "kubectl could not reach the API server. If the cluster's "
917 "endpoint_access is restricted to a CIDR allowlist, confirm "
918 "your egress IP is covered, or set endpoint_access to "
919 '"PUBLIC_AND_PRIVATE" in cdk.json and run: gco stacks deploy '
920 f"{config.project_name}-{region} -y"
921 )
922 else:
923 formatter.print_warning(
924 "kubectl connected but no nodes found (cluster may be scaling to zero)"
925 )
927 except subprocess.CalledProcessError as e:
928 formatter.print_error(f"Command failed: {e.stderr or e.stdout or str(e)}")
929 sys.exit(1)
930 except FileNotFoundError as e:
931 formatter.print_error(f"Required tool not found: {e}")
932 sys.exit(1)
933 except Exception as e:
934 formatter.print_error(f"Failed to set up access: {e}")
935 sys.exit(1)
938# =============================================================================
939# EKS access configuration commands
940# =============================================================================
943_CIDR_RE = re.compile(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})$")
946def _valid_cidr(value: str) -> bool:
947 """True for a syntactically valid IPv4 CIDR (octets 0-255, prefix 0-32)."""
948 match = _CIDR_RE.match(value)
949 if not match:
950 return False
951 octets = [int(part) for part in match.groups()[:4]]
952 prefix = int(match.group(5))
953 return all(octet <= 255 for octet in octets) and prefix <= 32
956@stacks.group("eks")
957@pass_config
958def eks_cmd(config: Any) -> None:
959 """EKS cluster access configuration (cdk.json, synth-time only)."""
962@eks_cmd.group("endpoint")
963@pass_config
964def eks_endpoint_cmd(config: Any) -> None:
965 """EKS API endpoint access mode and CIDR allowlist."""
968@eks_endpoint_cmd.command("set")
969@click.argument("mode", type=click.Choice(["PRIVATE", "PUBLIC_AND_PRIVATE"], case_sensitive=False))
970@click.option(
971 "--cidr",
972 "cidrs",
973 multiple=True,
974 metavar="CIDR",
975 help=(
976 "Public-endpoint CIDR allowlist entry (repeatable). Required for "
977 "PUBLIC_AND_PRIVATE — widening access without an explicit allowlist is refused."
978 ),
979)
980@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
981@pass_config
982def eks_endpoint_set(config: Any, mode: str, cidrs: tuple[str, ...], yes: bool) -> None:
983 """Set the EKS API endpoint access mode in cdk.json (audited, config only).
985 Synth-time only: nothing changes on AWS until 'gco stacks deploy'. Setting
986 PUBLIC_AND_PRIVATE requires at least one --cidr — opening the control
987 plane to 0.0.0.0/0 must be spelled out explicitly (--cidr 0.0.0.0/0), never
988 implied. The configured value appears in 'gco stacks status' as config
989 drift until the deploy converges the live endpoint.
991 Examples:
992 gco stacks eks endpoint set PUBLIC_AND_PRIVATE --cidr 203.0.113.7/32
993 gco stacks eks endpoint set PRIVATE -y
994 """
995 formatter = get_output_formatter(config)
996 normalized_mode = mode.upper()
998 if normalized_mode == "PUBLIC_AND_PRIVATE" and not cidrs:
999 formatter.print_error(
1000 "Refusing to widen the EKS API endpoint without an explicit CIDR "
1001 "allowlist. Pass at least one --cidr (e.g. --cidr 203.0.113.7/32); "
1002 "an internet-open endpoint must be spelled out as --cidr 0.0.0.0/0."
1003 )
1004 sys.exit(1)
1006 invalid = [cidr for cidr in cidrs if not _valid_cidr(cidr)]
1007 if invalid:
1008 formatter.print_error(
1009 f"Invalid CIDR(s): {', '.join(invalid)} (expected e.g. 203.0.113.0/24)"
1010 )
1011 sys.exit(1)
1013 summary = f"eks_cluster.endpoint_access -> {normalized_mode}"
1014 if cidrs:
1015 summary += f", public_access_cidrs -> [{', '.join(cidrs)}]"
1016 if normalized_mode == "PRIVATE" and cidrs:
1017 formatter.print_info(
1018 "Note: public_access_cidrs only takes effect while endpoint_access is "
1019 "PUBLIC_AND_PRIVATE; storing the allowlist for a later flip."
1020 )
1021 if not yes:
1022 confirm(f"Update cdk.json: {summary}?", abort=True)
1024 from ..stacks import update_eks_cluster_config
1026 settings: dict[str, Any] = {"endpoint_access": normalized_mode}
1027 if cidrs:
1028 settings["public_access_cidrs"] = list(cidrs)
1029 try:
1030 update_eks_cluster_config(settings)
1031 except RuntimeError as exc:
1032 formatter.print_error(str(exc))
1033 sys.exit(1)
1035 formatter.print_success(summary)
1036 formatter.print_info(
1037 "Config only — no stacks were deployed. Run 'gco stacks deploy "
1038 f"{config.project_name}-<region> -y' per regional stack to apply, then "
1039 "'gco stacks access' / 'gco cluster doctor' to verify access."
1040 )
1043# =============================================================================
1044# Deployment-region commands (managed-config engine veneers)
1045# =============================================================================
1048@stacks.group("regions")
1049@pass_config
1050def regions_cmd(config: Any) -> None:
1051 """Manage workload deployment Regions in cdk.json.
1053 These commands edit context.deployment_regions.regional through the
1054 managed-config engine: validated against the same rules CDK synth
1055 enforces, atomic, idempotent, and audited. They never deploy — run
1056 'gco stacks deploy' afterwards to apply the change.
1057 """
1058 pass
1061@regions_cmd.command("list")
1062@click.option("--config-path", help="Explicit cdk.json to use (default: nearest in cwd/parents)")
1063@pass_config
1064def regions_list(config: Any, config_path: Any) -> None:
1065 """Show the configured deployment-region topology.
1067 Reports the global/api_gateway/monitoring Regions, the workload Region
1068 list, the resolved AWS partition, and the cdk.json path backing the
1069 answer. On a broken configuration, partition_error explains what CDK
1070 synth would reject.
1071 """
1072 from ..managed_config import ManagedConfigError, get_deployment_regions_status
1074 formatter = get_output_formatter(config)
1076 try:
1077 status = get_deployment_regions_status(config_path=config_path)
1078 except ManagedConfigError as e:
1079 formatter.print_error(str(e))
1080 sys.exit(1)
1081 if config.output_format == "table":
1082 # The table cell renderer collapses lists to "[N items]"; join for
1083 # humans. JSON/YAML (the MCP path) keep the real list.
1084 status["regional"] = ", ".join(status["regional"])
1085 formatter.print(status)
1088@regions_cmd.command("add")
1089@click.argument("region")
1090@click.option("--config-path", help="Explicit cdk.json to use (default: nearest in cwd/parents)")
1091@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1092@pass_config
1093def regions_add(config: Any, region: Any, config_path: Any, yes: Any) -> None:
1094 """Add a workload Region to deployment_regions.regional.
1096 The Region must expose CloudFormation in the AWS SDK's endpoint data and
1097 belong to the same AWS partition as the already-configured Regions.
1098 Re-adding a present Region is a reported no-op.
1100 Examples:
1101 gco stacks regions add us-west-2
1102 gco stacks regions add eu-west-1 -y
1103 """
1104 from ..managed_config import ManagedConfigError, add_deployment_region
1106 formatter = get_output_formatter(config)
1108 if not yes:
1109 confirm(f"Add {region} to deployment_regions.regional in cdk.json?", abort=True)
1111 try:
1112 report = add_deployment_region(region, config_path=config_path)
1113 except ManagedConfigError as e:
1114 formatter.print_error(str(e))
1115 sys.exit(1)
1117 if report.changed:
1118 formatter.print_success(report.summary())
1119 formatter.print_info(
1120 "Config only — no stacks were deployed. "
1121 f"Run 'gco stacks deploy {config.project_name}-{region}' (or 'gco stacks deploy-all') to apply"
1122 )
1123 else:
1124 formatter.print_info(report.summary())
1127@regions_cmd.command("remove")
1128@click.argument("region")
1129@click.option("--config-path", help="Explicit cdk.json to use (default: nearest in cwd/parents)")
1130@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1131@pass_config
1132def regions_remove(config: Any, region: Any, config_path: Any, yes: Any) -> None:
1133 """Remove a workload Region from deployment_regions.regional.
1135 The resulting list must stay valid (at least one Region). Removing an
1136 absent Region is a reported no-op. Removing an unknown/typo'd entry from
1137 a hand-edited config is allowed — validation applies to the result, so
1138 this is also the repair path.
1140 Examples:
1141 gco stacks regions remove us-west-2
1142 gco stacks regions remove xx-typo-1 -y
1143 """
1144 from ..managed_config import ManagedConfigError, remove_deployment_region
1146 formatter = get_output_formatter(config)
1148 if not yes:
1149 formatter.print_warning(
1150 f"This only edits cdk.json — a deployed {config.project_name}-{region} "
1151 "stack is NOT destroyed by this change."
1152 )
1153 confirm(f"Remove {region} from deployment_regions.regional in cdk.json?", abort=True)
1155 try:
1156 report = remove_deployment_region(region, config_path=config_path)
1157 except ManagedConfigError as e:
1158 formatter.print_error(str(e))
1159 sys.exit(1)
1161 if report.changed:
1162 formatter.print_success(report.summary())
1163 formatter.print_info(
1164 f"Config only — if {config.project_name}-{region} is deployed, destroy it "
1165 f"explicitly with 'gco stacks destroy {config.project_name}-{region}'"
1166 )
1167 else:
1168 formatter.print_info(report.summary())
1171@regions_cmd.command("set")
1172@click.argument("role", type=click.Choice(["global", "api_gateway", "monitoring"]))
1173@click.argument("region")
1174@click.option("--config-path", help="Explicit cdk.json to use (default: nearest in cwd/parents)")
1175@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1176@pass_config
1177def regions_set(config: Any, role: Any, region: Any, config_path: Any, yes: Any) -> None:
1178 """Set a control-plane Region scalar (global/api_gateway/monitoring).
1180 The Region must be SDK-known and keep the whole topology (all three
1181 scalars plus the workload list) in one AWS partition. Setting the
1182 current value is a reported no-op.
1184 Examples:
1185 gco stacks regions set monitoring us-west-2
1186 gco stacks regions set global us-east-2 -y
1187 """
1188 from ..managed_config import ManagedConfigError, set_deployment_region_role
1190 formatter = get_output_formatter(config)
1192 if not yes:
1193 formatter.print_warning(
1194 "This only edits cdk.json — already-deployed stacks are not moved "
1195 "or destroyed; the next deploy creates the stack in the new Region."
1196 )
1197 confirm(f"Set deployment_regions.{role} to {region} in cdk.json?", abort=True)
1199 try:
1200 report = set_deployment_region_role(role, region, config_path=config_path)
1201 except ManagedConfigError as e:
1202 formatter.print_error(str(e))
1203 sys.exit(1)
1205 if report.changed:
1206 formatter.print_success(report.summary())
1207 formatter.print_info(
1208 "Config only — no stacks were deployed. Run 'gco stacks deploy-all' to apply, "
1209 "and clean up the stack in the previous Region yourself if it was deployed"
1210 )
1211 else:
1212 formatter.print_info(report.summary())
1215# =============================================================================
1216# Bedrock model default (managed-config engine veneer)
1217# =============================================================================
1220@stacks.group("bedrock")
1221@pass_config
1222def bedrock_cmd(config: Any) -> None:
1223 """Manage Bedrock model and reasoning defaults in cdk.json.
1225 Four independent model keys serve Mission sampling, the capacity advisor,
1226 Claude Code, and Codex. Codex also owns a reviewed reasoning-effort sibling.
1227 Every edit uses the shared managed-config engine: validated, atomic,
1228 idempotent, and audited.
1229 """
1230 pass
1233@bedrock_cmd.command("show")
1234@click.option("--config-path", help="Explicit cdk.json to use (default: nearest in cwd/parents)")
1235@pass_config
1236def bedrock_show(config: Any, config_path: Any) -> None:
1237 """Show every managed Bedrock model/reasoning default and its path."""
1238 from ..managed_config import ManagedConfigError, get_bedrock_model_status
1240 formatter = get_output_formatter(config)
1242 try:
1243 status = get_bedrock_model_status(config_path=config_path)
1244 except ManagedConfigError as e:
1245 formatter.print_error(str(e))
1246 sys.exit(1)
1247 formatter.print(status)
1250@bedrock_cmd.command("set-mission-model")
1251@click.argument("model_id")
1252@click.option("--config-path", help="Explicit cdk.json to use (default: nearest in cwd/parents)")
1253@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1254@pass_config
1255def bedrock_set_mission_model(config: Any, model_id: Any, config_path: Any, yes: Any) -> None:
1256 """Set context.bedrock.mission_default_model_id (Mission sampling).
1258 This is the default Mission sampling uses; the capacity advisor and
1259 `gco autopilot` have their own keys (see set-capacity-advisor-model and
1260 set-claude-code-model). Model and inference-profile IDs are free-form
1261 (custom profiles, marketplace models), so validation mirrors the runtime
1262 reader: a non-empty string without surrounding whitespace. Sibling
1263 settings (bedrock.generation_reasoning, the other model keys) are preserved.
1265 Examples:
1266 gco stacks bedrock set-mission-model us.amazon.nova-pro-v1:0
1267 gco stacks bedrock set-mission-model us.amazon.nova-2-lite-v1:0 -y
1268 """
1269 from ..managed_config import ManagedConfigError, set_mission_default_model
1271 formatter = get_output_formatter(config)
1273 if not yes:
1274 confirm(f"Set bedrock.mission_default_model_id to {model_id} in cdk.json?", abort=True)
1276 try:
1277 report = set_mission_default_model(model_id, config_path=config_path)
1278 except ManagedConfigError as e:
1279 formatter.print_error(str(e))
1280 sys.exit(1)
1282 if report.changed:
1283 formatter.print_success(report.summary())
1284 formatter.print_info(
1285 "Mission sampling picks this up on its next run; explicit "
1286 "--bedrock-model-id/env overrides still take precedence"
1287 )
1288 else:
1289 formatter.print_info(report.summary())
1292@bedrock_cmd.command("set-capacity-advisor-model")
1293@click.argument("model_id")
1294@click.option("--config-path", help="Explicit cdk.json to use (default: nearest in cwd/parents)")
1295@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1296@pass_config
1297def bedrock_set_capacity_advisor_model(
1298 config: Any, model_id: Any, config_path: Any, yes: Any
1299) -> None:
1300 """Set context.bedrock.capacity_advisor_default_model_id.
1302 This is the default `gco capacity advise` (and its historical variant)
1303 uses; Mission sampling and `gco autopilot` have their own keys (see
1304 set-mission-model and set-claude-code-model). Model and inference-profile
1305 IDs are free-form (custom profiles, marketplace models), so validation
1306 mirrors the runtime reader: a non-empty string without surrounding
1307 whitespace. Sibling settings (bedrock.generation_reasoning, the other model keys)
1308 are preserved.
1310 Examples:
1311 gco stacks bedrock set-capacity-advisor-model us.amazon.nova-pro-v1:0
1312 gco stacks bedrock set-capacity-advisor-model us.amazon.nova-2-lite-v1:0 -y
1313 """
1314 from ..managed_config import ManagedConfigError, set_capacity_advisor_default_model
1316 formatter = get_output_formatter(config)
1318 if not yes:
1319 confirm(
1320 f"Set bedrock.capacity_advisor_default_model_id to {model_id} in cdk.json?",
1321 abort=True,
1322 )
1324 try:
1325 report = set_capacity_advisor_default_model(model_id, config_path=config_path)
1326 except ManagedConfigError as e:
1327 formatter.print_error(str(e))
1328 sys.exit(1)
1330 if report.changed:
1331 formatter.print_success(report.summary())
1332 formatter.print_info(
1333 "The capacity advisor picks this up on its next run; explicit "
1334 "--model overrides still take precedence"
1335 )
1336 else:
1337 formatter.print_info(report.summary())
1340@bedrock_cmd.command("set-claude-code-model")
1341@click.argument("model_id")
1342@click.option("--config-path", help="Explicit cdk.json to use (default: nearest in cwd/parents)")
1343@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1344@pass_config
1345def bedrock_set_claude_code_model(config: Any, model_id: Any, config_path: Any, yes: Any) -> None:
1346 """Set context.bedrock.claude_code_default_model_id.
1348 This is the session model `gco autopilot` hands to Claude Code, kept
1349 separate from the generation defaults (see set-mission-model and
1350 set-capacity-advisor-model) so repointing the interactive agent never
1351 repoints Mission sampling or the capacity advisor. Validation mirrors the runtime reader: a non-empty string
1352 without surrounding whitespace. Sibling settings are preserved.
1354 Examples:
1355 gco stacks bedrock set-claude-code-model us.anthropic.claude-sonnet-4-6
1356 gco stacks bedrock set-claude-code-model us.anthropic.claude-opus-4-7 -y
1357 """
1358 from ..managed_config import ManagedConfigError, set_claude_code_default_model
1360 formatter = get_output_formatter(config)
1362 if not yes:
1363 confirm(
1364 f"Set bedrock.claude_code_default_model_id to {model_id} in cdk.json?",
1365 abort=True,
1366 )
1368 try:
1369 report = set_claude_code_default_model(model_id, config_path=config_path)
1370 except ManagedConfigError as e:
1371 formatter.print_error(str(e))
1372 sys.exit(1)
1374 if report.changed:
1375 formatter.print_success(report.summary())
1376 formatter.print_info(
1377 "New autopilot sessions pick this up at launch; explicit "
1378 "--model/GCO_AUTOPILOT_MODEL overrides still take precedence"
1379 )
1380 else:
1381 formatter.print_info(report.summary())
1384@bedrock_cmd.command("set-codex-model")
1385@click.argument("model_id")
1386@click.option("--config-path", help="Explicit cdk.json to use (default: nearest in cwd/parents)")
1387@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1388@pass_config
1389def bedrock_set_codex_model(config: Any, model_id: Any, config_path: Any, yes: Any) -> None:
1390 """Set context.bedrock.codex_default_model_id.
1392 This is the canonical model for Codex Autopilot sessions. The reviewed
1393 context.bedrock.codex.reasoning_effort remains independent and is preserved;
1394 review that pair together when changing model families. Explicit --model,
1395 GCO_AUTOPILOT_CODEX_MODEL, and GCO_AUTOPILOT_MODEL overrides still win.
1397 Examples:
1398 gco stacks bedrock set-codex-model global.openai.<model-id>
1399 gco stacks bedrock set-codex-model global.openai.<model-id> -y
1400 """
1401 from ..managed_config import ManagedConfigError, set_codex_default_model
1403 formatter = get_output_formatter(config)
1405 if not yes:
1406 confirm(
1407 f"Set bedrock.codex_default_model_id to {model_id} in cdk.json?",
1408 abort=True,
1409 )
1411 try:
1412 report = set_codex_default_model(model_id, config_path=config_path)
1413 except ManagedConfigError as e:
1414 formatter.print_error(str(e))
1415 sys.exit(1)
1417 if report.changed:
1418 formatter.print_success(report.summary())
1419 formatter.print_info(
1420 "Canonical Codex sessions pick this up at launch; explicit "
1421 "--model/GCO_AUTOPILOT_CODEX_MODEL/GCO_AUTOPILOT_MODEL overrides "
1422 "still take precedence"
1423 )
1424 else:
1425 formatter.print_info(report.summary())
1428@bedrock_cmd.command("set-codex-reasoning-effort")
1429@click.argument(
1430 "reasoning_effort",
1431 type=click.Choice(["minimal", "low", "medium", "high", "xhigh"], case_sensitive=True),
1432)
1433@click.option("--config-path", help="Explicit cdk.json to use (default: nearest in cwd/parents)")
1434@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1435@pass_config
1436def bedrock_set_codex_reasoning_effort(
1437 config: Any, reasoning_effort: Any, config_path: Any, yes: Any
1438) -> None:
1439 """Set context.bedrock.codex.reasoning_effort.
1441 The effort applies only when Codex uses the canonical default model; any
1442 explicit model override omits it. Allowed values are minimal, low, medium,
1443 high, and xhigh.
1445 Examples:
1446 gco stacks bedrock set-codex-reasoning-effort high
1447 gco stacks bedrock set-codex-reasoning-effort xhigh -y
1448 """
1449 from ..managed_config import ManagedConfigError, set_codex_reasoning_effort
1451 formatter = get_output_formatter(config)
1453 if not yes:
1454 confirm(
1455 f"Set bedrock.codex.reasoning_effort to {reasoning_effort} in cdk.json?",
1456 abort=True,
1457 )
1459 try:
1460 report = set_codex_reasoning_effort(reasoning_effort, config_path=config_path)
1461 except ManagedConfigError as e:
1462 formatter.print_error(str(e))
1463 sys.exit(1)
1465 if report.changed:
1466 formatter.print_success(report.summary())
1467 formatter.print_info(
1468 "Canonical Codex sessions pick this up at launch; explicit model "
1469 "overrides intentionally omit canonical reasoning"
1470 )
1471 else:
1472 formatter.print_info(report.summary())
1475# =============================================================================
1476# FSx commands
1477# =============================================================================
1480@stacks.group("fsx")
1481@pass_config
1482def fsx_cmd(config: Any) -> None:
1483 """Manage FSx for Lustre configuration."""
1484 pass
1487@fsx_cmd.command("status")
1488@click.option("--region", "-r", help="Show config for specific region")
1489@pass_config
1490def fsx_status(config: Any, region: Any) -> None:
1491 """Show current FSx for Lustre configuration status."""
1492 from ..stacks import get_fsx_config
1494 formatter = get_output_formatter(config)
1496 try:
1497 fsx_config = get_fsx_config(region)
1498 if region:
1499 formatter.print_info(f"FSx config for region: {region}")
1500 else:
1501 formatter.print_info("Global FSx config:")
1502 formatter.print(fsx_config)
1503 except Exception as e:
1504 formatter.print_error(f"Failed to get FSx config: {e}")
1505 sys.exit(1)
1508@fsx_cmd.command("enable")
1509@click.option("--region", "-r", help="Enable FSx for specific region only")
1510@click.option("--storage-capacity", "-s", default=1200, help="Storage capacity in GiB (min 1200)")
1511@click.option(
1512 "--deployment-type",
1513 "-d",
1514 type=click.Choice(["SCRATCH_1", "SCRATCH_2", "PERSISTENT_1", "PERSISTENT_2"]),
1515 default="SCRATCH_2",
1516 help="FSx deployment type",
1517)
1518@click.option("--throughput", "-t", default=200, help="Per-unit storage throughput (MB/s)")
1519@click.option("--compression", "-c", type=click.Choice(["LZ4", "NONE"]), default="LZ4")
1520@click.option("--import-path", help="S3 path for data import (s3://bucket/prefix)")
1521@click.option("--export-path", help="S3 path for data export (s3://bucket/prefix)")
1522@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1523@pass_config
1524def fsx_enable(
1525 config: Any,
1526 region: Any,
1527 storage_capacity: Any,
1528 deployment_type: Any,
1529 throughput: Any,
1530 compression: Any,
1531 import_path: Any,
1532 export_path: Any,
1533 yes: Any,
1534) -> None:
1535 """Enable FSx for Lustre in the stack configuration.
1537 FSx for Lustre provides high-performance parallel file system storage
1538 ideal for ML training workloads requiring high throughput and low latency.
1540 Examples:
1541 gco stacks fsx enable
1542 gco stacks fsx enable --region us-east-1
1543 gco stacks fsx enable --storage-capacity 2400 --deployment-type PERSISTENT_2
1544 gco stacks fsx enable -r us-west-2 --import-path s3://my-bucket/training-data
1545 """
1546 from ..stacks import update_fsx_config
1548 formatter = get_output_formatter(config)
1550 if storage_capacity < 1200:
1551 formatter.print_error("Storage capacity must be at least 1200 GiB")
1552 sys.exit(1)
1554 scope = f"region {region}" if region else "all regions (global)"
1556 if not yes:
1557 formatter.print_info(f"FSx for Lustre configuration for {scope}:")
1558 formatter.print_info(f" Storage Capacity: {storage_capacity} GiB")
1559 formatter.print_info(f" Deployment Type: {deployment_type}")
1560 formatter.print_info(f" Throughput: {throughput} MB/s per TiB")
1561 formatter.print_info(f" Compression: {compression}")
1562 if import_path:
1563 formatter.print_info(f" Import Path: {import_path}")
1564 if export_path:
1565 formatter.print_info(f" Export Path: {export_path}")
1566 confirm(f"\nEnable FSx for Lustre for {scope}?", abort=True)
1568 try:
1569 fsx_settings = {
1570 "enabled": True,
1571 "storage_capacity_gib": storage_capacity,
1572 "deployment_type": deployment_type,
1573 "per_unit_storage_throughput": throughput,
1574 "data_compression_type": compression,
1575 "import_path": import_path,
1576 "export_path": export_path,
1577 "auto_import_policy": "NEW_CHANGED_DELETED" if import_path else None,
1578 }
1580 update_fsx_config(fsx_settings, region)
1581 formatter.print_success(f"FSx for Lustre enabled in cdk.json for {scope}")
1582 if region:
1583 formatter.print_info(
1584 f"Run 'gco stacks deploy {config.project_name}-{region}' to apply changes"
1585 )
1586 else:
1587 formatter.print_info("Run 'gco stacks deploy' to apply changes")
1589 except Exception as e:
1590 formatter.print_error(f"Failed to enable FSx: {e}")
1591 sys.exit(1)
1594@fsx_cmd.command("disable")
1595@click.option("--region", "-r", help="Disable FSx for specific region only")
1596@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1597@pass_config
1598def fsx_disable(config: Any, region: Any, yes: Any) -> None:
1599 """Disable FSx for Lustre in the stack configuration.
1601 Note: This only updates the configuration. Run 'gco stacks deploy'
1602 to apply changes. Existing FSx file systems will be deleted.
1604 Examples:
1605 gco stacks fsx disable
1606 gco stacks fsx disable --region us-east-1
1607 """
1608 from ..stacks import update_fsx_config
1610 formatter = get_output_formatter(config)
1612 scope = f"region {region}" if region else "all regions (global)"
1614 if not yes:
1615 formatter.print_warning(f"This will disable FSx for Lustre for {scope}.")
1616 formatter.print_warning("Existing FSx file systems will be deleted on next deploy.")
1617 confirm("Are you sure?", abort=True)
1619 try:
1620 update_fsx_config({"enabled": False}, region)
1621 formatter.print_success(f"FSx for Lustre disabled in cdk.json for {scope}")
1622 if region:
1623 formatter.print_info(
1624 f"Run 'gco stacks deploy {config.project_name}-{region}' to apply changes"
1625 )
1626 else:
1627 formatter.print_info("Run 'gco stacks deploy' to apply changes")
1629 except Exception as e:
1630 formatter.print_error(f"Failed to disable FSx: {e}")
1631 sys.exit(1)
1634# =============================================================================
1635# Valkey commands
1636# =============================================================================
1639@stacks.group("valkey")
1640@pass_config
1641def valkey_cmd(config: Any) -> None:
1642 """Manage Valkey Serverless cache configuration."""
1643 pass
1646@valkey_cmd.command("status")
1647@pass_config
1648def valkey_status(config: Any) -> None:
1649 """Show current Valkey Serverless configuration status."""
1650 from ..stacks import get_valkey_config
1652 formatter = get_output_formatter(config)
1654 try:
1655 valkey_config = get_valkey_config()
1656 formatter.print_info("Valkey config:")
1657 formatter.print(valkey_config)
1658 except Exception as e:
1659 formatter.print_error(f"Failed to get Valkey config: {e}")
1660 sys.exit(1)
1663@valkey_cmd.command("enable")
1664@click.option("--max-storage", default=5, help="Max data storage in GB (default: 5)")
1665@click.option("--max-ecpu", default=5000, help="Max eCPU per second (default: 5000)")
1666@click.option("--snapshot-retention", default=1, help="Snapshot retention in days (default: 1)")
1667@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1668@pass_config
1669def valkey_enable(
1670 config: Any,
1671 max_storage: Any,
1672 max_ecpu: Any,
1673 snapshot_retention: Any,
1674 yes: Any,
1675) -> None:
1676 """Enable Valkey Serverless cache in the stack configuration.
1678 Valkey provides a serverless key-value cache for prompt caching,
1679 feature stores, session state, and low-latency data access.
1681 Examples:
1682 gco stacks valkey enable
1683 gco stacks valkey enable --max-storage 10 --max-ecpu 10000
1684 """
1685 from ..stacks import update_valkey_config
1687 formatter = get_output_formatter(config)
1689 if not yes:
1690 formatter.print_info("Valkey Serverless configuration:")
1691 formatter.print_info(f" Max Data Storage: {max_storage} GB")
1692 formatter.print_info(f" Max eCPU/second: {max_ecpu}")
1693 formatter.print_info(f" Snapshot Retention: {snapshot_retention} days")
1694 confirm("\nEnable Valkey Serverless?", abort=True)
1696 try:
1697 valkey_settings = {
1698 "enabled": True,
1699 "max_data_storage_gb": max_storage,
1700 "max_ecpu_per_second": max_ecpu,
1701 "snapshot_retention_limit": snapshot_retention,
1702 }
1704 update_valkey_config(valkey_settings)
1705 formatter.print_success("Valkey Serverless enabled in cdk.json")
1706 formatter.print_info("Run 'gco stacks deploy-all -y' to apply changes")
1708 except Exception as e:
1709 formatter.print_error(f"Failed to enable Valkey: {e}")
1710 sys.exit(1)
1713@valkey_cmd.command("disable")
1714@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1715@pass_config
1716def valkey_disable(config: Any, yes: Any) -> None:
1717 """Disable Valkey Serverless cache in the stack configuration.
1719 Note: This only updates the configuration. Run 'gco stacks deploy-all -y'
1720 to apply changes. Existing Valkey caches will be deleted.
1722 Examples:
1723 gco stacks valkey disable
1724 """
1725 from ..stacks import update_valkey_config
1727 formatter = get_output_formatter(config)
1729 if not yes:
1730 formatter.print_warning("This will disable Valkey Serverless.")
1731 formatter.print_warning("Existing Valkey caches will be deleted on next deploy.")
1732 confirm("Are you sure?", abort=True)
1734 try:
1735 update_valkey_config({"enabled": False})
1736 formatter.print_success("Valkey Serverless disabled in cdk.json")
1737 formatter.print_info("Run 'gco stacks deploy-all -y' to apply changes")
1739 except Exception as e:
1740 formatter.print_error(f"Failed to disable Valkey: {e}")
1741 sys.exit(1)
1744# =============================================================================
1745# Aurora pgvector commands
1746# =============================================================================
1749@stacks.group("aurora")
1750@pass_config
1751def aurora_cmd(config: Any) -> None:
1752 """Manage Aurora PostgreSQL (pgvector) configuration."""
1753 pass
1756@aurora_cmd.command("status")
1757@pass_config
1758def aurora_status(config: Any) -> None:
1759 """Show current Aurora PostgreSQL (pgvector) configuration status."""
1760 from ..stacks import get_aurora_config
1762 formatter = get_output_formatter(config)
1764 try:
1765 aurora_config = get_aurora_config()
1766 formatter.print_info("Aurora pgvector config:")
1767 formatter.print(aurora_config)
1768 except Exception as e:
1769 formatter.print_error(f"Failed to get Aurora config: {e}")
1770 sys.exit(1)
1773@aurora_cmd.command("enable")
1774@click.option("--min-acu", default=0, help="Minimum ACU (0 = scale to zero, default: 0)")
1775@click.option("--max-acu", default=16, help="Maximum ACU (default: 16)")
1776@click.option("--backup-retention", default=7, help="Backup retention in days (default: 7)")
1777@click.option(
1778 "--deletion-protection/--no-deletion-protection",
1779 default=False,
1780 help="Enable deletion protection",
1781)
1782@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1783@pass_config
1784def aurora_enable(
1785 config: Any,
1786 min_acu: Any,
1787 max_acu: Any,
1788 backup_retention: Any,
1789 deletion_protection: Any,
1790 yes: Any,
1791) -> None:
1792 """Enable Aurora PostgreSQL with pgvector in the stack configuration.
1794 Aurora Serverless v2 with pgvector provides vector similarity search
1795 for RAG applications, semantic search, and embedding storage.
1797 Examples:
1798 gco stacks aurora enable
1799 gco stacks aurora enable --min-acu 2 --max-acu 32 --deletion-protection
1800 """
1801 from ..stacks import update_aurora_config
1803 formatter = get_output_formatter(config)
1805 if min_acu < 0:
1806 formatter.print_error("Minimum ACU must be >= 0")
1807 sys.exit(1)
1808 if max_acu < 1:
1809 formatter.print_error("Maximum ACU must be >= 1")
1810 sys.exit(1)
1811 if max_acu < min_acu:
1812 formatter.print_error("Maximum ACU must be >= minimum ACU")
1813 sys.exit(1)
1815 if not yes:
1816 formatter.print_info("Aurora pgvector configuration:")
1817 formatter.print_info(f" Min ACU: {min_acu} {'(scale to zero)' if min_acu == 0 else ''}")
1818 formatter.print_info(f" Max ACU: {max_acu}")
1819 formatter.print_info(f" Backup Retention: {backup_retention} days")
1820 formatter.print_info(f" Deletion Protection: {deletion_protection}")
1821 confirm("\nEnable Aurora pgvector?", abort=True)
1823 try:
1824 aurora_settings = {
1825 "enabled": True,
1826 "min_acu": min_acu,
1827 "max_acu": max_acu,
1828 "backup_retention_days": backup_retention,
1829 "deletion_protection": deletion_protection,
1830 }
1832 update_aurora_config(aurora_settings)
1833 formatter.print_success("Aurora pgvector enabled in cdk.json")
1834 formatter.print_info("Run 'gco stacks deploy-all -y' to apply changes")
1836 except Exception as e:
1837 formatter.print_error(f"Failed to enable Aurora: {e}")
1838 sys.exit(1)
1841@aurora_cmd.command("disable")
1842@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1843@pass_config
1844def aurora_disable(config: Any, yes: Any) -> None:
1845 """Disable Aurora PostgreSQL (pgvector) in the stack configuration.
1847 Note: This only updates the configuration. Run 'gco stacks deploy-all -y'
1848 to apply changes. Existing Aurora clusters will be deleted unless
1849 deletion protection is enabled.
1851 Examples:
1852 gco stacks aurora disable
1853 """
1854 from ..stacks import update_aurora_config
1856 formatter = get_output_formatter(config)
1858 if not yes:
1859 formatter.print_warning("This will disable Aurora pgvector.")
1860 formatter.print_warning(
1861 "Existing Aurora clusters will be deleted on next deploy "
1862 "(unless deletion protection is enabled)."
1863 )
1864 confirm("Are you sure?", abort=True)
1866 try:
1867 update_aurora_config({"enabled": False})
1868 formatter.print_success("Aurora pgvector disabled in cdk.json")
1869 formatter.print_info("Run 'gco stacks deploy-all -y' to apply changes")
1871 except Exception as e:
1872 formatter.print_error(f"Failed to disable Aurora: {e}")
1873 sys.exit(1)
1876def _project_name() -> str:
1877 """Read project_name from cdk.json context (default 'gco')."""
1878 import json
1879 from pathlib import Path
1881 try:
1882 with open(Path.cwd() / "cdk.json", encoding="utf-8") as f:
1883 document = json.load(f)
1884 if not isinstance(document, dict):
1885 return "gco"
1886 ctx = document.get("context")
1887 if not isinstance(ctx, dict):
1888 return "gco"
1889 return str(ctx.get("project_name") or "gco")
1890 except OSError, ValueError:
1891 return "gco"
1894def _target_regions(config: Any, region: Any, all_regions: bool) -> list[str]:
1895 """Resolve which regions a command acts on.
1897 ``--all-regions`` returns every configured regional deployment region;
1898 otherwise an explicit ``--region``, else the first regional region, else
1899 the configured default.
1900 """
1901 cdk_regions = _load_cdk_json()
1902 regional = (
1903 list(cdk_regions["regional"]) if (cdk_regions and cdk_regions.get("regional")) else []
1904 )
1906 if all_regions:
1907 return regional
1908 if region:
1909 return [str(region)]
1910 if regional:
1911 return [str(regional[0])]
1912 return [str(config.default_region or "us-east-1")]
1915@stacks.group("addons")
1916@pass_config
1917def addons_cmd(config: Any) -> None:
1918 """Inspect and re-converge cluster add-ons (Helm charts).
1920 Add-on installation is decoupled from the CloudFormation rollback path: a
1921 chart that fails to install never rolls back the cluster. Use these commands
1922 to see per-chart status and re-run the installer without a full redeploy.
1923 """
1924 pass
1927@addons_cmd.command("status")
1928@click.option("--region", "-r", help="AWS region (default: first deployment region)")
1929@click.option("--all-regions", "-A", is_flag=True, help="Show status across all deployment regions")
1930@pass_config
1931def addons_status(config: Any, region: Any, all_regions: bool) -> None:
1932 """Show per-chart add-on install status (from SSM).
1934 Examples:
1935 gco stacks addons status
1936 gco stacks addons status -r us-west-2
1937 gco stacks addons status --all-regions
1938 """
1939 formatter = get_output_formatter(config)
1940 project = _project_name()
1941 for target in _target_regions(config, region, all_regions):
1942 _addons_status_one(formatter, project, target)
1945def _addons_status_one(formatter: Any, project: str, region: str) -> None:
1946 """Print the add-on status table for a single region."""
1947 import json
1949 import boto3
1951 prefix = f"/{project}/addons/{region}/"
1953 try:
1954 ssm = boto3.client("ssm", region_name=region)
1955 params: list[dict[str, Any]] = []
1956 paginator = ssm.get_paginator("get_parameters_by_path")
1957 for page in paginator.paginate(Path=prefix, Recursive=False):
1958 params.extend(page.get("Parameters", []))
1959 except Exception as e:
1960 formatter.print_error(f"[{region}] Failed to read add-on status from SSM: {e}")
1961 return
1963 rows = []
1964 for p in params:
1965 name = p["Name"].rsplit("/", 1)[-1]
1966 if name == "_input":
1967 continue
1968 try:
1969 data = json.loads(p["Value"])
1970 except ValueError:
1971 data = {"status": "unknown", "message": p.get("Value", "")}
1972 if not isinstance(data, dict):
1973 data = {"status": "unknown", "message": p.get("Value", "")}
1974 status = str(data.get("status", "unknown"))
1975 message = str(data.get("message", ""))
1976 rows.append((name, status, message[:80]))
1978 if not rows:
1979 formatter.print_info(
1980 f"[{region}] No add-on status recorded under {prefix} yet. "
1981 "The installer writes status as charts are processed."
1982 )
1983 return
1985 rows.sort()
1986 formatter.print_info(f"Add-on status for {project} in {region}:")
1987 for name, status, message in rows:
1988 line = f" {name:<28} {status:<12} {message}"
1989 if status in ("installed", "uninstalled", "absent", "applied"):
1990 formatter.print_success(line)
1991 else:
1992 formatter.print_error(line)
1995@addons_cmd.command("install")
1996@click.option("--region", "-r", help="AWS region (default: first deployment region)")
1997@click.option(
1998 "--all-regions", "-A", is_flag=True, help="Re-converge add-ons in all deployment regions"
1999)
2000@pass_config
2001def addons_install(config: Any, region: Any, all_regions: bool) -> None:
2002 """Re-run the Helm add-on installer (idempotent; never rolls back the cluster).
2004 Replays the last execution input persisted by the deploy, so chart config
2005 and IAM role wiring stay in one place. Use this to re-converge after a
2006 transient failure instead of a full stack redeploy.
2008 Examples:
2009 gco stacks addons install
2010 gco stacks addons install -r us-west-2
2011 gco stacks addons install --all-regions
2012 """
2013 formatter = get_output_formatter(config)
2014 project = _project_name()
2015 failures = 0
2016 for target in _target_regions(config, region, all_regions):
2017 if not _addons_install_one(formatter, project, target):
2018 failures += 1
2019 if failures:
2020 sys.exit(1)
2023def _decode_addon_replay_input(stored_value: str) -> str:
2024 """Reverse the helm orchestrator's zlib+base64 replay-input encoding.
2026 The orchestrator stores the execution input encoded because SSM rejects
2027 raw ``{{PLACEHOLDER}}`` tokens (see lambda/helm-orchestrator/handler.py).
2028 A leading ``{`` means a raw legacy JSON value; pass it through unchanged.
2029 """
2030 import base64
2031 import zlib
2033 if stored_value.lstrip().startswith("{"):
2034 return stored_value
2035 compressed = base64.b64decode(stored_value.encode("ascii"), validate=True)
2036 return zlib.decompress(compressed).decode("utf-8")
2039def _addons_install_one(formatter: Any, project: str, region: str) -> bool:
2040 """Start an add-on install for a single region. Returns True on success."""
2041 import boto3
2042 from botocore.exceptions import ClientError
2044 input_param = f"/{project}/addons/{region}/_input"
2045 fence_param = f"/{project}/addons/{region}/_teardown"
2047 try:
2048 ssm = boto3.client("ssm", region_name=region)
2049 try:
2050 ssm.get_parameter(Name=fence_param)
2051 except ClientError as exc:
2052 if exc.response.get("Error", {}).get("Code") != "ParameterNotFound":
2053 raise
2054 else:
2055 formatter.print_error(
2056 f"[{region}] Add-on teardown is active ({fence_param}); refusing to start."
2057 )
2058 return False
2059 stored_input = ssm.get_parameter(Name=input_param)["Parameter"]["Value"]
2060 execution_input = _decode_addon_replay_input(stored_input)
2061 except Exception as e:
2062 formatter.print_error(
2063 f"[{region}] Could not read {input_param}: {e}. "
2064 f"Deploy the regional stack at least once first (gco stacks deploy {project}-{region} -y)."
2065 )
2066 return False
2068 try:
2069 sfn = boto3.client("stepfunctions", region_name=region)
2070 machines = sfn.list_state_machines(maxResults=1000)["stateMachines"]
2071 arn = next(
2072 (m["stateMachineArn"] for m in machines if "HelmInstall" in m["name"]),
2073 None,
2074 )
2075 if not arn:
2076 formatter.print_error(f"[{region}] No HelmInstall state machine found.")
2077 return False
2078 resp = sfn.start_execution(stateMachineArn=arn, input=execution_input)
2079 except Exception as e:
2080 formatter.print_error(f"[{region}] Failed to start add-on install: {e}")
2081 return False
2083 formatter.print_success(f"[{region}] Started add-on install (idempotent re-converge).")
2084 formatter.print_info(f" execution: {resp['executionArn']}")
2085 formatter.print_info(f" track status with: gco stacks addons status -r {region}")
2086 return True