Coverage for gco / stacks / regional_stack.py: 100.00%
1234 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-10-28 16:19 +0000
1"""
2Regional stack for GCO (Global Capacity Orchestrator on AWS) - EKS cluster and ALB per region.
4It creates all regional resources for a single AWS region. One instance is
5deployed per region defined in cdk.json.
7Resources Created:
8 VPC & Networking:
9 - VPC spanning every AZ in the region, public subnets (NAT), private subnets (EKS and ALB)
10 - 2 NAT Gateways for high availability
11 - VPC endpoints for ECR, S3, STS, Secrets Manager, SSM, CloudWatch
12 - VPC Flow Logs (CloudWatch Logs, 30-day retention)
14 EKS Cluster (Auto Mode):
15 - Managed control plane with full logging (API, Audit, Authenticator, Controller Manager, Scheduler)
16 - Built-in NodePools: system, general-purpose
17 - Custom NodePools: gpu-x86-pool, gpu-arm-pool, gpu-inference-pool,
18 gpu-efa-pool, mooncake-efa-pool, neuron-pool, cpu-general-pool
19 - IRSA roles for service accounts (Secrets Manager, SQS, DynamoDB, CloudWatch, S3, EFS)
21 Load Balancing:
22 - Internal ALB created from Gateway API resources by the self-managed
23 AWS Load Balancer Controller
24 - Always-deployed regional API bridge reaches the ALB through a VPC Lambda;
25 direct caller access is optional in ``aws`` and required elsewhere
26 - Global Accelerator endpoint registration in commercial ``aws`` only
28 Storage:
29 - EFS with dynamic provisioning (CSI driver, access points, encryption at rest + in transit)
30 - FSx for Lustre (optional, toggled via cdk.json)
31 - Valkey Serverless cache (optional)
32 - Aurora Serverless v2 with pgvector (optional)
34 Lambda Functions:
35 - kubectl-applier: applies K8s manifests during deployment
36 - helm-installer: installs Helm charts (KEDA, Volcano, KubeRay, etc.)
37 - ga-registration: registers the ALB with Global Accelerator in ``aws``
38 - transaction-search: switches on CloudWatch Transaction Search for the
39 traced API services' X-Ray span export (never switches it off)
40 - regional-api-proxy: separate-stack VPC proxy used by the always-on
41 aggregation bridge and by optional direct callers in ``aws`` or the
42 required regional workload ingress in other partitions
44 Container Images:
45 - Docker image assets (CDK bootstrap asset repository, content-hash
46 tags) for health-monitor, manifest-processor, inference-proxy,
47 inference-monitor, queue-processor, cost-monitor
49 SQS:
50 - Regional job queue + dead letter queue (for gco jobs submit-sqs)
52Key Design Decisions:
53 - EKS Auto Mode handles node provisioning — no managed node groups or Karpenter provisioners
54 - NodePools use WhenEmpty consolidation for inference to avoid disrupting long-running pods
55 - IRSA (IAM Roles for Service Accounts) for least-privilege pod-level AWS access
56 - All optional features (FSx, Valkey, Aurora) are toggled via cdk.json context variables
57 - Template variables in K8s manifests ({{PLACEHOLDER}}) are replaced at deploy time
59Dependencies:
60 - GCOGlobalStack (partition-wide state and, in ``aws``, Global Accelerator endpoint groups)
61 - GCOApiGatewayGlobalStack (for auth secret ARN)
63Modification Guide:
64 - To add a new NodePool: add a YAML manifest in lambda/kubectl-applier-simple/manifests/ (40-49 range)
65 - To add a new service: add dockerfiles/Dockerfile.<service> (discovery picks it up),
66 call ``self._service_image_asset`` here, add a manifest in manifests/
67 - To add a new optional feature: add a cdk.json context toggle, guard with if/else in this file
68 - To change the EKS version: edit ``context.kubernetes_version`` in cdk.json and follow
69 "Upgrading the EKS Kubernetes version" in docs/MAINTENANCE.md (kubectl, kind, Calico,
70 add-on builds and the Python client move with it)
71"""
73from __future__ import annotations
75import ipaddress
76import os
77import re
78from collections.abc import Mapping
79from dataclasses import dataclass
80from datetime import UTC, datetime
81from decimal import Decimal
82from pathlib import Path
83from typing import Any, ClassVar, cast
85import aws_cdk.aws_eks_v2 as eks
86import yaml
87from aws_cdk import (
88 Acknowledgment,
89 Annotations,
90 CfnJson,
91 CfnOutput,
92 CfnTag,
93 CustomResource,
94 Duration,
95 Fn,
96 RemovalPolicy,
97 Stack,
98 Validations,
99)
100from aws_cdk import aws_ec2 as ec2
101from aws_cdk import aws_ecr_assets as ecr_assets
102from aws_cdk import aws_efs as efs
103from aws_cdk import aws_eks as eks_l1 # L1 constructs (CfnPodIdentityAssociation)
104from aws_cdk import aws_events as events
105from aws_cdk import aws_events_targets as events_targets
106from aws_cdk import aws_fsx as fsx
107from aws_cdk import aws_iam as iam
108from aws_cdk import aws_kms as kms
109from aws_cdk import aws_lambda as lambda_
110from aws_cdk import aws_logs as logs
111from aws_cdk import aws_s3 as s3
112from aws_cdk import aws_secretsmanager as secretsmanager
113from aws_cdk import aws_sns as sns
114from aws_cdk import aws_sqs as sqs
115from aws_cdk import aws_ssm as ssm
116from aws_cdk import aws_stepfunctions as sfn
117from aws_cdk import aws_stepfunctions_tasks as sfn_tasks
118from aws_cdk import custom_resources as cr
119from constructs import Construct
121from gco.argocd_config import (
122 argocd_chart_values,
123 compute_argocd_replacements,
124 validate_argocd_config,
125)
126from gco.config.config_loader import (
127 TRACING_CONTEXT_KEY,
128 TRACING_OVERRIDES_CONTEXT_KEY,
129 ConfigLoader,
130 resolve_tracing_config,
131)
132from gco.eks_capabilities_config import (
133 CAPABILITY_TYPE_API_NAMES,
134 compute_eks_capabilities_replacements,
135 enabled_capability_types,
136 kro_kubernetes_username,
137 normalize_eks_capabilities_config,
138)
139from gco.inference_proxy_config import (
140 compute_inference_proxy_tls_replacements as _compute_inference_proxy_tls_replacements,
141)
142from gco.manifest_security_policy import validate_manifest_security_policy
143from gco.service_images import discover_service_dockerfiles, service_dockerfile
144from gco.slurm_config import (
145 SLURM_ACCOUNTING_BACKUP_PREFIX,
146 SLURM_CLUSTER_CHART_NAME,
147 accounting_in_cluster,
148 compute_slurm_replacements,
149 slurm_chart_values,
150 validate_slurm_config,
151)
152from gco.stacks.aws_load_balancer_controller_policy import (
153 aws_load_balancer_controller_policy_document,
154)
155from gco.stacks.constants import (
156 AURORA_POSTGRES_VERSION,
157 DEFAULT_MANIFEST_RESOURCE_CAPS,
158 DEFAULT_RESOURCE_QUOTA,
159 EKS_ADDON_CLOUDWATCH_OBSERVABILITY,
160 EKS_ADDON_EFS_CSI_DRIVER,
161 EKS_ADDON_FSX_CSI_DRIVER,
162 EKS_ADDON_METRICS_SERVER,
163 EKS_ADDON_POD_IDENTITY_AGENT,
164 EKS_UNSUPPORTED_AZ_IDS,
165 LAMBDA_PYTHON_RUNTIME,
166 MOONCAKE_MASTER_DEFAULT_IMAGE,
167 api_gateway_auth_secret_name,
168 backend_tls_certificate_arn_parameter_name,
169 cluster_shared_ssm_parameter_prefix,
170 cost_report_ssm_parameter_prefix,
171 parse_k8s_quantity,
172 regional_shared_ssm_parameter_prefix,
173)
174from gco.stacks.lambda_assets import lambda_source_code
176# <pyflowchart-code-diagram> BEGIN - auto-inserted, do not edit
177# Generated at (UTC): 2026-08-30T12:00:00Z
178# Generated from Git commit: 8cb8e766461f5b7f35205e12db980988598cf6d5
179# Flowchart(s) generated from this file:
180# * ``GCORegionalStack.__init__`` -> ``diagrams/code_diagrams/gco/stacks/regional_stack.GCORegionalStack___init__.html``
181# (PNG: ``diagrams/code_diagrams/gco/stacks/regional_stack.GCORegionalStack___init__.png``)
182# * ``GCORegionalStack._get_volcano_image_mirror_config`` -> ``diagrams/code_diagrams/gco/stacks/regional_stack.GCORegionalStack__get_volcano_image_mirror_config.html``
183# (PNG: ``diagrams/code_diagrams/gco/stacks/regional_stack.GCORegionalStack__get_volcano_image_mirror_config.png``)
184# Regenerate with ``SOURCE_DATE_EPOCH=<unix-seconds> GCO_DIAGRAM_SOURCE_COMMIT=<40-char-sha> python diagrams/generate.py --code-only``.
185# <pyflowchart-code-diagram> END
188_LIVE_VALIDATION_PROVIDER_LOG_CONTEXT = "gco_live_validation_retain_provider_log_groups"
191@dataclass(frozen=True)
192class SharedBucketIdentity:
193 """Identity of the always-on ``Cluster_Shared_Bucket`` owned by ``GCOGlobalStack``.
195 Every regional stack resolves this identity from the three SSM parameters
196 ``/gco/cluster-shared-bucket/{name,arn,region}`` published by
197 ``GCOGlobalStack`` in the global region. The three values are used to
198 grant IAM permissions on the bucket to the regional job-pod role and to
199 populate the ``gco-cluster-shared-bucket`` ConfigMap applied to every
200 regional EKS cluster. Frozen so it can be safely shared across helper
201 methods without accidental mutation.
202 """
204 name: str
205 arn: str
206 region: str
209def _compute_kubectl_cluster_shared_replacements(
210 shared: SharedBucketIdentity,
211) -> dict[str, str]:
212 """Build the ``{{CLUSTER_SHARED_BUCKET*}}`` kubectl-applier replacements.
214 Pure helper kept at module scope so property and presence tests can
215 inspect the output without synthesizing a full regional stack. The
216 three keys are always populated — there is no feature toggle — because
217 the ``gco-cluster-shared-bucket`` ConfigMap is applied unconditionally
218 on every regional cluster.
219 """
220 return {
221 "{{CLUSTER_SHARED_BUCKET}}": shared.name,
222 "{{CLUSTER_SHARED_BUCKET_ARN}}": shared.arn,
223 "{{CLUSTER_SHARED_BUCKET_REGION}}": shared.region,
224 }
227def _compute_kubectl_regional_shared_replacements(
228 name: str,
229 arn: str,
230 region: str,
231) -> dict[str, str]:
232 """Build the ``{{REGIONAL_SHARED_BUCKET*}}`` kubectl-applier replacements.
234 Pure helper kept at module scope so property and presence tests can
235 inspect the output without synthesizing a full regional stack, mirroring
236 :func:`_compute_kubectl_cluster_shared_replacements`.
238 The three keys are always populated — there is no feature toggle —
239 because ``_create_regional_shared_bucket`` provisions the bucket
240 unconditionally, so the ``gco-regional-shared-bucket`` ConfigMap is
241 applied on every regional cluster and is never gated out of the
242 applier by an unresolved placeholder.
244 Unlike the cluster-shared helper this takes the three values directly
245 rather than a :class:`SharedBucketIdentity`: the regional bucket is a
246 local construct in this stack, so its name/ARN are CDK tokens resolved
247 at deploy time instead of values read back from cross-region SSM.
248 """
249 return {
250 "{{REGIONAL_SHARED_BUCKET}}": name,
251 "{{REGIONAL_SHARED_BUCKET_ARN}}": arn,
252 "{{REGIONAL_SHARED_BUCKET_REGION}}": region,
253 }
256#: StorageClass name for in-cluster observability PVCs (Prometheus, Grafana,
257#: Alertmanager). The value overrides reference this name, and the gated gp3
258#: StorageClass manifest (25-storage-observability-gp3.yaml) declares it. A
259#: synth test asserts the two stay in lockstep. The manifest keeps this name
260#: static (a placeholder in ``metadata.name`` would fail k8s schema
261#: validation), so the toggle gate lives in an annotation value instead.
262_OBSERVABILITY_STORAGE_CLASS = "gco-observability-gp3"
264#: Node affinity for the memory-heavy observability singletons (Prometheus,
265#: Grafana, MLflow): instances with more than 4 GiB, so 8 GiB or larger.
266#: EKS Auto Mode's built-in general-purpose pool bin-packs by requests and
267#: its smallest instances are 2 vCPU / 4 GiB. Three consecutive EKS 1.37
268#: live validations showed those nodes cannot carry one of these pods next
269#: to the ~1 GiB Bottlerocket host footprint (kubelet, containerd and one
270#: shim per pod, CoreDNS, the network-policy and node-monitoring agents, the
271#: EBS CSI node plugin) that the kubelet's 627Mi reservation under-counts:
272#: the node reached ~95% memory, reclaim evicted the host daemons' text
273#: pages (direct reclaim, ~1 GiB of active file pages churning, PSI memory
274#: "full" above 30% over five minutes, ~4k major page faults/s with up to
275#: 1.5 cores of kernel time on a 2-vCPU node), and every probe on the node
276#: timed out: Grafana never became ready, Prometheus answered OpenCost with
277#: 503s, and MLflow was liveness-killed in a loop. Resource requests alone
278#: did not help, because Karpenter kept picking 4 GiB instances for them.
279#: ``eks.amazonaws.com/instance-memory`` (MiB) is an Auto Mode well-known
280#: label, so Karpenter provisions a matching instance from the same pool
281#: and the kube-scheduler honors the same expression. Injected by the stack
282#: (not charts.yaml) because the label exists only on EKS Auto Mode nodes;
283#: the kind-based CI installs the same charts with the shipped values and
284#: would otherwise never schedule them. A plain literal assignment (no
285#: annotation) on purpose: the kind examples-smoke job lifts this module's
286#: literal constants with ``ast.Assign`` + ``ast.literal_eval`` to run
287#: ``_mlflow_chart_values`` outside CDK.
288_LARGE_MEMORY_NODE_AFFINITY = {
289 "nodeAffinity": {
290 "requiredDuringSchedulingIgnoredDuringExecution": {
291 "nodeSelectorTerms": [
292 {
293 "matchExpressions": [
294 {
295 "key": "eks.amazonaws.com/instance-memory",
296 "operator": "Gt",
297 "values": ["4096"],
298 }
299 ]
300 }
301 ]
302 }
303 }
304}
307#: The MLflow server's own port (the chart Service's and the container's), and
308#: its HTTPS front door: the post-Helm Service ``mlflow-tls``
309#: (post-helm-mlflow-tls.yaml) in front of the ``mlflow-tls-proxy`` sidecar,
310#: which terminates TLS with the ``mlflow-tls`` leaf and forwards to the
311#: server on pod loopback.
312_MLFLOW_SERVER_PORT = 5000
313_MLFLOW_TLS_PORT = 5443
315#: In-cluster names clients use to reach the MLflow tracking server. MLflow
316#: 3.x's host-validation middleware matches the raw Host header (port
317#: included — that is why both spellings are listed), and setting
318#: ``allowed-hosts`` REPLACES its built-in localhost/private-IP allowance
319#: rather than extending it, so the value override must carry the complete
320#: list (see ``_mlflow_allowed_hosts``). The TLS sidecar forwards the
321#: decrypted bytes unchanged, so an HTTPS client's Host header names the
322#: ``mlflow-tls`` Service in whichever of its four DNS forms it dialled. A
323#: plain literal, so the kind examples job can lift it from this source.
324_MLFLOW_SERVICE_HOSTS = (
325 "mlflow.monitoring",
326 "mlflow.monitoring:5000",
327 "mlflow-tls",
328 "mlflow-tls:5443",
329 "mlflow-tls.monitoring",
330 "mlflow-tls.monitoring:5443",
331 "mlflow-tls.monitoring.svc",
332 "mlflow-tls.monitoring.svc:5443",
333 "mlflow-tls.monitoring.svc.cluster.local",
334 "mlflow-tls.monitoring.svc.cluster.local:5443",
335)
337#: Loopback spellings a browser sends through the access tunnel.
338#:
339#: ``gco monitoring open --service mlflow`` is the ONLY human path to this
340#: server (ClusterIP, no Ingress), and it port-forwards to localhost — so the
341#: browser sends ``Host: localhost:5000`` or ``Host: 127.0.0.1:5000``. Because
342#: the flag replaces MLflow's built-in loopback allowance instead of extending
343#: it, omitting these makes the documented UI path answer 403 "possible DNS
344#: rebinding attack detected" while the server is perfectly healthy — the
345#: in-cluster DNS spellings return 200 through the very same tunnel (caught
346#: live 2026-08-15, taking the release screenshot).
347#:
348#: This restores upstream's own loopback posture rather than widening it: the
349#: rebinding attack these checks exist for is an external DNS name resolving
350#: to an internal address, which loopback literals cannot express. Reaching
351#: the port at all still requires the authenticated SSM tunnel, and arbitrary
352#: DNS names stay rejected.
353_MLFLOW_TUNNEL_HOSTS = ("localhost", "localhost:5000", "127.0.0.1", "127.0.0.1:5000")
356def _mlflow_allowed_hosts(vpc_endpoint_cidrs: list[str]) -> str:
357 """Compose the MLflow ``allowed-hosts`` list from the deployment's CIDRs.
359 The service-DNS and loopback spellings are static; the IP tail derives
360 from ``vpc_endpoint_cidrs`` (single source of truth — the same context key
361 the NetworkPolicy egress rules render) so widening the VPC range never
362 needs a matching charts.yaml edit. Every group is load-bearing:
363 Prometheus scrapes the pod IP directly, so dropping the pod-IP allowance
364 403s every ServiceMonitor scrape (caught live 2026-08-14); the tunnel
365 forwards to loopback, so dropping those 403s the only human UI path
366 (caught live 2026-08-15).
368 MLflow's allow-list is glob-based, so only octet-aligned prefixes convert
369 exactly; other masks WIDEN to the containing octet boundary (capped at
370 /24 granularity so the trailing ``.*`` still matches ``host:port``
371 Host headers). Widening is the safe direction — this is Host-header
372 hygiene layered over NetworkPolicies and a private ALB, and
373 under-matching is what breaks scrapes.
374 """
375 patterns: list[str] = []
376 for cidr in vpc_endpoint_cidrs:
377 network = ipaddress.ip_network(cidr, strict=False)
378 if network.version != 4:
379 raise ValueError(
380 f"vpc_endpoint_cidrs entry {cidr!r} is not IPv4; the MLflow "
381 "allowed-hosts derivation only understands IPv4 globs"
382 )
383 octets = str(network.network_address).split(".")
384 kept = min(max(network.prefixlen // 8, 1), 3)
385 patterns.append(".".join(octets[:kept]) + ".*")
386 return ",".join(dict.fromkeys([*_MLFLOW_SERVICE_HOSTS, *_MLFLOW_TUNNEL_HOSTS, *patterns]))
389#: Every service Dockerfile in the tree, discovered rather than listed, so a
390#: new ``dockerfiles/Dockerfile.<service>`` is excluded from the *other*
391#: services' asset hashes without anyone remembering to add it here.
392_SERVICE_IMAGE_BUILD_INPUTS = tuple(discover_service_dockerfiles().values())
393_SERVICE_IMAGE_COMMON_EXCLUDES = (
394 "cli/**",
395 "gco/stacks/**",
396 "dockerfiles/README.md",
397)
400def _service_image_asset_excludes(included_path: str) -> list[str]:
401 """Exclude inputs that cannot affect one production service image."""
402 return list(_SERVICE_IMAGE_COMMON_EXCLUDES) + [
403 path for path in _SERVICE_IMAGE_BUILD_INPUTS if path != included_path
404 ]
407#: cdk.json ``vpc_endpoints`` service keys → CDK endpoint services. The key
408#: sets are pinned to ``ConfigLoader``'s validation lists by
409#: ``tests/test_regional_stack.py`` so a name the loader accepts always maps.
410_GATEWAY_ENDPOINT_SERVICES: dict[str, ec2.GatewayVpcEndpointAwsService] = {
411 "s3": ec2.GatewayVpcEndpointAwsService.S3,
412 "dynamodb": ec2.GatewayVpcEndpointAwsService.DYNAMODB,
413}
414_INTERFACE_ENDPOINT_SERVICES: dict[str, ec2.InterfaceVpcEndpointAwsService] = {
415 "sts": ec2.InterfaceVpcEndpointAwsService.STS,
416 "ecr.api": ec2.InterfaceVpcEndpointAwsService.ECR,
417 "ecr.dkr": ec2.InterfaceVpcEndpointAwsService.ECR_DOCKER,
418 "logs": ec2.InterfaceVpcEndpointAwsService.CLOUDWATCH_LOGS,
419 "monitoring": ec2.InterfaceVpcEndpointAwsService.CLOUDWATCH_MONITORING,
420 "sqs": ec2.InterfaceVpcEndpointAwsService.SQS,
421 "ssm": ec2.InterfaceVpcEndpointAwsService.SSM,
422 "secretsmanager": ec2.InterfaceVpcEndpointAwsService.SECRETS_MANAGER,
423 "kms": ec2.InterfaceVpcEndpointAwsService.KMS,
424 "eks": ec2.InterfaceVpcEndpointAwsService.EKS,
425 "elasticfilesystem": ec2.InterfaceVpcEndpointAwsService.ELASTIC_FILESYSTEM,
426 "bedrock-runtime": ec2.InterfaceVpcEndpointAwsService.BEDROCK_RUNTIME,
427 # com.amazonaws.<region>.xray: the traced API services' OTLP span export.
428 "xray": ec2.InterfaceVpcEndpointAwsService.XRAY,
429}
432#: CDK context key that force-enables optional Helm charts for one deploy
433#: without editing cdk.json (comma-separated cdk.json helm-block key names,
434#: e.g. ``--context helm_enabled_overrides=yunikorn,slurm``). The live release
435#: validation harness uses this to exercise off-by-default schedulers against
436#: an otherwise pristine checkout; it is equally useful for trying one
437#: scheduler ahead of a config change. Overrides can only ENABLE — a chart
438#: disabled by an operator stays disabled unless named here.
439_HELM_OVERRIDE_CONTEXT_KEY = "helm_enabled_overrides"
441#: Live-validation-only context that prevents AWS-managed EFS automatic backups
442#: from outliving a disposable stack for the service's fixed retention window.
443#: Normal deployments omit this key and preserve automatic backups.
444_LIVE_VALIDATION_DISABLE_EFS_BACKUPS_CONTEXT = "gco_live_validation_disable_efs_automatic_backups"
447def _explicit_context_bool(raw: object, *, key: str) -> bool:
448 """Parse an optional CDK context boolean without truthy-string ambiguity."""
449 if raw is None:
450 return False
451 if isinstance(raw, bool):
452 return raw
453 if isinstance(raw, str):
454 normalized = raw.strip().casefold()
455 if normalized == "true":
456 return True
457 if normalized == "false":
458 return False
459 raise ValueError(f"{key} must be true or false")
462#: Every cdk.json helm-block key _get_enabled_charts understands. Kept in
463#: lockstep with its chart_map so an override typo fails the synth loudly
464#: instead of silently deploying without the requested chart.
465_HELM_CHART_CONFIG_KEYS = frozenset(
466 {
467 "aws_load_balancer_controller",
468 "keda",
469 "aws_efa_device_plugin",
470 "aws_neuron_device_plugin",
471 "volcano",
472 "kuberay",
473 "cert_manager",
474 "slurm",
475 "yunikorn",
476 "kubeflow_trainer",
477 "argocd",
478 "crossplane",
479 "kueue",
480 }
481)
483#: Charts that are mandatory platform components; the cdk.json toggle is
484#: ignored for these (see _get_enabled_charts for the rationale).
485_MANDATORY_CHART_KEYS = frozenset({"aws_load_balancer_controller", "keda"})
487#: Opt-in platform add-ons: an absent cdk.json block (or one without
488#: ``enabled``) means OFF, unlike the historical charts whose missing key
489#: defaults to on. A fork or an older cdk.json that predates the add-on must
490#: never start installing Argo CD or Crossplane on its next deploy.
491_OFF_BY_DEFAULT_CHART_KEYS = frozenset({"argocd", "crossplane"})
493#: Stack-delete Helm uninstall task budgets (minutes) for charts whose
494#: installer first deletes their custom resources (every discovered type,
495#: namespaced then cluster-scoped, with a finalizer-strip retry) before the
496#: ``helm uninstall`` itself. Every other non-LBC chart gets two minutes.
497_UNINSTALL_TIMEOUT_MINUTES: dict[str, int] = {
498 "keda": 4,
499 "argocd": 4,
500 "crossplane": 6,
501 # The Slinky cluster's NodeSets drain their Slurm nodes and carry
502 # job-protection PDBs; the installer deletes every slinky.slurm.net
503 # resource before `helm uninstall` so the controller is still there to
504 # clear the finalizers.
505 SLURM_CLUSTER_CHART_NAME: 4,
506}
509#: (container ceiling, namespace ceiling) pairs the resource-quota invariant
510#: compares; every dimension a container can request must fit the namespace.
511_RESOURCE_QUOTA_INVARIANTS = (
512 ("container_max_cpu", "max_cpu"),
513 ("container_max_memory", "max_memory"),
514 ("container_max_gpu", "max_gpu"),
515)
518def _validated_resource_quota(raw: object) -> dict[str, str]:
519 """Merge the ``resource_quota`` context over defaults and validate it.
521 The values are substituted verbatim into the gco-jobs ResourceQuota and
522 LimitRange manifests, where a typo or an incoherent pair (a per-container
523 ceiling that exceeds the namespace ceiling) previously deployed silently
524 and only surfaced as pods being forbidden at admission — with the reason
525 visible in namespace events alone. Fail the synth instead.
527 Raises:
528 ValueError: If the context is not a mapping, carries an unknown key,
529 a value that does not parse as a Kubernetes quantity, or a
530 per-container ceiling exceeding its namespace ceiling.
531 """
532 if not isinstance(raw, dict):
533 raise ValueError(
534 f"cdk.json context 'resource_quota' must be an object, got {type(raw).__name__}"
535 )
536 unknown = sorted(set(raw) - set(DEFAULT_RESOURCE_QUOTA))
537 if unknown:
538 allowed = ", ".join(sorted(DEFAULT_RESOURCE_QUOTA))
539 raise ValueError(
540 f"cdk.json context 'resource_quota' has unknown key(s) {unknown}; allowed: {allowed}"
541 )
542 merged = {key: str(raw.get(key, default)) for key, default in DEFAULT_RESOURCE_QUOTA.items()}
543 parsed: dict[str, float] = {}
544 for key, value in merged.items():
545 try:
546 parsed[key] = parse_k8s_quantity(value)
547 except ValueError as exc:
548 raise ValueError(
549 f"resource_quota.{key}={value!r} is not a valid Kubernetes quantity"
550 ) from exc
551 if parsed[key] < 0:
552 raise ValueError(f"resource_quota.{key}={value!r} must not be negative")
553 for container_key, namespace_key in _RESOURCE_QUOTA_INVARIANTS:
554 if parsed[container_key] > parsed[namespace_key]:
555 raise ValueError(
556 f"resource_quota.{container_key}={merged[container_key]!r} exceeds "
557 f"resource_quota.{namespace_key}={merged[namespace_key]!r}: a container "
558 "that passes the LimitRange could never be admitted by the namespace "
559 "ResourceQuota"
560 )
561 return merged
564#: (per-manifest cap, container ceiling, namespace ceiling) triples for the
565#: cross-layer invariant: container_max_* <= *_per_manifest <= max_*.
566_MANIFEST_CAP_INVARIANTS = (
567 ("max_cpu_per_manifest", "container_max_cpu", "max_cpu"),
568 ("max_memory_per_manifest", "container_max_memory", "max_memory"),
569 ("max_gpu_per_manifest", "container_max_gpu", "max_gpu"),
570)
573def _validated_manifest_caps(raw: object, resource_quota: dict[str, str]) -> dict[str, str]:
574 """Merge ``job_validation_policy.resource_quotas`` over defaults; validate.
576 Three layers govern job resources and must tell one story: the
577 manifest/queue processors cap what a single submitted manifest may total
578 (these values), the LimitRange caps each container, and the namespace
579 ResourceQuota caps the aggregate. Enforce
580 ``container_max_* <= *_per_manifest <= max_*`` at synth so the front door
581 never rejects a manifest whose pods the namespace would admit and never
582 accepts one that can never run — previously the defaults disagreed
583 (per-manifest 4 GPUs vs the platform's own 16-GPU EFA training example).
585 Raises:
586 ValueError: On unknown keys, unparseable quantities, or a violated
587 layering invariant.
588 """
589 if not isinstance(raw, dict):
590 raise ValueError(
591 "cdk.json context 'job_validation_policy.resource_quotas' must be an "
592 f"object, got {type(raw).__name__}"
593 )
594 unknown = sorted(set(raw) - set(DEFAULT_MANIFEST_RESOURCE_CAPS))
595 if unknown:
596 allowed = ", ".join(sorted(DEFAULT_MANIFEST_RESOURCE_CAPS))
597 raise ValueError(
598 "cdk.json context 'job_validation_policy.resource_quotas' has unknown "
599 f"key(s) {unknown}; allowed: {allowed}"
600 )
601 merged = {
602 key: str(raw.get(key, default)) for key, default in DEFAULT_MANIFEST_RESOURCE_CAPS.items()
603 }
604 parsed: dict[str, float] = {}
605 for key, value in merged.items():
606 try:
607 parsed[key] = parse_k8s_quantity(value)
608 except ValueError as exc:
609 raise ValueError(
610 f"job_validation_policy.resource_quotas.{key}={value!r} is not a "
611 "valid Kubernetes quantity"
612 ) from exc
613 for manifest_key, container_key, namespace_key in _MANIFEST_CAP_INVARIANTS:
614 container_value = parse_k8s_quantity(resource_quota[container_key])
615 namespace_value = parse_k8s_quantity(resource_quota[namespace_key])
616 if parsed[manifest_key] < container_value:
617 raise ValueError(
618 f"job_validation_policy.resource_quotas.{manifest_key}="
619 f"{merged[manifest_key]!r} is below resource_quota.{container_key}="
620 f"{resource_quota[container_key]!r}: the front door would reject a "
621 "manifest whose single container the LimitRange admits"
622 )
623 if parsed[manifest_key] > namespace_value:
624 raise ValueError(
625 f"job_validation_policy.resource_quotas.{manifest_key}="
626 f"{merged[manifest_key]!r} exceeds resource_quota.{namespace_key}="
627 f"{resource_quota[namespace_key]!r}: the front door would accept a "
628 "manifest the namespace ResourceQuota can never admit"
629 )
630 return merged
633def _parse_helm_enabled_overrides(raw: object) -> frozenset[str]:
634 """Parse and validate the ``helm_enabled_overrides`` context value.
636 Accepts a comma-separated string (the only shape the CDK CLI can pass
637 with ``--context``) or a list of strings (cdk.json-style), returning the
638 validated set of helm-block keys to force-enable. Unknown names raise at
639 synth time with the valid list.
640 """
641 if raw is None:
642 return frozenset()
643 if isinstance(raw, str):
644 names = [part.strip() for part in raw.split(",") if part.strip()]
645 elif isinstance(raw, list) and all(isinstance(part, str) for part in raw):
646 names = [part.strip() for part in raw if part.strip()]
647 else:
648 raise ValueError(
649 f"{_HELM_OVERRIDE_CONTEXT_KEY} must be a comma-separated string or string list"
650 )
651 unknown = sorted(set(names) - _HELM_CHART_CONFIG_KEYS)
652 if unknown:
653 valid = ", ".join(sorted(_HELM_CHART_CONFIG_KEYS))
654 raise ValueError(
655 f"Unknown {_HELM_OVERRIDE_CONTEXT_KEY} name(s): {', '.join(unknown)}. Valid: {valid}"
656 )
657 return frozenset(names)
660def _helm_chart_enabled(
661 helm_config: Mapping[str, Any],
662 overrides: frozenset[str],
663 config_key: str,
664) -> bool:
665 """Resolve one helm-block key's effective enablement.
667 Single source of truth shared by _get_enabled_charts and the
668 kubectl-applier gate replacements so the installed chart set and the
669 gated manifests can never disagree: mandatory charts are always on, a
670 context override forces on, and otherwise the cdk.json toggle decides
671 (missing key defaults to enabled, matching the historical behavior —
672 except the opt-in add-ons in ``_OFF_BY_DEFAULT_CHART_KEYS``, which
673 default to disabled).
674 """
675 if config_key in _MANDATORY_CHART_KEYS or config_key in overrides:
676 return True
677 default = config_key not in _OFF_BY_DEFAULT_CHART_KEYS
678 chart_config = helm_config.get(config_key, {})
679 return bool(chart_config.get("enabled", default)) if isinstance(chart_config, dict) else default
682def _compute_kubectl_scheduler_replacements(
683 *, kueue_enabled: bool, slurm_enabled: bool, kubeflow_trainer_enabled: bool = False
684) -> dict[str, str]:
685 """Build the kubectl-applier replacements that gate scheduler manifests.
687 When Kueue is enabled the ``{{KUEUE_ENABLED}}`` gate resolves so the
688 default queue topology (post-helm-kueue-default-queues.yaml) applies;
689 when Slurm is enabled the ``{{SLURM_ENABLED}}`` gate resolves so the
690 Slinky NetworkPolicies (post-helm-slurm-network.yaml) apply; when the
691 Kubeflow Trainer is enabled the ``{{KUBEFLOW_TRAINER_ENABLED}}`` gate
692 resolves so the built-in ClusterTrainingRuntime blueprints
693 (post-helm-kubeflow-trainer-runtimes.yaml) apply. A disabled
694 scheduler leaves its placeholder unreplaced, the applier skips the file,
695 and _FEATURE_RESOURCE_INVENTORY prunes previously applied objects — the
696 same optional-feature gating observability, FSx, and Valkey use.
697 """
698 replacements: dict[str, str] = {}
699 if kueue_enabled:
700 replacements["{{KUEUE_ENABLED}}"] = "true"
701 if slurm_enabled:
702 replacements["{{SLURM_ENABLED}}"] = "true"
703 if kubeflow_trainer_enabled:
704 replacements["{{KUBEFLOW_TRAINER_ENABLED}}"] = "true"
705 return replacements
708def _compute_kubectl_observability_replacements(
709 enabled: bool, *, grafana_admin_password_rotation_schedule: str = ""
710) -> dict[str, str]:
711 """Build the kubectl-applier replacements that gate the observability manifests.
713 Pure helper kept at module scope so presence/absence can be asserted
714 without synthesizing a full regional stack. When observability is enabled
715 the ``{{CLUSTER_OBSERVABILITY_ENABLED}}`` gate resolves to ``"true"`` so the
716 gp3 StorageClass, ServiceMonitors, dashboards, and credential-rotation
717 CronJob render and apply, and ``{{GRAFANA_ADMIN_PASSWORD_ROTATION_SCHEDULE}}``
718 resolves to the configured cron. When disabled the dict is empty, so those
719 manifests keep an unreplaced ``{{...}}`` token and the applier skips them —
720 the same optional-feature gating FSx and Valkey already rely on.
721 """
722 if not enabled:
723 return {}
724 return {
725 "{{CLUSTER_OBSERVABILITY_ENABLED}}": "true",
726 "{{GRAFANA_ADMIN_PASSWORD_ROTATION_SCHEDULE}}": grafana_admin_password_rotation_schedule,
727 }
730def _decimal_string(value: float) -> str:
731 """Render a float as a plain decimal (``1e-05`` -> ``"0.00001"``).
733 ``repr`` is the shortest string that round-trips the float, and
734 formatting its ``Decimal`` with ``"f"`` removes any exponent, so the
735 services' ``float()`` parse sees exactly the configured value.
736 """
737 return format(Decimal(repr(float(value))), "f")
740def _compute_kubectl_tracing_replacements(tracing: Mapping[str, Any]) -> dict[str, str]:
741 """Build the ``{{TRACING_*}}`` kubectl-applier replacements.
743 Unlike the feature gates above, both keys are ALWAYS emitted: they sit
744 in the four core API Deployments (health-monitor, manifest-processor,
745 inference-proxy, cost-monitor), and a placeholder left unreplaced makes
746 the applier skip the whole file. Tracing off therefore renders
747 ``GCO_TRACING_ENABLED: "false"`` rather than withholding the key.
748 """
749 return {
750 "{{TRACING_ENABLED}}": "true" if tracing["enabled"] else "false",
751 "{{TRACING_SAMPLE_RATIO}}": _decimal_string(tracing["sample_ratio"]),
752 }
755#: IAM actions the traced API services use to export spans. The X-Ray OTLP
756#: endpoint authorizes ``xray:PutSpans``; ``xray:PutTraceSegments`` covers the
757#: segment API. X-Ray supports no resource-level scoping for either, so the
758#: grant is ``Resource: *`` and write-only.
759_TRACING_EXPORT_ACTIONS = ("xray:PutTraceSegments", "xray:PutSpans")
761#: Part of the AwsSolutions-IAM5 ``Resource::*`` reason of every traced
762#: service role while tracing is enabled. cdk-nag records one reason per
763#: finding id and construct (the first acknowledgment wins), so a role that
764#: already acknowledges ``Resource::*`` gets this appended to that reason
765#: instead of a second acknowledgment the report would never show.
766_TRACING_EXPORT_NAG_REASON = (
767 "Tracing span export uses xray:PutTraceSegments and xray:PutSpans, which "
768 "X-Ray grants only on Resource:* (no resource-level permissions); that "
769 "statement is write-only and carries no other action."
770)
772#: Namespaces whose pods GCO instruments itself (the traced API services in
773#: gco-system) or must never mutate (model pods in gco-inference).
774#: Application Signals auto-monitor in the CloudWatch Observability add-on
775#: injects ADOT auto-instrumentation into matching workloads by default; it
776#: would double-instrument the services and rewrite model pods.
777_APPLICATION_SIGNALS_EXCLUDED_NAMESPACES = ("gco-system", "gco-inference")
779#: Every language the add-on's auto-monitor can instrument (its
780#: ``manager.applicationSignals.autoMonitor.exclude`` keys).
781_APPLICATION_SIGNALS_LANGUAGES = ("java", "python", "dotnet", "nodejs")
783#: Where every GCO TLS sidecar reads its cert-manager leaf; these are also the
784#: ``gco.services.tls_proxy`` defaults.
785_TLS_PROXY_KEYPAIR_DIR = "/var/run/gco/tls"
787#: How long a chart-pod TLS sidecar waits for its leaf Secret. The Helm charts
788#: start OpenCost, Grafana and MLflow before the post-Helm manifests create their
789#: Certificates, so the Secret volume is optional and the sidecar polls for
790#: the keypair instead of failing closed immediately.
791_CHART_TLS_PROXY_KEYPAIR_WAIT_SECONDS = 1800
793#: The service images run as this uid/gid (the synthesized ``gco`` user).
794#: Chart pods run their own containers as other uids, so the sidecar sets it
795#: explicitly; ``runAsNonRoot`` needs a numeric uid to verify a named user.
796_SERVICE_IMAGE_UID = 1000
799def _chart_tls_proxy_sidecar(
800 *,
801 name: str,
802 image: str,
803 port: int,
804 upstream_port: int,
805 secret_name: str,
806) -> tuple[dict[str, Any], dict[str, Any]]:
807 """Return the ``(container, volume)`` of a TLS sidecar for a Helm chart pod.
809 OpenCost, Grafana and MLflow serve plain HTTP; the sidecar runs
810 ``gco.services.tls_proxy`` from a GCO service image, terminates TLS on
811 ``port`` (container port ``https``, which the post-Helm ``*-tls``
812 Services target) with the cert-manager leaf in ``secret_name``, and
813 forwards to ``127.0.0.1:upstream_port``.
815 The Secret volume is ``optional`` and world-readable (``0444``): the
816 chart pod starts before its Certificate exists, and GCO does not own the
817 chart pod's uid/fsGroup. There is deliberately no readiness or startup
818 probe, so the chart pod becomes Ready on its own container while the
819 sidecar waits for the keypair (``TLS_PROXY_KEYPAIR_WAIT_SECONDS``); the
820 tcpSocket liveness probe only starts after that wait budget.
821 """
822 # Leaves out what Kubernetes and tls_proxy already default to (the port
823 # protocol, the keypair file paths under /var/run/gco/tls): these specs
824 # ride in the convergence replay input, which SSM caps at 8 KiB encoded.
825 volume_name = "gco-tls"
826 container: dict[str, Any] = {
827 "name": name,
828 "image": image,
829 "imagePullPolicy": "IfNotPresent",
830 "command": ["python", "-m", "gco.services.tls_proxy"],
831 "ports": [{"name": "https", "containerPort": port}],
832 "env": [
833 {"name": "TLS_PROXY_PORT", "value": str(port)},
834 {"name": "TLS_PROXY_UPSTREAM_HOST", "value": "127.0.0.1"},
835 {"name": "TLS_PROXY_UPSTREAM_PORT", "value": str(upstream_port)},
836 {
837 "name": "TLS_PROXY_KEYPAIR_WAIT_SECONDS",
838 "value": str(_CHART_TLS_PROXY_KEYPAIR_WAIT_SECONDS),
839 },
840 ],
841 "securityContext": {
842 "runAsNonRoot": True,
843 "runAsUser": _SERVICE_IMAGE_UID,
844 "runAsGroup": _SERVICE_IMAGE_UID,
845 "allowPrivilegeEscalation": False,
846 "readOnlyRootFilesystem": True,
847 "capabilities": {"drop": ["ALL"]},
848 "seccompProfile": {"type": "RuntimeDefault"},
849 },
850 "resources": {
851 "requests": {"cpu": "10m", "memory": "32Mi"},
852 "limits": {"cpu": "100m", "memory": "128Mi"},
853 },
854 "livenessProbe": {
855 "tcpSocket": {"port": "https"},
856 "initialDelaySeconds": _CHART_TLS_PROXY_KEYPAIR_WAIT_SECONDS + 60,
857 "periodSeconds": 20,
858 "timeoutSeconds": 5,
859 "failureThreshold": 3,
860 },
861 "volumeMounts": [
862 {"name": volume_name, "mountPath": _TLS_PROXY_KEYPAIR_DIR, "readOnly": True}
863 ],
864 }
865 volume: dict[str, Any] = {
866 "name": volume_name,
867 "secret": {"secretName": secret_name, "optional": True, "defaultMode": 0o444},
868 }
869 return container, volume
872def _helm_container_list_string(containers: list[dict[str, Any]]) -> str:
873 """Render containers as the YAML block list a Grafana ``extraContainers`` needs.
875 The Grafana subchart renders ``extraContainers`` with ``tpl`` into the
876 pod's ``containers:`` sequence, so the value must be a *string* of block
877 list items, not a list. The width is unbounded so no scalar is ever
878 folded, which keeps CDK token markers inside the image URI intact until
879 CloudFormation resolves them.
880 """
881 return yaml.safe_dump(
882 containers,
883 default_flow_style=False,
884 sort_keys=False,
885 width=2**31 - 1,
886 )
889def _augment_trusted_registries_with_project_ecr(
890 base: list[str],
891 *,
892 account: str,
893 regions: list[str],
894 global_region: str,
895 url_suffix: str,
896) -> list[str]:
897 """Return the configured trusted registries plus the project's own ECR.
899 The new ``gco images build`` flow pushes images to a per-account ECR
900 registry under ``<account>.dkr.ecr.<region>.<url-suffix>/gco/<name>``.
901 Without this augmentation the queue/manifest validators would treat
902 those URIs as untrusted and reject every job that uses one — which
903 defeats the whole point of the image registry feature.
905 Returns the unique union of the operator-configured ``base`` list
906 plus the per-region project ECR hostnames (one per deployed region,
907 plus the global region where ``gco-global`` provisions the source
908 repo). Order is stable so the rendered ConfigMap doesn't churn
909 between deploys.
910 """
911 augmented: list[str] = list(base)
912 seen = set(augmented)
913 targets = list(dict.fromkeys([global_region, *regions]))
914 if account:
915 for region in targets:
916 host = f"{account}.dkr.ecr.{region}.{url_suffix}"
917 if host not in seen:
918 augmented.append(host)
919 seen.add(host)
920 return augmented
923def _deployment_timestamp() -> str:
924 """Return the synth-time token that deliberately retriggers convergence."""
925 return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
928def _load_helm_chart_order() -> list[str]:
929 """Return helm chart names in their canonical install order.
931 Reads ``lambda/helm-installer/charts.yaml`` (the source of truth, in file
932 order) so the Step Functions state machine has exactly one task per chart,
933 in the same order every deploy — kueue stays last because its mutating
934 webhook intercepts every Job/Deployment. Missing or malformed chart data
935 aborts synthesis rather than silently omitting every Helm install/uninstall.
936 """
937 charts_path = Path(__file__).resolve().parents[2] / "lambda" / "helm-installer" / "charts.yaml"
938 try:
939 with open(charts_path, encoding="utf-8") as f:
940 data = yaml.safe_load(f)
941 except (OSError, yaml.YAMLError) as exc:
942 raise RuntimeError(f"Unable to load Helm chart order from {charts_path}: {exc}") from exc
943 if not isinstance(data, dict):
944 raise RuntimeError(f"Helm chart config {charts_path} must be an object")
945 charts = data.get("charts")
946 if not isinstance(charts, dict) or not charts:
947 raise RuntimeError(
948 f"Helm chart config {charts_path} must contain a non-empty charts object"
949 )
950 if any(not isinstance(name, str) or not name for name in charts):
951 raise RuntimeError(f"Helm chart config {charts_path} contains an invalid chart name")
952 return list(charts)
955class GCORegionalStack(Stack):
956 """
957 Regional resources stack for a single AWS region.
959 Creates EKS cluster, load balancers, and supporting infrastructure
960 for running GCO services in a specific region.
962 Attributes:
963 vpc: VPC with public/private subnets
964 cluster: EKS Auto Mode cluster
965 """
967 @staticmethod
968 def _create_irsa_role(
969 scope: GCORegionalStack,
970 id: str,
971 oidc_provider_arn: str,
972 oidc_issuer_url: str,
973 service_account_names: list[str],
974 namespaces: list[str],
975 *,
976 include_pod_identity: bool = True,
977 ) -> iam.Role:
978 """Create an OIDC IRSA role, optionally trusted by EKS Pod Identity.
980 IRSA is the primary credential mechanism — it works reliably on EKS Auto
981 Mode by projecting a service-account token that the AWS SDK exchanges for
982 temporary credentials via the OIDC provider. General platform roles retain
983 Pod Identity as a secondary path; controller roles can disable it to keep
984 their trust policy bound to one exact Kubernetes service account.
986 Uses CfnJson to defer OIDC condition key resolution to deploy time,
987 because the issuer URL is a CloudFormation token that can't be used
988 as a Python dict key at synth time.
989 """
990 # Strip https:// from issuer URL for the OIDC condition
991 issuer = Fn.select(1, Fn.split("//", oidc_issuer_url))
993 # Build OIDC conditions using CfnJson to defer token resolution
994 # The issuer URL is a CFN token — can't be used as a dict key at synth time
995 aud_key = Fn.join("", [issuer, ":aud"])
996 sub_key = Fn.join("", [issuer, ":sub"])
998 conditions_json = CfnJson(
999 scope,
1000 f"{id}OidcConditions",
1001 value={
1002 aud_key: "sts.amazonaws.com",
1003 sub_key: [
1004 f"system:serviceaccount:{ns}:{sa}"
1005 for ns in namespaces
1006 for sa in service_account_names
1007 ],
1008 },
1009 )
1011 role = iam.Role(
1012 scope,
1013 id,
1014 assumed_by=iam.FederatedPrincipal(
1015 federated=oidc_provider_arn,
1016 conditions={
1017 "StringEquals": conditions_json,
1018 },
1019 assume_role_action="sts:AssumeRoleWithWebIdentity",
1020 ),
1021 )
1023 if include_pod_identity:
1024 # Secondary credential path for platform workloads. Dedicated
1025 # controllers such as LBC deliberately remain OIDC-only.
1026 assert role.assume_role_policy is not None
1027 role.assume_role_policy.add_statements(
1028 iam.PolicyStatement(
1029 effect=iam.Effect.ALLOW,
1030 principals=[iam.ServicePrincipal("pods.eks.amazonaws.com")],
1031 actions=["sts:AssumeRole", "sts:TagSession"],
1032 )
1033 )
1034 return role
1036 def __init__(
1037 self,
1038 scope: Construct,
1039 construct_id: str,
1040 config: ConfigLoader,
1041 region: str,
1042 auth_secret_arn: str,
1043 **kwargs: Any,
1044 ) -> None:
1045 super().__init__(scope, construct_id, **kwargs)
1047 self.config = config
1048 self.deployment_region = region
1049 self.auth_secret_arn = auth_secret_arn
1050 self.disable_efs_automatic_backups = _explicit_context_bool(
1051 self.node.try_get_context(_LIVE_VALIDATION_DISABLE_EFS_BACKUPS_CONTEXT),
1052 key=_LIVE_VALIDATION_DISABLE_EFS_BACKUPS_CONTEXT,
1053 )
1054 retain_provider_logs = self.node.try_get_context(_LIVE_VALIDATION_PROVIDER_LOG_CONTEXT)
1055 self.provider_log_group_removal_policy = (
1056 RemovalPolicy.RETAIN
1057 if retain_provider_logs is True
1058 or (
1059 isinstance(retain_provider_logs, str)
1060 and retain_provider_logs.strip().casefold() == "true"
1061 )
1062 else RemovalPolicy.DESTROY
1063 )
1064 supports_global_accelerator = getattr(config, "supports_global_accelerator", None)
1065 self.global_accelerator_enabled = (
1066 bool(supports_global_accelerator()) if callable(supports_global_accelerator) else True
1067 )
1069 # Get cluster configuration for this region
1070 cluster_config = self.config.get_cluster_config(region)
1071 self.cluster_config = cluster_config
1073 # Create VPC for the EKS cluster.
1074 #
1075 # ``max_azs=99`` is the CDK idiom for "span every Availability Zone the
1076 # region offers" — CDK caps the value at the number of AZs actually
1077 # returned for this account+region, so each AZ gets one public and one
1078 # private subnet. This only enumerates the *real* AZ list when the stack
1079 # is environment-specific (account + region both resolved); app.py sets
1080 # the account from CDK_DEFAULT_ACCOUNT for exactly this reason. In an
1081 # environment-agnostic synth (no account, e.g. some CI paths) CDK falls
1082 # back to a fixed placeholder AZ list rather than the full set.
1083 self.vpc = ec2.Vpc(
1084 self,
1085 "GCOVpc",
1086 # vpc_name intentionally omitted - let CDK generate unique name
1087 max_azs=99, # use every AZ in the region (each AZ gets 1 public + 1 private subnet)
1088 nat_gateways=2, # For high availability
1089 subnet_configuration=[
1090 ec2.SubnetConfiguration(
1091 name="PublicSubnet", subnet_type=ec2.SubnetType.PUBLIC, cidr_mask=24
1092 ),
1093 ec2.SubnetConfiguration(
1094 name="PrivateSubnet",
1095 subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS,
1096 cidr_mask=24,
1097 ),
1098 ],
1099 )
1101 # Enable VPC Flow Logs for network traffic analysis and security monitoring
1102 self._create_vpc_flow_logs()
1104 # Keep AWS API traffic inside the VPC where the operator asked for it
1105 self._create_vpc_endpoints()
1107 # Create SQS queue for job ingestion
1108 self._create_sqs_queue()
1110 # Build the platform service images as CDK Docker image assets
1111 self._create_container_images()
1113 # Pre-create the execution role shared by every ``cr.AwsCustomResource``
1114 # in this stack. See ``_create_aws_custom_resource_role`` for the full
1115 # rationale — in short, CDK's default behavior of auto-generating a
1116 # Lambda role per ``AwsCustomResource`` (and then merging all the
1117 # ``policy=`` statements onto it during deploy) triggers an IAM
1118 # propagation race on cold creates. We sidestep the race by creating
1119 # a single long-lived role up front and attaching policies to it as
1120 # each consumer is built; every ``AwsCustomResource`` then passes
1121 # ``role=self.aws_custom_resource_role`` instead of ``policy=``, so
1122 # the singleton Lambda runs against a role whose inline policy has
1123 # already replicated globally.
1124 self._create_aws_custom_resource_role()
1126 # Resolve this region's fixed ACM certificate ARN from the global
1127 # backend-TLS registry before rendering the HTTPS-only Gateway.
1128 self.backend_tls_certificate_arn = self._resolve_backend_tls_certificate_arn()
1130 # Create EKS cluster
1131 self._create_eks_cluster(cluster_config)
1133 # Optional Volcano image mirror (cdk.json ``volcano_image_mirror``).
1134 # When enabled this resolves ``self.volcano_mirror_registry`` — the
1135 # gco/* ECR namespace that Volcano's ``basic.image_registry`` is
1136 # redirected to, so its docker.io-only images are pulled from the
1137 # project's own ECR (populated out-of-band by
1138 # ``gco images mirror``) instead of rate-limited Docker
1139 # Hub. Creates no CloudFormation resources; must run before
1140 # ``_apply_kubernetes_manifests`` builds the ``HelmInstallCharts`` custom
1141 # resource, which reads the override via ``_helm_chart_value_overrides()``.
1142 self._configure_volcano_image_mirror()
1144 # Resolve the always-on Cluster_Shared_Bucket identity from SSM
1145 # (owned by GCOGlobalStack) and attach RW + KMS grants to the
1146 # job-pod role. Runs unconditionally — the ConfigMap and IAM
1147 # statements are always present on every regional cluster. Must
1148 # run after
1149 # _create_pod_identity_associations (which created service_account_role)
1150 # and before _apply_kubernetes_manifests (which consumes the
1151 # replacements in the KubectlApplyManifests CustomResource).
1152 self.cluster_shared_identity = self._resolve_cluster_shared_bucket_from_ssm()
1153 self._grant_cluster_shared_bucket_to_job_role(self.cluster_shared_identity)
1155 # MLflow artifact storage: a dedicated OIDC-only IRSA role for the
1156 # tracking server's service account, scoped to the mlflow-artifacts/
1157 # prefix of the same shared bucket. Created here (not with the other
1158 # IRSA roles) because it needs the resolved bucket identity above;
1159 # must precede _apply_kubernetes_manifests, whose value overrides
1160 # inject the role ARN into the chart's service-account annotation.
1161 if self._mlflow_active():
1162 self._create_mlflow_artifact_role(self.cluster_shared_identity)
1164 # Create the always-on general-purpose regional bucket (KMS key +
1165 # access-logs bucket + primary bucket). Provisioned unconditionally —
1166 # there is no cdk.json toggle and no feature flag gating its existence —
1167 # in addition to the central buckets owned by GCOGlobalStack.
1168 self._create_regional_shared_bucket()
1170 # Create EFS for shared storage
1171 self._create_efs()
1173 # Create FSx for Lustre (if enabled) for high-performance storage
1174 self._create_fsx_lustre()
1176 # Create Valkey Serverless cache (if enabled) for K/V caching
1177 self._create_valkey_cache()
1179 # Create Aurora Serverless v2 + pgvector (if enabled) for vector DB
1180 self._create_aurora_pgvector()
1182 # Discover and publish the Gateway ALB in every partition. Global
1183 # Accelerator registration is an optional extension of this same exact
1184 # ownership path where the service is available.
1185 self._create_ga_registration_lambda()
1187 # Provider framework Lambdas can emit their final delete-event log after
1188 # CloudFormation has otherwise finished the custom resource. Strict live
1189 # validation retains this explicit group through stack deletion so the
1190 # harness can remove the same checkpointed generation after every target
1191 # stack is absent. Ordinary deployments keep DESTROY semantics and do not
1192 # accumulate retained groups.
1193 self.helm_installer_provider_log_group = logs.LogGroup(
1194 self,
1195 "HelmInstallerProviderLogGroup",
1196 retention=logs.RetentionDays.ONE_WEEK,
1197 removal_policy=self.provider_log_group_removal_policy,
1198 )
1200 # Create Helm installer Lambda for KEDA and other Helm-based installations
1201 self._create_helm_installer_lambda()
1203 # Apply Kubernetes manifests (after EFS so IDs are available)
1204 self._apply_kubernetes_manifests()
1206 # Switch on CloudWatch Transaction Search in this Region: the X-Ray
1207 # OTLP endpoint the traced API services export to requires it
1208 # (tracing.enabled + tracing.enable_transaction_search).
1209 self._create_transaction_search_resource()
1211 # Create CloudFormation drift detection (daily schedule + SNS alerts)
1212 self._create_drift_detection()
1214 # Create dedicated IAM role for MCP server
1215 self._create_mcp_role()
1217 # Export cluster information
1218 self._create_outputs()
1220 # Apply cdk-nag suppressions for this stack
1221 self._apply_nag_suppressions()
1223 def _create_vpc_flow_logs(self) -> None:
1224 """Create VPC Flow Logs for network traffic monitoring.
1226 Flow logs capture information about IP traffic going to and from
1227 network interfaces in the VPC. This is required for security
1228 monitoring and compliance (HIPAA, SOC2, etc.).
1229 """
1230 # Create CloudWatch Log Group for flow logs
1231 flow_log_group = logs.LogGroup(
1232 self,
1233 "VpcFlowLogGroup",
1234 # log_group_name intentionally omitted - let CDK generate unique name
1235 retention=logs.RetentionDays.ONE_MONTH,
1236 removal_policy=RemovalPolicy.DESTROY,
1237 )
1239 # Create IAM role for VPC Flow Logs
1240 flow_log_role = iam.Role(
1241 self,
1242 "VpcFlowLogRole",
1243 assumed_by=iam.ServicePrincipal("vpc-flow-logs.amazonaws.com"),
1244 )
1246 flow_log_role.add_to_policy(
1247 iam.PolicyStatement(
1248 actions=[
1249 "logs:CreateLogStream",
1250 "logs:PutLogEvents",
1251 "logs:DescribeLogGroups",
1252 "logs:DescribeLogStreams",
1253 ],
1254 resources=[flow_log_group.log_group_arn, f"{flow_log_group.log_group_arn}:*"],
1255 )
1256 )
1258 # Create VPC Flow Log
1259 ec2.FlowLog(
1260 self,
1261 "VpcFlowLog",
1262 resource_type=ec2.FlowLogResourceType.from_vpc(self.vpc),
1263 destination=ec2.FlowLogDestination.to_cloud_watch_logs(flow_log_group, flow_log_role),
1264 traffic_type=ec2.FlowLogTrafficType.ALL,
1265 )
1267 def _create_vpc_endpoints(self) -> None:
1268 """Create the VPC endpoints selected by ``cdk.json`` ``vpc_endpoints``.
1270 Gateway endpoints (S3, DynamoDB) are free route-table entries: with
1271 them, the platform's largest data path — model and dataset pulls,
1272 checkpoints, MLflow artifacts, cost reports — stops flowing through the
1273 NAT gateways' per-GB metering and never leaves the VPC. Interface
1274 endpoints are PrivateLink ENIs billed per AZ-hour, so they are opt-in;
1275 CDK gives each one a security group admitting HTTPS from the VPC CIDR
1276 and private DNS, so callers keep using the public service hostnames.
1278 NetworkPolicy egress is unaffected either way: S3 traffic still
1279 resolves to public S3 addresses that the route table steers into the
1280 endpoint, which is why 03-network-policies.yaml allows HTTPS by port
1281 rather than by destination.
1282 """
1283 selection = self.config.get_vpc_endpoints_config()
1284 self.vpc_gateway_endpoints: dict[str, ec2.GatewayVpcEndpoint] = {}
1285 self.vpc_interface_endpoints: dict[str, ec2.InterfaceVpcEndpoint] = {}
1286 for service in selection["gateway"]:
1287 self.vpc_gateway_endpoints[service] = self.vpc.add_gateway_endpoint(
1288 f"VpcEndpoint-{service}",
1289 service=_GATEWAY_ENDPOINT_SERVICES[service],
1290 )
1291 for service in selection["interface"]:
1292 self.vpc_interface_endpoints[service] = self.vpc.add_interface_endpoint(
1293 f"VpcEndpoint-{service.replace('.', '-')}",
1294 service=_INTERFACE_ENDPOINT_SERVICES[service],
1295 subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
1296 )
1298 def _apply_nag_suppressions(self) -> None:
1299 """Apply cdk-nag suppressions for this stack."""
1300 from gco.stacks.nag_suppressions import apply_all_suppressions
1302 apply_all_suppressions(
1303 self,
1304 stack_type="regional",
1305 regions=self.config.get_regions(),
1306 global_region=self.config.get_global_region(),
1307 project_name=self.config.get_project_name(),
1308 )
1310 def _create_sqs_queue(self) -> None:
1311 """Create SQS queue for job ingestion.
1313 Creates an SQS queue that serves as the default job ingestion point
1314 for this region. Jobs submitted to this queue are processed by the
1315 manifest processor and KEDA scales based on queue depth.
1317 Also creates a dead-letter queue for failed messages.
1318 Both queues use server-side encryption with AWS managed keys.
1319 """
1320 project_name = self.config.get_project_name()
1322 # Create dead-letter queue for failed messages
1323 self.job_dlq = sqs.Queue(
1324 self,
1325 "JobDeadLetterQueue",
1326 queue_name=f"{project_name}-jobs-dlq-{self.deployment_region}",
1327 retention_period=Duration.days(14),
1328 removal_policy=RemovalPolicy.DESTROY,
1329 enforce_ssl=True, # Require SSL for all requests
1330 encryption=sqs.QueueEncryption.SQS_MANAGED, # Server-side encryption
1331 )
1333 # Create main job queue
1334 self.job_queue = sqs.Queue(
1335 self,
1336 "JobQueue",
1337 queue_name=f"{project_name}-jobs-{self.deployment_region}",
1338 visibility_timeout=Duration.minutes(5), # Match Lambda timeout
1339 retention_period=Duration.days(7),
1340 dead_letter_queue=sqs.DeadLetterQueue(
1341 max_receive_count=3, # Move to DLQ after 3 failed attempts
1342 queue=self.job_dlq,
1343 ),
1344 removal_policy=RemovalPolicy.DESTROY,
1345 enforce_ssl=True, # Require SSL for all requests
1346 encryption=sqs.QueueEncryption.SQS_MANAGED, # Server-side encryption
1347 )
1349 # Output queue information
1350 CfnOutput(
1351 self,
1352 "JobQueueUrl",
1353 value=self.job_queue.queue_url,
1354 description=f"SQS Job Queue URL for {self.deployment_region}",
1355 export_name=f"{project_name}-job-queue-url-{self.deployment_region}",
1356 )
1358 CfnOutput(
1359 self,
1360 "JobQueueArn",
1361 value=self.job_queue.queue_arn,
1362 description=f"SQS Job Queue ARN for {self.deployment_region}",
1363 export_name=f"{project_name}-job-queue-arn-{self.deployment_region}",
1364 )
1366 CfnOutput(
1367 self,
1368 "JobDlqUrl",
1369 value=self.job_dlq.queue_url,
1370 description=f"SQS Dead Letter Queue URL for {self.deployment_region}",
1371 export_name=f"{project_name}-job-dlq-url-{self.deployment_region}",
1372 )
1374 def _create_aws_custom_resource_role(self) -> None:
1375 """Pre-create the execution role shared by every ``AwsCustomResource``.
1377 CDK's ``cr.AwsCustomResource`` defaults to auto-generating a per-
1378 construct Lambda execution role from the ``policy=`` parameter.
1379 Internally, CDK deduplicates those auto-generated roles onto a
1380 single *singleton* provider Lambda (logical id prefix
1381 ``AWS679f53fac002430cb0da5b7982bd22872``), and merges each custom
1382 resource's policy statements onto that Lambda's role at stack
1383 create time. On cold deploys, CloudFormation invokes the Lambda
1384 within 2-3 seconds of attaching a new policy statement, which is
1385 faster than IAM's global propagation window. The symptom is a
1386 ``iam:PassRole NOT authorized`` failure on whichever addon role
1387 update happens to run right after its ``iam:PassRole`` policy
1388 statement was attached but before it had replicated.
1390 The fix is to create the role up front, attach every policy
1391 statement the stack will need during stack creation, and pass
1392 ``role=self.aws_custom_resource_role`` to every
1393 ``AwsCustomResource`` instead of ``policy=``. Because the role
1394 already exists — and its inline policy has had minutes to
1395 replicate by the time any ``AwsCustomResource`` actually fires —
1396 the race disappears entirely.
1398 This method creates the role with the statements we can compute
1399 without a cluster reference (EKS ``UpdateAddon`` / ``DescribeAddon``
1400 scoped to this cluster, and SSM ``GetParameter`` for the endpoint
1401 group ARN). ``iam:PassRole`` statements for individual addon
1402 roles (EFS CSI, FSx CSI, CloudWatch Observability) are appended
1403 by each ``_create_*_addon`` method after the corresponding IRSA
1404 role has been created, so every PassRole ``resources=`` list
1405 stays precise (no wildcards) and cdk-nag stays happy.
1406 """
1407 project_name = self.config.get_project_name()
1408 global_region = self.config.get_global_region()
1410 self.aws_custom_resource_role = iam.Role(
1411 self,
1412 "AwsCustomResourceRole",
1413 assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
1414 description=(
1415 "Shared execution role for every cr.AwsCustomResource in this "
1416 "stack. Pre-created to avoid the IAM policy propagation race "
1417 "that occurs when CDK auto-generates per-CR roles and the "
1418 "singleton provider Lambda fires before the freshly-attached "
1419 "policy has replicated globally."
1420 ),
1421 managed_policies=[
1422 iam.ManagedPolicy.from_aws_managed_policy_name(
1423 "service-role/AWSLambdaBasicExecutionRole"
1424 ),
1425 ],
1426 )
1428 # EKS UpdateAddon / DescribeAddon — used by the three updateAddon
1429 # custom resources (EFS CSI, FSx CSI, CloudWatch Observability).
1430 # Scoped to this cluster's addons by ARN.
1431 self.aws_custom_resource_role.add_to_policy(
1432 iam.PolicyStatement(
1433 effect=iam.Effect.ALLOW,
1434 actions=["eks:UpdateAddon", "eks:DescribeAddon"],
1435 resources=[
1436 f"arn:{self.partition}:eks:{self.deployment_region}:{self.account}"
1437 f":addon/{self.cluster_config.cluster_name}/*"
1438 ],
1439 )
1440 )
1442 # SSM GetParameter — used by the GetEndpointGroupArn custom
1443 # resource in _create_ga_registration_lambda to read the ARN of
1444 # the Global Accelerator endpoint group published by the global
1445 # stack during its deploy.
1446 self.aws_custom_resource_role.add_to_policy(
1447 iam.PolicyStatement(
1448 effect=iam.Effect.ALLOW,
1449 actions=["ssm:GetParameter"],
1450 resources=[
1451 f"arn:{self.partition}:ssm:{global_region}:{self.account}:"
1452 f"parameter/{project_name}/*"
1453 ],
1454 )
1455 )
1457 # cdk-nag suppressions: the two wildcard-bearing ARNs above are
1458 # intentional and both scoped as tightly as AWS IAM permits.
1459 #
1460 # - The ``eks:UpdateAddon`` / ``eks:DescribeAddon`` statement uses
1461 # ``addon/<cluster>/*`` as its resource because the same shared
1462 # role is consumed by three different updateAddon custom
1463 # resources (EFS CSI, FSx CSI, CloudWatch Observability). Each
1464 # addon has its own ARN and we'd otherwise need three separate
1465 # statements that each grant access to a known addon name. The
1466 # wildcard is scoped to a single cluster in a single region in
1467 # a single account — it cannot be used against any addon
1468 # belonging to a different cluster or a different service.
1469 #
1470 # - The ``ssm:GetParameter`` statement uses
1471 # ``parameter/<project>/*`` because the exact parameter name
1472 # (``endpoint-group-<region>-arn``) is only known at Global
1473 # Accelerator registration time and the endpoint path
1474 # structure is ``<project>/<parameter>``. Scoping to the
1475 # project prefix restricts access to parameters owned by this
1476 # project only.
1477 from gco.stacks.nag_suppressions import acknowledge_nag_findings
1479 acknowledge_nag_findings(
1480 self.aws_custom_resource_role,
1481 [
1482 {
1483 "id": "AwsSolutions-IAM5",
1484 "reason": (
1485 "Scoped to a single EKS cluster's addons "
1486 "(addon/<cluster>/*) and this project's SSM "
1487 "parameters (parameter/<project>/*). Both wildcards "
1488 "are as tight as AWS IAM permits: addon names and "
1489 "parameter names are not known at stack synthesis "
1490 "time because the addons are created later in the "
1491 "same stack and the GA endpoint group ARN is "
1492 "published by a separate stack during deploy. The "
1493 "shared role pattern itself is deliberate — see "
1494 "_create_aws_custom_resource_role docstring for why "
1495 "we pre-create instead of letting CDK auto-generate "
1496 "per-CR roles."
1497 ),
1498 "appliesTo": [
1499 f"Resource::arn:<AWS::Partition>:eks:{self.deployment_region}"
1500 f":<AWS::AccountId>:addon/{self.cluster_config.cluster_name}/*",
1501 f"Resource::arn:<AWS::Partition>:ssm:{global_region}"
1502 f":<AWS::AccountId>:parameter/{project_name}/*",
1503 ],
1504 },
1505 ],
1506 )
1508 def _resolve_backend_tls_certificate_arn(self) -> str:
1509 """Read this region's stable imported ACM ARN from global-region SSM.
1511 The certificate manager publishes one fixed ARN per workload region.
1512 The regional Ingress consumes the token directly, which creates a
1513 CloudFormation dependency ensuring the certificate exists before the
1514 HTTPS listener is reconciled. The shared custom-resource role is
1515 already restricted to this project's SSM namespace.
1516 """
1517 project_name = self.config.get_project_name()
1518 parameter_name = backend_tls_certificate_arn_parameter_name(
1519 project_name, self.deployment_region
1520 )
1521 reader = cr.AwsCustomResource(
1522 self,
1523 "GetBackendTlsCertificateArn",
1524 on_create=cr.AwsSdkCall(
1525 service="SSM",
1526 action="getParameter",
1527 parameters={"Name": parameter_name},
1528 region=self.config.get_global_region(),
1529 physical_resource_id=cr.PhysicalResourceId.of(
1530 f"{project_name}-backend-tls-certificate-{self.deployment_region}"
1531 ),
1532 ),
1533 on_update=cr.AwsSdkCall(
1534 service="SSM",
1535 action="getParameter",
1536 parameters={"Name": parameter_name},
1537 region=self.config.get_global_region(),
1538 ),
1539 role=self.aws_custom_resource_role,
1540 )
1541 reader.node.add_dependency(self.aws_custom_resource_role)
1542 return str(reader.get_response_field("Parameter.Value"))
1544 def _service_image_asset(self, construct_id: str, service: str) -> ecr_assets.DockerImageAsset:
1545 """Build one platform service image from ``dockerfiles/Dockerfile.<service>``.
1547 The build context is the repository root because every service image
1548 ``COPY``s shared ``gco/`` code, and the exclude list drops the inputs
1549 that cannot affect this service so an unrelated edit does not churn
1550 its asset hash. All images target AMD64 (x86_64) only (an image asset
1551 is single-platform), so every pod that runs one pins
1552 ``kubernetes.io/arch: amd64``: the ``cpu-general`` NodePool also
1553 provisions Graviton nodes.
1554 """
1555 dockerfile = service_dockerfile(service)
1556 return ecr_assets.DockerImageAsset(
1557 self,
1558 construct_id,
1559 directory=".",
1560 file=dockerfile,
1561 platform=ecr_assets.Platform.LINUX_AMD64,
1562 exclude=_service_image_asset_excludes(dockerfile),
1563 )
1565 def _create_container_images(self) -> None:
1566 """Build the platform service images as CDK Docker image assets.
1568 Every image is a ``DockerImageAsset``: CDK builds it at synth, pushes
1569 it to the bootstrap asset repository under a content-hash tag, and the
1570 manifests receive that URI through ``{{*_IMAGE}}`` replacements. The
1571 stack creates no ECR repositories of its own — three empty
1572 per-service repositories used to be declared here and were never
1573 pushed to or referenced; they only added resources to create, scan,
1574 and delete on every deploy.
1575 """
1577 self.health_monitor_image = self._service_image_asset(
1578 "HealthMonitorImage", "health-monitor"
1579 )
1580 self.manifest_processor_image = self._service_image_asset(
1581 "ManifestProcessorImage", "manifest-processor"
1582 )
1583 # The inference-only data-plane proxy is its own image. Keeping it
1584 # separate from manifest-processor prevents model traffic from sharing
1585 # its Kubernetes API/RBAC and queue-worker process surface.
1586 self.inference_proxy_image = self._service_image_asset(
1587 "InferenceProxyImage", "inference-proxy"
1588 )
1590 # Output image URIs for reference
1591 CfnOutput(
1592 self,
1593 "HealthMonitorImageUri",
1594 value=self.health_monitor_image.image_uri,
1595 description="Health Monitor Docker image URI",
1596 )
1598 CfnOutput(
1599 self,
1600 "ManifestProcessorImageUri",
1601 value=self.manifest_processor_image.image_uri,
1602 description="Manifest Processor Docker image URI",
1603 )
1605 CfnOutput(
1606 self,
1607 "InferenceProxyImageUri",
1608 value=self.inference_proxy_image.image_uri,
1609 description="Inference Proxy Docker image URI",
1610 )
1612 self.inference_monitor_image = self._service_image_asset(
1613 "InferenceMonitorImage", "inference-monitor"
1614 )
1616 CfnOutput(
1617 self,
1618 "InferenceMonitorImageUri",
1619 value=self.inference_monitor_image.image_uri,
1620 description="Inference Monitor Docker image URI",
1621 )
1623 # Build and push queue processor Docker image (if enabled).
1624 # The queue processor is a KEDA ScaledJob that consumes manifests from
1625 # the regional SQS queue. It can be disabled in cdk.json if users want
1626 # to implement their own consumer. When disabled, the post-helm-sqs-consumer.yaml
1627 # manifest is skipped (unreplaced template variables cause it to be skipped).
1628 queue_processor_config = self.node.try_get_context("queue_processor") or {}
1629 self.queue_processor_enabled = queue_processor_config.get("enabled", True)
1631 if self.queue_processor_enabled:
1632 self.queue_processor_image = self._service_image_asset(
1633 "QueueProcessorImage", "queue-processor"
1634 )
1636 CfnOutput(
1637 self,
1638 "QueueProcessorImageUri",
1639 value=self.queue_processor_image.image_uri,
1640 description="Queue Processor Docker image URI",
1641 )
1643 # Build and push the cost-monitor image only when the cost monitoring
1644 # pipeline deploys to this region — skipping the build keeps opted-out
1645 # deployments' synth/deploy time unchanged (same gating rationale as
1646 # the queue processor above).
1647 if self._cost_monitoring_active():
1648 self.cost_monitor_image = self._service_image_asset("CostMonitorImage", "cost-monitor")
1650 CfnOutput(
1651 self,
1652 "CostMonitorImageUri",
1653 value=self.cost_monitor_image.image_uri,
1654 description="Cost Monitor Docker image URI",
1655 )
1657 def _resolve_unsupported_az_names(self) -> list[str]:
1658 """Resolve this region's EKS-unsupported AZ *IDs* to this account's AZ *names*.
1660 EKS rejects cluster subnets in a small set of Availability Zones,
1661 published by AZ ID (``EKS_UNSUPPORTED_AZ_IDS``). AZ *names* are
1662 randomized per account, so the disallowed ``use1-az3`` may be
1663 ``us-east-1e`` in one account and a different name in another — we must
1664 map ID -> name for the deploy account.
1666 Returns an empty list when the region has no restriction (the common
1667 case) or when the deploy account is not resolved. A credentialed,
1668 environment-specific synth fails closed if EC2 cannot resolve every
1669 restricted AZ ID; selecting all private subnets in that case could hand
1670 EKS a known-unsupported control-plane subnet.
1671 """
1672 unsupported_ids = EKS_UNSUPPORTED_AZ_IDS.get(self.deployment_region, ())
1673 if not unsupported_ids:
1674 return []
1675 # Only reach EC2 during a credentialed, environment-specific synth or
1676 # deploy. The CDK CLI exports CDK_DEFAULT_ACCOUNT from the active
1677 # identity; unit tests and agnostic synth don't, so we never call AWS
1678 # (nor block synthesis on missing credentials) there.
1679 if not os.environ.get("CDK_DEFAULT_ACCOUNT"):
1680 return []
1681 try:
1682 import boto3
1683 from botocore.config import Config
1685 ec2_client = boto3.client(
1686 "ec2",
1687 region_name=self.deployment_region,
1688 config=Config(connect_timeout=5, read_timeout=5, retries={"max_attempts": 2}),
1689 )
1690 response = ec2_client.describe_availability_zones(
1691 Filters=[{"Name": "zone-id", "Values": list(unsupported_ids)}]
1692 )
1693 except Exception as exc:
1694 raise RuntimeError(
1695 f"Unable to resolve EKS-unsupported Availability Zones in "
1696 f"{self.deployment_region}: {exc}"
1697 ) from exc
1699 zones = response.get("AvailabilityZones")
1700 if not isinstance(zones, list):
1701 raise RuntimeError(
1702 f"EC2 returned malformed Availability Zone data for {self.deployment_region}"
1703 )
1704 names_by_id = {
1705 zone_id: zone_name
1706 for zone in zones
1707 if isinstance(zone, dict)
1708 and isinstance((zone_id := zone.get("ZoneId")), str)
1709 and isinstance((zone_name := zone.get("ZoneName")), str)
1710 and zone_id in unsupported_ids
1711 and zone_name
1712 }
1713 missing_ids = [zone_id for zone_id in unsupported_ids if zone_id not in names_by_id]
1714 if missing_ids:
1715 raise RuntimeError(
1716 f"EC2 did not resolve EKS-unsupported Availability Zone IDs in "
1717 f"{self.deployment_region}: {', '.join(missing_ids)}"
1718 )
1719 return [names_by_id[zone_id] for zone_id in unsupported_ids]
1721 def _eks_control_plane_subnets(self) -> ec2.SubnetSelection:
1722 """Private-subnet selection for the EKS control plane, excluding any AZ
1723 EKS does not support for cluster subnets.
1725 Records the outcome on ``self`` (``eks_unsupported_az_names`` and
1726 ``eks_control_plane_subnets``) so tests and operators can introspect
1727 exactly which subnets the cluster was given.
1728 """
1729 unsupported = set(self._resolve_unsupported_az_names())
1730 usable = [
1731 subnet
1732 for subnet in self.vpc.private_subnets
1733 if subnet.availability_zone not in unsupported
1734 ]
1735 self.eks_unsupported_az_names = sorted(unsupported)
1736 self.eks_control_plane_subnets = usable
1737 if not unsupported:
1738 # No restricted AZ in this region: keep the subnet-type selection so
1739 # the synthesized template is identical to before for the common case.
1740 return ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS)
1741 return ec2.SubnetSelection(subnets=usable)
1743 def _create_eks_cluster(self, cluster_config: Any) -> None:
1744 """Create the EKS cluster with auto mode and GPU node groups"""
1746 # Create cluster admin role
1747 # role_name intentionally omitted - let CDK generate unique name
1748 cluster_admin_role = iam.Role(
1749 self,
1750 "ClusterAdminRole",
1751 assumed_by=iam.ServicePrincipal("eks.amazonaws.com"),
1752 managed_policies=[
1753 iam.ManagedPolicy.from_aws_managed_policy_name("AmazonEKSClusterPolicy")
1754 ],
1755 )
1757 # No separate node role is created here: EKS Auto Mode nodes use the
1758 # node role the Cluster construct provisions for ``compute``. A former
1759 # standalone, unreferenced "NodeGroupRole" was removed because it
1760 # granted worker-node policies to nothing and, once something outside
1761 # the stack placed it in an instance profile, blocked stack deletion.
1763 # Create EKS Auto Mode cluster with built-in system and general-purpose nodepools
1764 # Auto Mode automatically manages compute resources and comes with essential addons
1765 # Get endpoint access configuration
1766 eks_config = self.config.get_eks_cluster_config()
1767 endpoint_access_mode = eks_config.get("endpoint_access", "PRIVATE")
1769 # Map config string to EKS EndpointAccess enum. When the public
1770 # endpoint is enabled, eks_cluster.public_access_cidrs restricts who
1771 # can reach it; enabling it with no allowlist is a deliberate,
1772 # loudly-announced 0.0.0.0/0 exposure rather than a silent default.
1773 public_access_cidrs = [str(cidr) for cidr in (eks_config.get("public_access_cidrs") or [])]
1774 if endpoint_access_mode == "PRIVATE":
1775 endpoint_access = eks.EndpointAccess.PRIVATE
1776 elif public_access_cidrs:
1777 endpoint_access = eks.EndpointAccess.PUBLIC_AND_PRIVATE.only_from(*public_access_cidrs)
1778 else:
1779 Annotations.of(self).add_warning(
1780 "eks_cluster.endpoint_access is PUBLIC_AND_PRIVATE with no "
1781 "public_access_cidrs allowlist: the EKS API server accepts "
1782 "connections from 0.0.0.0/0 (authentication still applies). "
1783 "Set eks_cluster.public_access_cidrs to your egress CIDRs "
1784 "(gco stacks eks endpoint set PUBLIC_AND_PRIVATE --cidr <cidr>) "
1785 "or use PRIVATE with `gco cluster tunnel`."
1786 )
1787 endpoint_access = eks.EndpointAccess.PUBLIC_AND_PRIVATE
1789 # Create KMS key for EKS secrets encryption
1790 self.eks_encryption_key = kms.Key(
1791 self,
1792 "EksSecretsEncryptionKey",
1793 description="KMS key for EKS Kubernetes secrets encryption",
1794 enable_key_rotation=True,
1795 removal_policy=RemovalPolicy.RETAIN,
1796 )
1798 # The cluster minor comes from cdk.json (``context.kubernetes_version``).
1799 # aws-cdk-lib's KubernetesVersion enum trails EKS by a release or two;
1800 # ``KubernetesVersion.of()`` is the library's supported path for a minor
1801 # it does not name yet and renders the same AWS::EKS::Cluster Version.
1802 k8s_version_str = cluster_config.kubernetes_version
1803 try:
1804 k8s_version = getattr(eks.KubernetesVersion, f"V{k8s_version_str.replace('.', '_')}")
1805 except AttributeError:
1806 k8s_version = eks.KubernetesVersion.of(k8s_version_str)
1808 self.cluster = eks.Cluster(
1809 self,
1810 "GCOEksCluster",
1811 cluster_name=cluster_config.cluster_name,
1812 version=k8s_version,
1813 vpc=self.vpc,
1814 compute=eks.ComputeConfig(
1815 # Enable both built-in node pools - Auto Mode manages these automatically
1816 node_pools=["system", "general-purpose"]
1817 ),
1818 # SECURITY: Endpoint access controlled via cdk.json eks_cluster.endpoint_access
1819 # PRIVATE (default): EKS API accessible only from within VPC - most secure
1820 # Job submission works via API Gateway → Lambda (in VPC) or SQS
1821 # For kubectl access, use a bastion host, VPN, or AWS SSM Session Manager
1822 # PUBLIC_AND_PRIVATE: EKS API accessible from internet and VPC
1823 # Allows direct kubectl access but less secure
1824 endpoint_access=endpoint_access,
1825 role=cluster_admin_role,
1826 # The VPC spans every AZ, but EKS refuses control-plane subnets in a
1827 # few AZs (by stable AZ ID; see EKS_UNSUPPORTED_AZ_IDS). Select the
1828 # private subnets in supported AZs only — worker/other subnets in the
1829 # excluded AZs still exist in the VPC.
1830 vpc_subnets=[self._eks_control_plane_subnets()],
1831 # Enable all control plane logging for security and compliance
1832 cluster_logging=[
1833 eks.ClusterLoggingTypes.API,
1834 eks.ClusterLoggingTypes.AUDIT,
1835 eks.ClusterLoggingTypes.AUTHENTICATOR,
1836 eks.ClusterLoggingTypes.CONTROLLER_MANAGER,
1837 eks.ClusterLoggingTypes.SCHEDULER,
1838 ],
1839 # SECURITY: Enable envelope encryption for Kubernetes secrets using KMS
1840 secrets_encryption_key=self.eks_encryption_key,
1841 )
1843 # The EKS cluster security group's auto-generated ingress rule allows
1844 # 443 from the VPC CIDR, expressed as a CloudFormation token. cdk-nag's
1845 # SG-ingress rules can't resolve the token and throw; scope the
1846 # acknowledgment to the cluster construct so it can't mask an
1847 # open-ingress finding elsewhere in the stack.
1848 from gco.stacks.nag_suppressions import acknowledge_security_group_cidr_findings
1850 acknowledge_security_group_cidr_findings(
1851 self.cluster,
1852 reason=(
1853 "The EKS Auto Mode cluster security group allows HTTPS (443) "
1854 "ingress from the VPC CIDR only, referenced via an "
1855 "``Fn::GetAtt`` token that cdk-nag cannot resolve at synth "
1856 "time. Ingress is restricted to intra-VPC traffic, the "
1857 "tightest possible source for the Kubernetes API and webhook "
1858 "endpoints."
1859 ),
1860 )
1862 # Auto Mode comes with essential addons pre-configured:
1863 # - AWS Load Balancer Controller (for ALB/NLB integration)
1864 # - CoreDNS, kube-proxy, VPC CNI (standard Kubernetes components)
1866 # OIDC provider for IRSA — the primary credential injection mechanism.
1867 # IRSA uses projected service-account tokens exchanged via the OIDC provider
1868 # for temporary AWS credentials. This works reliably on EKS Auto Mode.
1869 self.oidc_provider = eks.OidcProviderNative(
1870 self,
1871 "OidcProvider",
1872 url=self.cluster.cluster_open_id_connect_issuer_url,
1873 )
1875 # Pod Identity Agent add-on — registers the admission webhook that injects
1876 # Pod Identity credentials. On Auto Mode the DaemonSet schedules 0 pods
1877 # (the agent is built into the node), but the add-on registration is still
1878 # needed for the control-plane webhook. Kept as a secondary credential path.
1879 self._create_pod_identity_agent_addon()
1881 # Add Metrics Server add-on for HPA and resource monitoring
1882 self._create_metrics_server_addon()
1884 # Add EFS CSI Driver add-on for shared storage
1885 self._create_efs_csi_driver_addon()
1887 # Add CloudWatch Observability add-on for Container Insights metrics
1888 self._create_cloudwatch_observability_addon()
1890 # NOTE: GPU compute is configured via Karpenter NodePools (not managed node groups)
1891 # NodePool manifests are located in lambda/kubectl-applier-simple/manifests/:
1892 # - 40-nodepool-gpu-x86.yaml: active x86_64 general GPU instances (g4dn, g5, g6/g6e/g6f/gr6/gr6f, g7/g7e; deprecated p3/p3dn excluded)
1893 # - 41-nodepool-gpu-arm.yaml: ARM64 GPU instances (g5g)
1894 # - 42-nodepool-inference.yaml: inference-optimized GPU instances
1895 # - 43-nodepool-efa.yaml: EFA-enabled instances (p4d, p4de, p5/p5e/p5en, p6-b200/p6-b300/p6e-gb200)
1896 # - 44-nodepool-neuron.yaml: Trainium/Inferentia instances
1897 # These will be applied by the kubectl Lambda custom resource (created below)
1899 # Developer access entries (eks_cluster.developer_access). The cluster
1900 # authenticates through EKS access entries only, and until now no human
1901 # principal could be granted one at deploy time — every access entry
1902 # belonged to a platform Lambda. Absent or empty config synthesizes
1903 # exactly today's entries.
1904 self._create_developer_access_entries(eks_config)
1906 # EKS Capabilities (eks_capabilities): AWS-managed ACK and kro attached
1907 # as AWS::EKS::Capability resources. Every type is off by default; the
1908 # shipped cdk.json synthesizes exactly today's template.
1909 self._create_eks_capabilities()
1911 # Create IRSA role for service account to access secrets
1912 self._create_service_account_role()
1914 # Create kubectl Lambda for applying Kubernetes manifests
1915 self._create_kubectl_lambda()
1917 def _create_developer_access_entries(self, eks_config: dict[str, Any]) -> None:
1918 """Synthesize one EKS access entry per configured developer principal.
1920 Each ``eks_cluster.developer_access`` element is
1921 ``{principal_arn, scope, namespaces}``. The default is deliberately
1922 the narrow grant — AmazonEKSEditPolicy scoped to the ``gco-jobs``
1923 namespace, enough to submit and inspect jobs — not cluster-admin.
1924 ``scope: cluster`` opts into AmazonEKSClusterAdminPolicy explicitly.
1925 Config errors fail synthesis: a typo'd access grant must never
1926 silently synthesize as nothing.
1927 """
1928 entries = eks_config.get("developer_access") or []
1929 if not isinstance(entries, list):
1930 raise ValueError("eks_cluster.developer_access must be a list of entries")
1931 for index, entry in enumerate(entries):
1932 if not isinstance(entry, dict):
1933 raise ValueError(
1934 f"eks_cluster.developer_access[{index}] must be an object with "
1935 "principal_arn, and optionally scope and namespaces"
1936 )
1937 principal_arn = entry.get("principal_arn")
1938 if not isinstance(principal_arn, str) or not principal_arn.startswith("arn:"):
1939 raise ValueError(
1940 f"eks_cluster.developer_access[{index}].principal_arn must be an "
1941 f"IAM principal ARN, got {principal_arn!r}"
1942 )
1943 scope = str(entry.get("scope", "namespace")).strip().lower()
1944 if scope == "cluster":
1945 access_policy = eks.AccessPolicy.from_access_policy_name(
1946 "AmazonEKSClusterAdminPolicy",
1947 access_scope_type=eks.AccessScopeType.CLUSTER,
1948 )
1949 elif scope == "namespace":
1950 namespaces = entry.get("namespaces") or ["gco-jobs"]
1951 if not isinstance(namespaces, list) or not all(
1952 isinstance(namespace, str) and namespace for namespace in namespaces
1953 ):
1954 raise ValueError(
1955 f"eks_cluster.developer_access[{index}].namespaces must be a "
1956 "list of namespace names"
1957 )
1958 access_policy = eks.AccessPolicy.from_access_policy_name(
1959 "AmazonEKSEditPolicy",
1960 access_scope_type=eks.AccessScopeType.NAMESPACE,
1961 namespaces=namespaces,
1962 )
1963 else:
1964 raise ValueError(
1965 f"eks_cluster.developer_access[{index}].scope must be 'namespace' "
1966 f"or 'cluster', got {entry.get('scope')!r}"
1967 )
1968 eks.AccessEntry(
1969 self,
1970 f"DeveloperAccessEntry{index}",
1971 cluster=self.cluster, # type: ignore[arg-type]
1972 principal=principal_arn,
1973 access_policies=[access_policy],
1974 )
1976 # ── EKS Capabilities (AWS-managed ACK / kro) ─────────────────────────
1977 #: cdk.json type name -> construct-id / CfnOutput suffix.
1978 _EKS_CAPABILITY_CONSTRUCT_SUFFIXES: ClassVar[dict[str, str]] = {
1979 "ack": "Ack",
1980 "kro": "Kro",
1981 }
1983 def _eks_capabilities_config(self) -> dict[str, Any]:
1984 """The normalized ``eks_capabilities`` block, or the all-off defaults.
1986 Many stack tests build the regional stack with hand-rolled config
1987 doubles that predate this knob (or with ``MagicMock``), so the accessor
1988 is optional: a missing method or a non-dict result reads as "every
1989 capability off", which is also the shipped cdk.json default.
1990 """
1991 getter = getattr(self.config, "get_eks_capabilities_config", None)
1992 raw = getter() if callable(getter) else None
1993 if not isinstance(raw, dict):
1994 return normalize_eks_capabilities_config(None)
1995 return raw
1997 def _argocd_config(self) -> dict[str, Any]:
1998 """The validated ``helm.argocd`` block, or the all-default block.
2000 Same optional-accessor posture as :meth:`_eks_capabilities_config`:
2001 config doubles without ``get_argocd_config`` (or a ``MagicMock``
2002 returning a non-dict) read as the shipped default (Argo CD off, no
2003 GitOps hand-off).
2004 """
2005 getter = getattr(self.config, "get_argocd_config", None)
2006 raw = getter() if callable(getter) else None
2007 if not isinstance(raw, dict):
2008 return validate_argocd_config(None)
2009 return raw
2011 def _slurm_config(self) -> dict[str, Any]:
2012 """The validated ``helm.slurm`` block, or the all-default block.
2014 Same optional-accessor posture as :meth:`_argocd_config`: config
2015 doubles without ``get_slurm_config`` (or a ``MagicMock`` returning a
2016 non-dict) read as the shipped default (Slurm off; when forced on by a
2017 run-scoped override, the shipped partitions).
2018 """
2019 getter = getattr(self.config, "get_slurm_config", None)
2020 raw = getter() if callable(getter) else None
2021 if not isinstance(raw, dict):
2022 return validate_slurm_config(None)
2023 return raw
2025 def _slurm_fsx_enabled(self) -> bool:
2026 """Whether this region's cluster has the FSx for Lustre claim Slurm can mount."""
2027 getter = getattr(self.config, "get_fsx_lustre_config", None)
2028 fsx = getter(self.deployment_region) if callable(getter) else None
2029 return bool(fsx.get("enabled")) if isinstance(fsx, dict) else False
2031 def _tracing_config(self) -> dict[str, Any]:
2032 """The validated ``tracing`` block (``enabled``, ``sample_ratio``, ...).
2034 Same optional-accessor posture as :meth:`_eks_capabilities_config`,
2035 except that a config double without ``get_tracing_config`` (or a
2036 ``MagicMock``) resolves this stack's own ``tracing`` /
2037 ``tracing_overrides`` context through the loader's resolver. An
2038 absent block yields the shipped defaults (tracing on), so stack tests
2039 exercise the default deployment's resources.
2040 """
2041 getter = getattr(self.config, "get_tracing_config", None)
2042 raw = getter() if callable(getter) else None
2043 if isinstance(raw, dict):
2044 return raw
2045 return resolve_tracing_config(
2046 self.node.try_get_context(TRACING_CONTEXT_KEY),
2047 self.node.try_get_context(TRACING_OVERRIDES_CONTEXT_KEY),
2048 )
2050 def _transaction_search_active(self) -> bool:
2051 """Whether this region's stack manages CloudWatch Transaction Search."""
2052 tracing = self._tracing_config()
2053 return bool(tracing["enabled"]) and bool(tracing["enable_transaction_search"])
2055 def _create_eks_capabilities(self) -> None:
2056 """Attach the enabled EKS Capabilities to this region's cluster.
2058 For every type enabled for this region (``eks_capabilities.<type>``
2059 with ``enabled: true`` and either an empty ``regions`` list or one
2060 naming this region) synthesize:
2062 * one capability IAM role trusted by ``capabilities.eks.amazonaws.com``
2063 (``sts:AssumeRole`` + ``sts:TagSession``, the documented trust
2064 policy). The role carries only what the operator configured: ACK
2065 may assume exactly the ``assume_role_arns`` (ACK IAM Role Selectors)
2066 and holds exactly the ``iam_policy_arns`` managed policies; kro gets
2067 no AWS permissions at all (it only needs Kubernetes RBAC, which
2068 ``07-kro-tenant-access.yaml`` grants in the tenant namespaces);
2069 * one ``AWS::EKS::Capability`` named ``<project>-<type>`` (a
2070 deterministic name the CLI and the live checks describe directly),
2071 with ``RETAIN`` delete propagation (the only supported value) so
2072 workloads the tools created survive a capability removal;
2073 * ``CfnOutput``s for the capability and role ARNs.
2074 """
2075 self.eks_capabilities: dict[str, eks_l1.CfnCapability] = {}
2076 self.eks_capability_roles: dict[str, iam.Role] = {}
2077 self.eks_capabilities_config = self._eks_capabilities_config()
2078 enabled_types = enabled_capability_types(
2079 self.eks_capabilities_config, self.deployment_region
2080 )
2081 if not enabled_types:
2082 return
2084 project_name = self.config.get_project_name()
2085 for type_name in enabled_types:
2086 block = self.eks_capabilities_config[type_name]
2087 suffix = self._EKS_CAPABILITY_CONSTRUCT_SUFFIXES[type_name]
2088 role = self._create_eks_capability_role(type_name, block)
2089 capability = eks_l1.CfnCapability(
2090 self,
2091 f"EksCapability{suffix}",
2092 capability_name=f"{project_name}-{type_name}",
2093 cluster_name=self.cluster.cluster_name,
2094 type=CAPABILITY_TYPE_API_NAMES[type_name],
2095 role_arn=role.role_arn,
2096 # RETAIN is the only value the API accepts today; spelled out so
2097 # a future PURGE option is an explicit decision, not a default.
2098 delete_propagation_policy="RETAIN",
2099 configuration=self._eks_capability_configuration(type_name, block),
2100 )
2101 # The cluster_name token already orders creation after the cluster;
2102 # the explicit edge keeps deletion ordered too (capability before
2103 # cluster) when CloudFormation reverses the graph.
2104 capability.node.add_dependency(self.cluster)
2105 capability.node.add_dependency(role)
2106 self.eks_capabilities[type_name] = capability
2107 self.eks_capability_roles[type_name] = role
2109 CfnOutput(
2110 self,
2111 f"EksCapability{suffix}Arn",
2112 value=capability.attr_arn,
2113 description=f"ARN of the AWS-managed {type_name} EKS Capability",
2114 )
2115 CfnOutput(
2116 self,
2117 f"EksCapability{suffix}RoleArn",
2118 value=role.role_arn,
2119 description=f"IAM role the {type_name} EKS Capability runs as",
2120 )
2122 def _create_eks_capability_role(self, type_name: str, block: Mapping[str, Any]) -> iam.Role:
2123 """The IAM role one capability assumes, carrying only configured grants."""
2124 suffix = self._EKS_CAPABILITY_CONSTRUCT_SUFFIXES[type_name]
2125 statements: list[iam.PolicyStatement] = []
2126 wildcard_arns: list[str] = []
2127 managed_policies: list[iam.IManagedPolicy] = []
2128 aws_managed_policy_names: list[str] = []
2129 if type_name == "ack":
2130 target_role_arns = [str(arn) for arn in block.get("assume_role_arns") or []]
2131 if target_role_arns:
2132 statements.append(
2133 iam.PolicyStatement(
2134 sid="AssumeAckControllerRoles",
2135 effect=iam.Effect.ALLOW,
2136 actions=["sts:AssumeRole"],
2137 resources=target_role_arns,
2138 )
2139 )
2140 wildcard_arns.extend(arn for arn in target_role_arns if "*" in arn)
2141 # The documented "simple permission setup": service permissions on
2142 # the capability role itself. Only the managed policies the
2143 # operator names are attached. AWS managed policies are rebuilt
2144 # from their name so the ARN follows this stack's partition.
2145 for index, raw_arn in enumerate(block.get("iam_policy_arns") or []):
2146 arn = str(raw_arn)
2147 if ":iam::aws:policy/" in arn:
2148 name = arn.split(":policy/", 1)[1]
2149 aws_managed_policy_names.append(name)
2150 managed_policies.append(iam.ManagedPolicy.from_aws_managed_policy_name(name))
2151 else:
2152 managed_policies.append(
2153 iam.ManagedPolicy.from_managed_policy_arn(
2154 self, f"EksCapability{suffix}Policy{index}", arn
2155 )
2156 )
2158 # role_name intentionally omitted - let CDK generate a unique name.
2159 # inline_policies keeps the grants inside the AWS::IAM::Role resource
2160 # itself, so the capability never observes a role whose policy has
2161 # not been attached yet; managed_policies attach the same way.
2162 role = iam.Role(
2163 self,
2164 f"EksCapability{suffix}Role",
2165 assumed_by=iam.ServicePrincipal("capabilities.eks.amazonaws.com").with_session_tags(),
2166 description=f"Capability role for the AWS-managed {type_name} EKS Capability",
2167 inline_policies=(
2168 {f"EksCapability{suffix}Grants": iam.PolicyDocument(statements=statements)}
2169 if statements
2170 else None
2171 ),
2172 managed_policies=managed_policies or None,
2173 )
2174 if wildcard_arns:
2175 # An operator may deliberately configure an ARN pattern (one role
2176 # per account under a naming scheme). Acknowledge exactly those
2177 # patterns so cdk-nag still flags any other wildcard on this role.
2178 from gco.stacks.nag_suppressions import acknowledge_nag_findings
2180 acknowledge_nag_findings(
2181 role,
2182 [
2183 {
2184 "id": "AwsSolutions-IAM5",
2185 "reason": (
2186 f"eks_capabilities.{type_name} names these resource ARN "
2187 "patterns explicitly in cdk.json; the capability role is "
2188 "granted nothing else."
2189 ),
2190 "appliesTo": [f"Resource::{arn}" for arn in sorted(set(wildcard_arns))],
2191 }
2192 ],
2193 )
2194 if aws_managed_policy_names:
2195 from gco.stacks.nag_suppressions import suppress_managed_policy_opt_in
2197 for name in sorted(set(aws_managed_policy_names)):
2198 # ACK manages AWS resources on the operator's behalf; which AWS
2199 # managed policies it holds is the operator's explicit choice,
2200 # acknowledged policy by policy so any other managed policy on
2201 # this role still surfaces.
2202 suppress_managed_policy_opt_in(
2203 role,
2204 managed_policy_name=name,
2205 reason=(
2206 f"eks_capabilities.{type_name}.iam_policy_arns names this AWS managed "
2207 "policy explicitly in cdk.json (off by default: the capability role "
2208 "holds no service permissions unless the operator lists them)."
2209 ),
2210 )
2211 return role
2213 def _eks_capability_configuration(
2214 self, type_name: str, block: Mapping[str, Any]
2215 ) -> eks_l1.CfnCapability.CapabilityConfigurationProperty | None:
2216 """Type-specific ``Configuration``; kro takes none."""
2217 if type_name == "ack":
2218 disabled_services = [str(service) for service in block.get("disabled_services") or []]
2219 return eks_l1.CfnCapability.CapabilityConfigurationProperty(
2220 ack=eks_l1.CfnCapability.AckProperty(
2221 disabled_services=disabled_services or None,
2222 enable_cross_namespace=bool(block.get("enable_cross_namespace", False)),
2223 )
2224 )
2225 return None
2227 # ── Shared toleration config for EKS add-ons ──────────────────────────
2228 # All GCO nodepools apply taints (nvidia.com/gpu, aws.amazon.com/neuron,
2229 # vpc.amazonaws.com/efa) that prevent DaemonSet pods from scheduling.
2230 # Every add-on component that runs as a DaemonSet (storage drivers' node
2231 # agents, metrics/log agents, node exporters) must tolerate these taints
2232 # so infrastructure works on every node type.
2233 #
2234 # Deployment-shaped add-on components (metrics-server, the CSI
2235 # *controllers*) must NOT carry these tolerations: a toleration makes
2236 # EKS Auto Mode consider the tainted accelerator pools for them, and
2237 # during a deploy's pod surge it will happily launch GPU instances for
2238 # zero-GPU pods and then consolidation-flap them — live release
2239 # validation run sched241-350ffc7d caught exactly that (two g4dn.xlarge
2240 # NodeClaims requesting ``nvidia.com/gpu: "0"``, churned mid-install,
2241 # failing GPU DaemonSet convergence checks).
2242 _ADDON_NODE_TOLERATIONS: ClassVar[list[dict[str, str]]] = [
2243 {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"},
2244 {"key": "aws.amazon.com/neuron", "operator": "Exists", "effect": "NoSchedule"},
2245 {"key": "vpc.amazonaws.com/efa", "operator": "Exists", "effect": "NoSchedule"},
2246 ]
2248 def _create_pod_identity_agent_addon(self) -> None:
2249 """Create EKS Pod Identity Agent add-on.
2251 On Auto Mode the DaemonSet schedules 0 pods (the agent is built into
2252 the node runtime), but the add-on registration is still required for
2253 the control-plane admission webhook that injects Pod Identity tokens.
2254 """
2255 eks.Addon(
2256 self,
2257 "PodIdentityAgentAddon",
2258 cluster=self.cluster, # type: ignore[arg-type]
2259 addon_name="eks-pod-identity-agent",
2260 addon_version=EKS_ADDON_POD_IDENTITY_AGENT,
2261 preserve_on_delete=False,
2262 configuration_values={
2263 "tolerations": self._ADDON_NODE_TOLERATIONS,
2264 },
2265 )
2267 def _create_metrics_server_addon(self) -> None:
2268 """Create Metrics Server add-on for resource metrics.
2270 The Metrics Server collects resource metrics from kubelets and exposes
2271 them via the Kubernetes API server. This is required for:
2272 - Horizontal Pod Autoscaler (HPA)
2273 - Vertical Pod Autoscaler (VPA)
2274 - kubectl top commands
2275 - Resource monitoring dashboards
2277 Note: Metrics Server doesn't require an IRSA role as it only needs
2278 in-cluster permissions which are handled by its service account.
2279 """
2280 # Deployment-shaped: no accelerator tolerations on purpose (see
2281 # _ADDON_NODE_TOLERATIONS) — metrics-server runs fine on the default
2282 # CPU pool and a toleration invites Auto Mode to launch GPU nodes
2283 # for it during deploy pod surges.
2284 eks.Addon(
2285 self,
2286 "MetricsServerAddon",
2287 cluster=self.cluster, # type: ignore[arg-type]
2288 addon_name="metrics-server",
2289 addon_version=EKS_ADDON_METRICS_SERVER,
2290 preserve_on_delete=False,
2291 )
2293 def _create_efs_csi_driver_addon(self) -> None:
2294 """Create EFS CSI Driver add-on for shared storage support.
2296 The EFS CSI driver enables Kubernetes pods to mount EFS file systems
2297 as persistent volumes. This is required for the shared storage feature.
2299 We create a Pod Identity role for the EFS CSI driver and update the add-on
2300 to use it via a custom resource after the add-on is created.
2301 """
2302 # Create IAM role for EFS CSI Driver using IRSA + Pod Identity
2303 self.efs_csi_role = GCORegionalStack._create_irsa_role(
2304 self,
2305 "EfsCsiDriverRole",
2306 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
2307 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
2308 service_account_names=["efs-csi-controller-sa"],
2309 namespaces=["kube-system"],
2310 )
2312 # Add EFS CSI driver permissions
2313 self.efs_csi_role.add_managed_policy(
2314 iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AmazonEFSCSIDriverPolicy")
2315 )
2317 # Create EFS CSI Driver add-on
2318 efs_addon = eks.Addon(
2319 self,
2320 "EfsCsiDriverAddon",
2321 cluster=self.cluster, # type: ignore[arg-type]
2322 addon_name="aws-efs-csi-driver",
2323 addon_version=EKS_ADDON_EFS_CSI_DRIVER,
2324 preserve_on_delete=False,
2325 configuration_values={
2326 # DaemonSet node agent must run on every node type; the
2327 # Deployment-shaped controller deliberately carries no
2328 # accelerator tolerations (see _ADDON_NODE_TOLERATIONS).
2329 "node": {
2330 "tolerations": self._ADDON_NODE_TOLERATIONS,
2331 },
2332 },
2333 )
2335 # Append the PassRole statement for the EFS CSI role to the shared
2336 # AwsCustomResource execution role. See the role's creation in
2337 # _create_aws_custom_resource_role for the full rationale on why
2338 # we pre-create + attach up-front instead of letting CDK
2339 # auto-generate per-CR roles.
2340 self.aws_custom_resource_role.add_to_policy(
2341 iam.PolicyStatement(
2342 effect=iam.Effect.ALLOW,
2343 actions=["iam:PassRole"],
2344 resources=[self.efs_csi_role.role_arn],
2345 )
2346 )
2348 # Update the add-on to use the IRSA role via custom resource
2349 # This is needed because the eks v2 alpha Addon doesn't support service_account_role directly
2350 update_addon = cr.AwsCustomResource(
2351 self,
2352 "UpdateEfsCsiAddonRole",
2353 on_create=cr.AwsSdkCall(
2354 service="EKS",
2355 action="updateAddon",
2356 parameters={
2357 "clusterName": self.cluster.cluster_name,
2358 "addonName": "aws-efs-csi-driver",
2359 "serviceAccountRoleArn": self.efs_csi_role.role_arn,
2360 },
2361 physical_resource_id=cr.PhysicalResourceId.of(
2362 f"{self.cluster.cluster_name}-efs-csi-role-update"
2363 ),
2364 ),
2365 on_update=cr.AwsSdkCall(
2366 service="EKS",
2367 action="updateAddon",
2368 parameters={
2369 "clusterName": self.cluster.cluster_name,
2370 "addonName": "aws-efs-csi-driver",
2371 "serviceAccountRoleArn": self.efs_csi_role.role_arn,
2372 },
2373 ),
2374 role=self.aws_custom_resource_role,
2375 )
2377 # Ensure the update happens after the add-on is created. We also
2378 # depend on the shared execution role so CloudFormation has fully
2379 # attached + replicated its inline policy before the Lambda fires.
2380 update_addon.node.add_dependency(efs_addon)
2381 update_addon.node.add_dependency(self.efs_csi_role)
2382 update_addon.node.add_dependency(self.aws_custom_resource_role)
2384 # Expose the update-addon resource so _apply_kubernetes_manifests can
2385 # make the kubectl Lambda wait for the IRSA annotation patch to land
2386 # before it tries to rollout-restart the efs-csi-controller. Without
2387 # this ordering, the restart could fire before EKS has re-attached
2388 # the role ARN, leaving the new pods just as credential-less as the
2389 # old ones and causing every EFS CreateAccessPoint to fail with a
2390 # 401 from IMDS.
2391 self._efs_csi_addon_role_update = update_addon
2393 def _create_cloudwatch_observability_addon(self) -> None:
2394 """Create CloudWatch Observability add-on for Container Insights.
2396 The CloudWatch Observability add-on enables Container Insights metrics
2397 for the EKS cluster, providing visibility into:
2398 - Cluster CPU and memory utilization
2399 - Node-level metrics
2400 - Pod and container metrics
2401 - Application logs (optional)
2403 These metrics are used by the monitoring dashboard to display
2404 cluster health and resource utilization.
2405 """
2407 # Create IAM role for CloudWatch agent using IRSA + Pod Identity
2408 self.cloudwatch_role = GCORegionalStack._create_irsa_role(
2409 self,
2410 "CloudWatchObservabilityRole",
2411 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
2412 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
2413 service_account_names=["cloudwatch-agent"],
2414 namespaces=["amazon-cloudwatch"],
2415 )
2417 # Add CloudWatch agent permissions
2418 self.cloudwatch_role.add_managed_policy(
2419 iam.ManagedPolicy.from_aws_managed_policy_name("CloudWatchAgentServerPolicy")
2420 )
2421 self.cloudwatch_role.add_managed_policy(
2422 iam.ManagedPolicy.from_aws_managed_policy_name("AWSXrayWriteOnlyAccess")
2423 )
2425 # Create CloudWatch Observability add-on
2426 cw_addon = eks.Addon(
2427 self,
2428 "CloudWatchObservabilityAddon",
2429 cluster=self.cluster, # type: ignore[arg-type]
2430 addon_name="amazon-cloudwatch-observability",
2431 addon_version=EKS_ADDON_CLOUDWATCH_OBSERVABILITY,
2432 preserve_on_delete=False,
2433 configuration_values={
2434 "tolerations": self._ADDON_NODE_TOLERATIONS,
2435 # Enable Container Insights with application log collection
2436 # Logs are sent to /aws/containerinsights/{cluster}/application
2437 "containerLogs": {
2438 "enabled": True,
2439 },
2440 # Application Signals auto-monitor injects ADOT
2441 # auto-instrumentation into matching workloads. The GCO API
2442 # services already export their own OpenTelemetry spans and
2443 # model pods must run exactly as rendered, so both GCO
2444 # namespaces are excluded for every supported language
2445 # (schema path verified against this add-on version).
2446 "manager": {
2447 "applicationSignals": {
2448 "autoMonitor": {
2449 "exclude": {
2450 language: {
2451 "namespaces": list(_APPLICATION_SIGNALS_EXCLUDED_NAMESPACES)
2452 }
2453 for language in _APPLICATION_SIGNALS_LANGUAGES
2454 },
2455 },
2456 },
2457 },
2458 },
2459 )
2461 # Append the PassRole statement for the CloudWatch Observability
2462 # role to the shared AwsCustomResource execution role. See
2463 # _create_aws_custom_resource_role for the full rationale.
2464 self.aws_custom_resource_role.add_to_policy(
2465 iam.PolicyStatement(
2466 effect=iam.Effect.ALLOW,
2467 actions=["iam:PassRole"],
2468 resources=[self.cloudwatch_role.role_arn],
2469 )
2470 )
2472 # Update the add-on to use the IRSA role via custom resource
2473 update_cw_addon = cr.AwsCustomResource(
2474 self,
2475 "UpdateCloudWatchAddonRole",
2476 on_create=cr.AwsSdkCall(
2477 service="EKS",
2478 action="updateAddon",
2479 parameters={
2480 "clusterName": self.cluster.cluster_name,
2481 "addonName": "amazon-cloudwatch-observability",
2482 "serviceAccountRoleArn": self.cloudwatch_role.role_arn,
2483 },
2484 physical_resource_id=cr.PhysicalResourceId.of(
2485 f"{self.cluster.cluster_name}-cw-obs-role-update"
2486 ),
2487 ),
2488 on_update=cr.AwsSdkCall(
2489 service="EKS",
2490 action="updateAddon",
2491 parameters={
2492 "clusterName": self.cluster.cluster_name,
2493 "addonName": "amazon-cloudwatch-observability",
2494 "serviceAccountRoleArn": self.cloudwatch_role.role_arn,
2495 },
2496 ),
2497 role=self.aws_custom_resource_role,
2498 )
2500 # Ensure the update happens after the add-on is created. Depend on
2501 # the shared execution role so CFN has fully attached + replicated
2502 # its inline policy before the Lambda fires. No CR→CR dependency
2503 # chain needed anymore — the race it was serializing against is
2504 # eliminated by pre-creating the role.
2505 update_cw_addon.node.add_dependency(cw_addon)
2506 update_cw_addon.node.add_dependency(self.cloudwatch_role)
2507 update_cw_addon.node.add_dependency(self.aws_custom_resource_role)
2509 # Expose the update-addon resource so _apply_kubernetes_manifests can
2510 # make the kubectl Lambda wait for the IRSA annotation patch to land
2511 # before it rollout-restarts the cloudwatch-agent DaemonSet. See the
2512 # EFS CSI equivalent for the full rationale — same race, same fix.
2513 self._cloudwatch_addon_role_update = update_cw_addon
2515 def _create_service_account_role(self) -> None:
2516 """Create IAM role for Kubernetes service account using EKS Pod Identity.
2518 Pod Identity is the recommended mechanism for EKS Auto Mode. It's simpler
2519 and more reliable than IRSA — no OIDC provider, no webhook injection, no
2520 projected tokens. EKS manages the credential injection automatically.
2522 The general workload role is deliberately separate from the manifest
2523 processor role. Job and inference workload service accounts must never
2524 receive queue-table mutation privileges; only the platform API/worker
2525 identity can claim, fence, or transition centralized queue records.
2526 """
2527 self.service_account_role = GCORegionalStack._create_irsa_role(
2528 self,
2529 "ServiceAccountRole",
2530 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
2531 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
2532 service_account_names=[
2533 "gco-service-account",
2534 "gco-inference-monitor-sa",
2535 ],
2536 namespaces=["gco-system", "gco-jobs", "gco-inference"],
2537 )
2539 self.manifest_processor_role = GCORegionalStack._create_irsa_role(
2540 self,
2541 "ManifestProcessorRole",
2542 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
2543 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
2544 service_account_names=["gco-manifest-processor-sa"],
2545 namespaces=["gco-system"],
2546 )
2548 self.inference_proxy_role = GCORegionalStack._create_irsa_role(
2549 self,
2550 "InferenceProxyRole",
2551 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
2552 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
2553 service_account_names=["gco-inference-proxy-sa"],
2554 namespaces=["gco-system"],
2555 )
2557 self.health_monitor_role = GCORegionalStack._create_irsa_role(
2558 self,
2559 "HealthMonitorRole",
2560 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
2561 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
2562 service_account_names=["gco-health-monitor-sa"],
2563 namespaces=["gco-system"],
2564 )
2566 if self._cost_monitoring_active():
2567 self.cost_monitor_role = GCORegionalStack._create_irsa_role(
2568 self,
2569 "CostMonitorRole",
2570 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
2571 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
2572 service_account_names=["gco-cost-monitor-sa"],
2573 namespaces=["gco-system"],
2574 )
2575 self._grant_cost_report_bucket_discovery_to_cost_monitor()
2577 self._create_aws_load_balancer_controller_role()
2579 # OpenTelemetry span export (cdk.json ``tracing``): each traced API
2580 # service signs its own OTLP requests with its own role. The returned
2581 # note extends each role's existing IAM5 reason below.
2582 tracing_nag_note = self._grant_tracing_export_to_service_roles()
2584 # Grant permission to read the auth secret.
2585 #
2586 # The resource is built from the secret's name, the API Gateway Region
2587 # (where the API Gateway stack creates it) and this stack's account,
2588 # never from ``self.auth_secret_arn``. That is a cross-stack reference
2589 # (``api_gateway_stack.secret.secret_arn``) and renders differently by
2590 # topology: in another Region, an ARN built from the name without the
2591 # random suffix; in the same Region, a native export
2592 # (``<project>-api-gateway:ExportsOutputRefGCOAuthSecret<hash>``).
2593 # Issue #125 was a grant built on the export.
2594 #
2595 # ``-??????`` matches the six random characters Secrets Manager appends
2596 # to the name in the ARN, and nothing else. IAM evaluates the request
2597 # against that full ARN whether the caller names the secret by full ARN
2598 # or, as pods outside the API Gateway Region do, by the suffix-less one.
2599 # A trailing ``*`` would also have matched any other secret whose name
2600 # starts the same way, and cost an AwsSolutions-IAM5 acknowledgment.
2601 auth_secret_resource = (
2602 f"arn:{self.partition}:secretsmanager:{self.config.get_api_gateway_region()}"
2603 f":{self.account}:secret:"
2604 f"{api_gateway_auth_secret_name(self.config.get_project_name())}-??????"
2605 )
2606 self.manifest_processor_role.add_to_policy(
2607 iam.PolicyStatement(
2608 effect=iam.Effect.ALLOW,
2609 actions=[
2610 "secretsmanager:GetSecretValue",
2611 "secretsmanager:DescribeSecret",
2612 ],
2613 resources=[auth_secret_resource],
2614 )
2615 )
2616 self.inference_proxy_role.add_to_policy(
2617 iam.PolicyStatement(
2618 effect=iam.Effect.ALLOW,
2619 actions=[
2620 "secretsmanager:GetSecretValue",
2621 "secretsmanager:DescribeSecret",
2622 ],
2623 resources=[auth_secret_resource],
2624 )
2625 )
2626 self.health_monitor_role.add_to_policy(
2627 iam.PolicyStatement(
2628 effect=iam.Effect.ALLOW,
2629 actions=[
2630 "secretsmanager:GetSecretValue",
2631 "secretsmanager:DescribeSecret",
2632 ],
2633 resources=[auth_secret_resource],
2634 )
2635 )
2637 from gco.stacks.nag_suppressions import acknowledge_nag_findings
2639 # The SQS queue processor runs as gco-manifest-processor-sa. Keep
2640 # queue consumption on that dedicated platform identity; KEDA has its
2641 # own read-only queue role and general workload identities must not be
2642 # able to receive or delete submitted jobs.
2643 self.manifest_processor_role.add_to_policy(
2644 iam.PolicyStatement(
2645 effect=iam.Effect.ALLOW,
2646 actions=[
2647 "sqs:ReceiveMessage",
2648 "sqs:DeleteMessage",
2649 ],
2650 resources=[self.job_queue.queue_arn],
2651 )
2652 )
2654 # Manifest API/worker metrics are emitted only by the dedicated
2655 # platform identity, never by user workload service accounts.
2656 self.manifest_processor_role.add_to_policy(
2657 iam.PolicyStatement(
2658 effect=iam.Effect.ALLOW,
2659 actions=["cloudwatch:PutMetricData"],
2660 resources=["*"],
2661 conditions={"StringEquals": {"cloudwatch:namespace": "GCO/ManifestProcessor"}},
2662 )
2663 )
2665 # The central queue worker's spot price gate reads current spot
2666 # pricing before dispatching price-capped jobs.
2667 # ec2:DescribeSpotPriceHistory is a read-only Describe* action that
2668 # does not support resource-level scoping (Resource must be *).
2669 self.manifest_processor_role.add_to_policy(
2670 iam.PolicyStatement(
2671 effect=iam.Effect.ALLOW,
2672 actions=["ec2:DescribeSpotPriceHistory"],
2673 resources=["*"],
2674 )
2675 )
2677 # Add DynamoDB permissions for templates, webhooks, and job queue
2678 # Tables are created in the global stack and accessed from all regions
2679 project_name = self.config.get_project_name()
2680 global_region = self.config.get_global_region()
2682 # Health-monitor runtime grants are isolated from the shared workload
2683 # role. It can read/repair one endpoint-registry parameter, read webhook
2684 # subscriptions, publish only its metric namespace, and read the auth
2685 # secret granted above.
2686 self.health_monitor_role.add_to_policy(
2687 iam.PolicyStatement(
2688 effect=iam.Effect.ALLOW,
2689 actions=["ssm:GetParameter", "ssm:PutParameter"],
2690 resources=[
2691 f"arn:{self.partition}:ssm:{global_region}:{self.account}:"
2692 f"parameter/{project_name}/alb-hostname-{self.deployment_region}"
2693 ],
2694 )
2695 )
2696 self.health_monitor_role.add_to_policy(
2697 iam.PolicyStatement(
2698 effect=iam.Effect.ALLOW,
2699 actions=["dynamodb:Query", "dynamodb:Scan"],
2700 resources=[
2701 f"arn:{self.partition}:dynamodb:{global_region}:{self.account}:"
2702 f"table/{project_name}-webhooks",
2703 f"arn:{self.partition}:dynamodb:{global_region}:{self.account}:"
2704 f"table/{project_name}-webhooks/index/namespace-index",
2705 ],
2706 )
2707 )
2708 self.health_monitor_role.add_to_policy(
2709 iam.PolicyStatement(
2710 effect=iam.Effect.ALLOW,
2711 actions=["cloudwatch:PutMetricData"],
2712 resources=["*"],
2713 conditions={"StringEquals": {"cloudwatch:namespace": "GCO/HealthMonitor"}},
2714 )
2715 )
2716 acknowledge_nag_findings(
2717 self.health_monitor_role,
2718 [
2719 {
2720 "id": "AwsSolutions-IAM5",
2721 "reason": (
2722 "HealthMonitorRole has two unavoidable wildcard shapes: the "
2723 "Secrets Manager random ARN suffix and cloudwatch:PutMetricData's "
2724 "required Resource:*. PutMetricData is constrained to the exact "
2725 "GCO/HealthMonitor namespace; all SSM and DynamoDB resources are exact."
2726 + tracing_nag_note
2727 ),
2728 "appliesTo": ["Resource::*"],
2729 }
2730 ],
2731 )
2733 # The manifest processor is the only identity that can mutate the
2734 # centralized queue. Workload identities receive no access to the jobs
2735 # table, preventing submitted pods from forging queue state.
2736 manifest_table_prefix = (
2737 f"arn:{self.partition}:dynamodb:{global_region}:{self.account}:table/{project_name}"
2738 )
2739 self.manifest_processor_role.add_to_policy(
2740 iam.PolicyStatement(
2741 effect=iam.Effect.ALLOW,
2742 actions=[
2743 "dynamodb:GetItem",
2744 "dynamodb:PutItem",
2745 "dynamodb:UpdateItem",
2746 "dynamodb:DeleteItem",
2747 "dynamodb:Query",
2748 "dynamodb:Scan",
2749 ],
2750 resources=[
2751 f"{manifest_table_prefix}-job-templates",
2752 f"{manifest_table_prefix}-job-templates/index/*",
2753 f"{manifest_table_prefix}-webhooks",
2754 f"{manifest_table_prefix}-webhooks/index/*",
2755 ],
2756 )
2757 )
2758 self.manifest_processor_role.add_to_policy(
2759 iam.PolicyStatement(
2760 effect=iam.Effect.ALLOW,
2761 actions=[
2762 "dynamodb:GetItem",
2763 "dynamodb:PutItem",
2764 "dynamodb:UpdateItem",
2765 "dynamodb:Query",
2766 "dynamodb:Scan",
2767 ],
2768 resources=[
2769 f"{manifest_table_prefix}-jobs",
2770 f"{manifest_table_prefix}-jobs/index/*",
2771 ],
2772 )
2773 )
2775 # The inference proxy needs only point reads of endpoint state. It has
2776 # no write, scan, index, S3, Kubernetes, or queue permissions.
2777 self.inference_proxy_role.add_to_policy(
2778 iam.PolicyStatement(
2779 effect=iam.Effect.ALLOW,
2780 actions=["dynamodb:GetItem"],
2781 resources=[f"{manifest_table_prefix}-inference-endpoints"],
2782 )
2783 )
2784 acknowledge_nag_findings(
2785 self.inference_proxy_role,
2786 [
2787 {
2788 "id": "AwsSolutions-IAM5",
2789 "reason": (
2790 "InferenceProxyRole uses one wildcard only for the random "
2791 "Secrets Manager ARN suffix. DynamoDB access is an exact-table "
2792 "GetItem grant, and the role has no Kubernetes, queue, or write access."
2793 + tracing_nag_note
2794 ),
2795 "appliesTo": ["Resource::*"],
2796 }
2797 ],
2798 )
2800 # The inference monitor still owns desired-state reconciliation, but
2801 # this shared role intentionally has no jobs-table ARN.
2802 self.service_account_role.add_to_policy(
2803 iam.PolicyStatement(
2804 effect=iam.Effect.ALLOW,
2805 actions=[
2806 "dynamodb:GetItem",
2807 "dynamodb:PutItem",
2808 "dynamodb:UpdateItem",
2809 "dynamodb:DeleteItem",
2810 "dynamodb:Query",
2811 "dynamodb:Scan",
2812 ],
2813 resources=[
2814 f"{manifest_table_prefix}-inference-endpoints",
2815 f"{manifest_table_prefix}-inference-endpoints/index/*",
2816 ],
2817 )
2818 )
2819 acknowledge_nag_findings(
2820 self.manifest_processor_role,
2821 [
2822 {
2823 "id": "AwsSolutions-IAM5",
2824 "reason": (
2825 "ManifestProcessorRole has only four required wildcard shapes: "
2826 "DynamoDB secondary indexes, the Secrets Manager generated ARN "
2827 "suffix, cloudwatch:PutMetricData Resource:*, and the read-only "
2828 "ec2:DescribeSpotPriceHistory Resource:* (Describe* actions do "
2829 "not support resource-level scoping; the central queue worker's "
2830 "spot price gate needs current pricing). DynamoDB table names "
2831 "and the CloudWatch namespace are otherwise exact." + tracing_nag_note
2832 ),
2833 "appliesTo": ["Resource::*"],
2834 }
2835 ],
2836 )
2838 # Workload pods must run as gco-service-account to write artifacts
2839 # (it is the only identity with RW on the regional shared bucket plus
2840 # KMS encrypt), and without this statement that same role could not
2841 # publish training metrics — CloudWatch denied PutMetricData with a
2842 # warning most trainers swallow. Grant exactly one namespace,
2843 # configurable via cdk.json::workload_metrics.cloudwatch_namespace so
2844 # a deployment can point its own consumers at it; the default is a
2845 # GCO-owned workload namespace. PutMetricData does not support
2846 # resource-level scoping (Resource must be *), so the namespace
2847 # condition carries the whole restriction — the same shape as every
2848 # platform role's metrics grant.
2849 workload_metrics_config = self.node.try_get_context("workload_metrics") or {}
2850 workload_metrics_namespace = str(
2851 workload_metrics_config.get("cloudwatch_namespace") or "GCO/Workloads"
2852 )
2853 self.service_account_role.add_to_policy(
2854 iam.PolicyStatement(
2855 effect=iam.Effect.ALLOW,
2856 actions=["cloudwatch:PutMetricData"],
2857 resources=["*"],
2858 conditions={"StringEquals": {"cloudwatch:namespace": workload_metrics_namespace}},
2859 )
2860 )
2861 acknowledge_nag_findings(
2862 self.service_account_role,
2863 [
2864 {
2865 "id": "AwsSolutions-IAM5",
2866 "reason": (
2867 "cloudwatch:PutMetricData does not support resource-level "
2868 "scoping and requires Resource:*. The statement is "
2869 "constrained by a StringEquals condition to exactly one "
2870 "configured workload metric namespace, matching the "
2871 "namespace-conditioned metrics grants on the platform roles."
2872 ),
2873 "appliesTo": ["Resource::*"],
2874 }
2875 ],
2876 )
2878 # Vector-store read path for workloads (opt-in feature). Deliberately
2879 # LOCAL-region ARNs: the store is a DynamoDB global table with a
2880 # replica in this cluster's region, so pods query locally instead of
2881 # crossing regions to the primary. Every resource is exact — the
2882 # table and index names are deterministic from config — and the
2883 # grant is read-only: writes belong to the ingest Lambda in the
2884 # global stack, so a compromised workload cannot poison the corpus.
2885 # The embedding-model grant lets pods embed their own query text
2886 # with the exact model the corpus was ingested with.
2887 if self.config.get_vector_store_enabled():
2888 vector_store_table_prefix = (
2889 f"arn:{self.partition}:dynamodb:{self.region}:{self.account}:"
2890 f"table/{project_name}-vector-store"
2891 )
2892 self.service_account_role.add_to_policy(
2893 iam.PolicyStatement(
2894 effect=iam.Effect.ALLOW,
2895 actions=[
2896 "dynamodb:SearchVectors",
2897 "dynamodb:GetItem",
2898 "dynamodb:Query",
2899 ],
2900 resources=[
2901 vector_store_table_prefix,
2902 f"{vector_store_table_prefix}/index/corpus-embedding-index",
2903 ],
2904 )
2905 )
2906 self.service_account_role.add_to_policy(
2907 iam.PolicyStatement(
2908 effect=iam.Effect.ALLOW,
2909 actions=["bedrock:InvokeModel"],
2910 resources=[
2911 f"arn:{self.partition}:bedrock:{self.region}::foundation-model/"
2912 f"{self.config.get_vector_store_config()['embedding_model_id']}"
2913 ],
2914 )
2915 )
2917 # Add S3 permissions for model weights bucket (used by inference init containers)
2918 self.service_account_role.add_to_policy(
2919 iam.PolicyStatement(
2920 effect=iam.Effect.ALLOW,
2921 actions=[
2922 "s3:GetObject",
2923 "s3:ListBucket",
2924 ],
2925 resources=[
2926 f"arn:{self.partition}:s3:::{project_name}-*",
2927 f"arn:{self.partition}:s3:::{project_name}-*/*",
2928 ],
2929 )
2930 )
2932 # KMS decrypt for model weights bucket (S3-scoped)
2933 self.service_account_role.add_to_policy(
2934 iam.PolicyStatement(
2935 effect=iam.Effect.ALLOW,
2936 actions=["kms:Decrypt", "kms:GenerateDataKey"],
2937 resources=[f"arn:{self.partition}:kms:*:{self.account}:key/*"],
2938 conditions={
2939 "StringLike": {
2940 "kms:ViaService": f"s3.*.{self.url_suffix}",
2941 }
2942 },
2943 )
2944 )
2946 # Create KEDA operator IAM role for SQS access
2947 self._create_keda_operator_role()
2949 # Create Pod Identity Associations for all service accounts
2950 self._create_pod_identity_associations()
2952 def _grant_tracing_export_to_service_roles(self) -> str:
2953 """Let the four traced API services export spans to X-Ray.
2955 Adds one write-only ``xray:PutTraceSegments`` / ``xray:PutSpans``
2956 statement to the health-monitor, manifest-processor, inference-proxy
2957 and (when cost monitoring deploys) cost-monitor roles — the roles
2958 their Deployments assume through the ``*_ROLE_ARN`` placeholders.
2959 Nothing is granted while ``tracing.enabled`` is false.
2961 Returns the sentence the callers append to the roles' existing
2962 AwsSolutions-IAM5 ``Resource::*`` reasons (empty when tracing is
2963 off). The cost-monitor role has no other wildcard, so it gets its own
2964 acknowledgment here.
2965 """
2966 if not self._tracing_config()["enabled"]:
2967 return ""
2969 roles = [self.health_monitor_role, self.manifest_processor_role, self.inference_proxy_role]
2970 if self._cost_monitoring_active():
2971 roles.append(self.cost_monitor_role)
2972 for role in roles:
2973 role.add_to_policy(
2974 iam.PolicyStatement(
2975 effect=iam.Effect.ALLOW,
2976 actions=list(_TRACING_EXPORT_ACTIONS),
2977 resources=["*"],
2978 )
2979 )
2981 if self._cost_monitoring_active():
2982 from gco.stacks.nag_suppressions import acknowledge_nag_findings
2984 acknowledge_nag_findings(
2985 self.cost_monitor_role,
2986 [
2987 {
2988 "id": "AwsSolutions-IAM5",
2989 "reason": (
2990 "CostMonitorRole's only wildcard is tracing. "
2991 + _TRACING_EXPORT_NAG_REASON
2992 ),
2993 "appliesTo": ["Resource::*"],
2994 }
2995 ],
2996 )
2997 return " " + _TRACING_EXPORT_NAG_REASON
2999 def _create_aws_load_balancer_controller_role(self) -> None:
3000 """Create the controller's exact OIDC-only IRSA role and v3.4.2 policy."""
3001 self.aws_load_balancer_controller_role = GCORegionalStack._create_irsa_role(
3002 self,
3003 "AwsLoadBalancerControllerRole",
3004 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
3005 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
3006 service_account_names=["aws-load-balancer-controller"],
3007 namespaces=["kube-system"],
3008 include_pod_identity=False,
3009 )
3010 self.aws_load_balancer_controller_policy = iam.Policy(
3011 self,
3012 "AwsLoadBalancerControllerPolicy",
3013 document=iam.PolicyDocument.from_json(
3014 aws_load_balancer_controller_policy_document(self.partition)
3015 ),
3016 )
3017 self.aws_load_balancer_controller_role.attach_inline_policy(
3018 self.aws_load_balancer_controller_policy
3019 )
3021 from gco.stacks.nag_suppressions import acknowledge_nag_findings
3023 acknowledge_nag_findings(
3024 self.aws_load_balancer_controller_policy,
3025 [
3026 {
3027 "id": "AwsSolutions-IAM5",
3028 "reason": (
3029 "This is the exact upstream AWS Load Balancer Controller v3.4.2 "
3030 "IAM policy. Its Resource::* entries cover AWS APIs that cannot "
3031 "be resource-scoped, while its wildcard ARN segments are limited "
3032 "to the controller's supported EC2 and ELB resource types and "
3033 "constrained by upstream cluster ownership tag conditions. The "
3034 "role trust is restricted to kube-system/aws-load-balancer-controller."
3035 ),
3036 "appliesTo": [
3037 "Resource::*",
3038 "Resource::arn:<AWS::Partition>:ec2:*:*:security-group/*",
3039 (
3040 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
3041 "loadbalancer/app/*/*"
3042 ),
3043 (
3044 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
3045 "loadbalancer/net/*/*"
3046 ),
3047 ("Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:targetgroup/*/*"),
3048 (
3049 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
3050 "listener-rule/app/*/*/*"
3051 ),
3052 (
3053 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
3054 "listener-rule/net/*/*/*"
3055 ),
3056 (
3057 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
3058 "listener/app/*/*/*"
3059 ),
3060 (
3061 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
3062 "listener/net/*/*/*"
3063 ),
3064 ],
3065 }
3066 ],
3067 )
3069 def _create_keda_operator_role(self) -> None:
3070 """Create IAM role for KEDA operator service account using EKS Pod Identity.
3072 This role allows the KEDA operator to access SQS queues for scaling
3073 based on queue depth. The role is assumed by the keda-operator service
3074 account in the keda namespace.
3075 """
3076 # Create IAM role with IRSA (OIDC) trust + Pod Identity trust
3077 self.keda_operator_role = GCORegionalStack._create_irsa_role(
3078 self,
3079 "KedaOperatorRole",
3080 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
3081 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
3082 service_account_names=["keda-operator"],
3083 namespaces=["keda"],
3084 )
3086 # Add SQS permissions for KEDA to read queue metrics
3087 self.keda_operator_role.add_to_policy(
3088 iam.PolicyStatement(
3089 effect=iam.Effect.ALLOW,
3090 actions=[
3091 "sqs:GetQueueAttributes",
3092 "sqs:GetQueueUrl",
3093 ],
3094 resources=[
3095 self.job_queue.queue_arn,
3096 self.job_dlq.queue_arn,
3097 ],
3098 )
3099 )
3101 # CloudWatch read permissions for GPU-based autoscaling. The KEDA
3102 # aws-cloudwatch scaler reads ContainerInsights GPU utilization metrics
3103 # to scale inference roles that request GPUs — GPU is not a native HPA
3104 # resource metric, so this is the only path that can drive GPU scaling.
3105 # The CloudWatch read APIs do not support resource-level IAM scoping, so
3106 # they are granted account-wide (read-only).
3107 self.keda_operator_role.add_to_policy(
3108 iam.PolicyStatement(
3109 effect=iam.Effect.ALLOW,
3110 actions=[
3111 "cloudwatch:GetMetricData",
3112 "cloudwatch:GetMetricStatistics",
3113 "cloudwatch:ListMetrics",
3114 ],
3115 resources=["*"],
3116 )
3117 )
3119 # cdk-nag suppression: the CloudWatch metric-read APIs do not support
3120 # resource-level IAM scoping — Resource: * is the only valid form.
3121 from gco.stacks.nag_suppressions import acknowledge_nag_findings
3123 acknowledge_nag_findings(
3124 self.keda_operator_role,
3125 [
3126 {
3127 "id": "AwsSolutions-IAM5",
3128 "reason": (
3129 "The KEDA operator reads CloudWatch metrics "
3130 "(GetMetricData, GetMetricStatistics, ListMetrics) to "
3131 "drive GPU-based autoscaling. These APIs do not support "
3132 "resource-level IAM scoping — Resource: * is the only "
3133 "valid form. The grant is read-only."
3134 ),
3135 "appliesTo": ["Resource::*"],
3136 },
3137 ],
3138 )
3140 def _create_pod_identity_associations(self) -> None:
3141 """Create EKS Pod Identity Associations for all service accounts.
3143 Pod Identity is the recommended mechanism for EKS Auto Mode. Each
3144 association links an IAM role to a Kubernetes service account in a
3145 specific namespace. EKS manages credential injection automatically.
3147 Stores associations in self._pod_identity_associations so the
3148 kubectl-applier custom resource can declare an explicit dependency,
3149 ensuring credentials are available before workloads start.
3150 """
3151 self._pod_identity_associations: list[Any] = []
3153 # Health monitor — isolated write access for ALB hostname self-healing.
3154 health_assoc = eks_l1.CfnPodIdentityAssociation(
3155 self,
3156 "PodIdentity-health-monitor",
3157 cluster_name=self.cluster.cluster_name,
3158 namespace="gco-system",
3159 service_account="gco-health-monitor-sa",
3160 role_arn=self.health_monitor_role.role_arn,
3161 )
3162 self._pod_identity_associations.append(health_assoc)
3164 # Manifest API and central queue worker — dedicated queue mutation role.
3165 manifest_assoc = eks_l1.CfnPodIdentityAssociation(
3166 self,
3167 "PodIdentity-manifest-processor",
3168 cluster_name=self.cluster.cluster_name,
3169 namespace="gco-system",
3170 service_account="gco-manifest-processor-sa",
3171 role_arn=self.manifest_processor_role.role_arn,
3172 )
3173 self._pod_identity_associations.append(manifest_assoc)
3175 # Inference data plane — exact secret + endpoint-table read role, with
3176 # no Kubernetes RBAC binding.
3177 inference_proxy_assoc = eks_l1.CfnPodIdentityAssociation(
3178 self,
3179 "PodIdentity-inference-proxy",
3180 cluster_name=self.cluster.cluster_name,
3181 namespace="gco-system",
3182 service_account="gco-inference-proxy-sa",
3183 role_arn=self.inference_proxy_role.role_arn,
3184 )
3185 self._pod_identity_associations.append(inference_proxy_assoc)
3187 # Inference monitor — reconciles endpoints with the shared platform
3188 # role (its IRSA subject is already in that role's trust policy).
3189 # Every other platform Deployment gets both credential paths; this one
3190 # had only the IRSA annotation.
3191 inference_monitor_assoc = eks_l1.CfnPodIdentityAssociation(
3192 self,
3193 "PodIdentity-inference-monitor",
3194 cluster_name=self.cluster.cluster_name,
3195 namespace="gco-system",
3196 service_account="gco-inference-monitor-sa",
3197 role_arn=self.service_account_role.role_arn,
3198 )
3199 self._pod_identity_associations.append(inference_monitor_assoc)
3201 # Cost monitor — only when the pipeline deploys here (the role and
3202 # 34-cost-monitor.yaml are gated the same way).
3203 if self._cost_monitoring_active():
3204 cost_monitor_assoc = eks_l1.CfnPodIdentityAssociation(
3205 self,
3206 "PodIdentity-cost-monitor",
3207 cluster_name=self.cluster.cluster_name,
3208 namespace="gco-system",
3209 service_account="gco-cost-monitor-sa",
3210 role_arn=self.cost_monitor_role.role_arn,
3211 )
3212 self._pod_identity_associations.append(cost_monitor_assoc)
3214 # Shared GCO service account for user job and inference workloads —
3215 # the two namespaces 01-serviceaccounts.yaml actually declares it in.
3216 # (A gco-system association used to be created as well; no such
3217 # ServiceAccount exists there, so it never bound anything.)
3218 for namespace in ["gco-jobs", "gco-inference"]:
3219 assoc = eks_l1.CfnPodIdentityAssociation(
3220 self,
3221 f"PodIdentity-gco-sa-{namespace}",
3222 cluster_name=self.cluster.cluster_name,
3223 namespace=namespace,
3224 service_account="gco-service-account",
3225 role_arn=self.service_account_role.role_arn,
3226 )
3227 self._pod_identity_associations.append(assoc)
3229 # KEDA operator — needs SQS access for queue-based scaling
3230 keda_assoc = eks_l1.CfnPodIdentityAssociation(
3231 self,
3232 "PodIdentity-keda-operator",
3233 cluster_name=self.cluster.cluster_name,
3234 namespace="keda",
3235 service_account="keda-operator",
3236 role_arn=self.keda_operator_role.role_arn,
3237 )
3238 self._pod_identity_associations.append(keda_assoc)
3240 # EFS CSI driver — needs EFS access for shared storage
3241 efs_assoc = eks_l1.CfnPodIdentityAssociation(
3242 self,
3243 "PodIdentity-efs-csi",
3244 cluster_name=self.cluster.cluster_name,
3245 namespace="kube-system",
3246 service_account="efs-csi-controller-sa",
3247 role_arn=self.efs_csi_role.role_arn,
3248 )
3249 self._pod_identity_associations.append(efs_assoc)
3251 # CloudWatch agent — needs CloudWatch access for observability
3252 cw_assoc = eks_l1.CfnPodIdentityAssociation(
3253 self,
3254 "PodIdentity-cloudwatch",
3255 cluster_name=self.cluster.cluster_name,
3256 namespace="amazon-cloudwatch",
3257 service_account="cloudwatch-agent",
3258 role_arn=self.cloudwatch_role.role_arn,
3259 )
3260 self._pod_identity_associations.append(cw_assoc)
3262 # FSx CSI driver — only when FSx is enabled (created later in _create_fsx_lustre)
3263 # The FSx Pod Identity association is added in _create_fsx_lustre instead
3265 def _resolve_cluster_shared_bucket_from_ssm(self) -> SharedBucketIdentity:
3266 """Resolve the ``Cluster_Shared_Bucket`` identity from cross-region SSM.
3268 ``GCOGlobalStack`` publishes three ``ssm.StringParameter``s in the
3269 global region at ``/gco/cluster-shared-bucket/{name,arn,region}``.
3270 This method reads them back from the regional stack via
3271 ``cr.AwsCustomResource`` with ``service="SSM"``,
3272 ``action="getParameter"``, and ``region=<global-region>`` — matching
3273 the cross-region read pattern already used in
3274 ``_create_ga_registration_lambda`` for the Global Accelerator
3275 endpoint group ARN.
3277 Runs unconditionally in ``__init__`` — no feature toggle, no
3278 conditional guard. The returned :class:`SharedBucketIdentity` feeds
3279 ``_grant_cluster_shared_bucket_to_job_role`` (IAM) and the
3280 ``image_replacements`` dict (ConfigMap) downstream.
3282 Returns:
3283 :class:`SharedBucketIdentity` with ``name``, ``arn``, and
3284 ``region`` populated as CDK tokens that resolve at deploy time.
3285 """
3286 from gco.stacks.nag_suppressions import acknowledge_nag_findings
3288 global_region = self.config.get_global_region()
3289 cluster_shared_prefix = cluster_shared_ssm_parameter_prefix(self.config.get_project_name())
3290 resolved: dict[str, str] = {}
3292 for suffix in ("name", "arn", "region"):
3293 parameter_name = f"{cluster_shared_prefix}/{suffix}"
3294 read_cr = cr.AwsCustomResource(
3295 self,
3296 f"ReadClusterSharedBucket{suffix.capitalize()}",
3297 on_create=cr.AwsSdkCall(
3298 service="SSM",
3299 action="getParameter",
3300 parameters={"Name": parameter_name},
3301 region=global_region,
3302 physical_resource_id=cr.PhysicalResourceId.of(f"cluster-shared-{suffix}"),
3303 ),
3304 on_update=cr.AwsSdkCall(
3305 service="SSM",
3306 action="getParameter",
3307 parameters={"Name": parameter_name},
3308 region=global_region,
3309 physical_resource_id=cr.PhysicalResourceId.of(f"cluster-shared-{suffix}"),
3310 ),
3311 # Cross-region SSM GetParameter doesn't support resource-level
3312 # scoping cleanly — the principal evaluating the call lives in
3313 # this stack's region but the parameter lives in the global
3314 # region. ANY_RESOURCE is the AWS-documented escape hatch; the
3315 # resulting AwsSolutions-IAM5 nag finding is suppressed in a
3316 # scoped add_resource_suppressions call below.
3317 policy=cr.AwsCustomResourcePolicy.from_sdk_calls(
3318 resources=cr.AwsCustomResourcePolicy.ANY_RESOURCE
3319 ),
3320 )
3322 # Scoped suppression: the CR policy is Resource::* because the
3323 # SSM parameter lives in the global region (cross-region calls
3324 # don't support resource-level scoping cleanly). The action is
3325 # fixed to ssm:GetParameter and the parameter Name is fixed to
3326 # a literal string, so the wildcard can only ever read one
3327 # parameter.
3328 acknowledge_nag_findings(
3329 read_cr,
3330 [
3331 {
3332 "id": "AwsSolutions-IAM5",
3333 "reason": (
3334 "Cross-region ssm:GetParameter for "
3335 f"{parameter_name} in the global region. The "
3336 "AwsCustomResource SDK-call policy is scoped to "
3337 "a single fixed action (ssm:GetParameter) with "
3338 "a fixed parameter Name — the Resource: * is "
3339 "the CDK-documented escape hatch because the "
3340 "parameter ARN is not known to the calling "
3341 "principal's region. Effective blast radius: "
3342 "one parameter."
3343 ),
3344 "appliesTo": ["Resource::*"],
3345 },
3346 ],
3347 )
3349 resolved[suffix] = read_cr.get_response_field("Parameter.Value")
3351 return SharedBucketIdentity(
3352 name=resolved["name"],
3353 arn=resolved["arn"],
3354 region=resolved["region"],
3355 )
3357 def _grant_cluster_shared_bucket_to_job_role(self, shared: SharedBucketIdentity) -> None:
3358 """Attach RW + KMS permissions on ``Cluster_Shared_Bucket`` to the job-pod role.
3360 Two ``iam.PolicyStatement``s are added to ``self.service_account_role``
3361 (the EKS Pod Identity role used by every pod in ``gco-jobs``,
3362 ``gco-system``, and ``gco-inference``):
3364 1. S3 object + bucket-level actions (``GetObject``, ``PutObject``,
3365 ``DeleteObject``, ``ListBucket``, ``GetBucketLocation``) scoped
3366 to ``<shared.arn>`` and ``<shared.arn>/*`` — the literal ARN
3367 resolved from SSM (the bucket's name is CloudFormation-generated,
3368 so no prefix pattern is involved).
3369 2. KMS ``Decrypt`` / ``GenerateDataKey`` scoped by the
3370 ``kms:ViaService=s3.<shared.region>.<AWS::URLSuffix>`` condition —
3371 ``resources=["*"]`` because the KMS key ARN is not known to this
3372 stack (it lives in the global region and is referenced indirectly
3373 through the S3 service). The condition is what actually restricts
3374 the grant to the cluster-shared bucket's key.
3376 Runs unconditionally — the grant is
3377 present on every regional cluster whether or not analytics is
3378 enabled.
3379 """
3380 from gco.stacks.nag_suppressions import acknowledge_nag_findings
3382 self.service_account_role.add_to_policy(
3383 iam.PolicyStatement(
3384 effect=iam.Effect.ALLOW,
3385 actions=[
3386 "s3:GetObject",
3387 "s3:PutObject",
3388 "s3:DeleteObject",
3389 "s3:ListBucket",
3390 "s3:GetBucketLocation",
3391 ],
3392 resources=[shared.arn, f"{shared.arn}/*"],
3393 )
3394 )
3396 self.service_account_role.add_to_policy(
3397 iam.PolicyStatement(
3398 effect=iam.Effect.ALLOW,
3399 actions=["kms:Decrypt", "kms:GenerateDataKey"],
3400 resources=["*"],
3401 conditions={
3402 "StringEquals": {
3403 "kms:ViaService": f"s3.{shared.region}.{self.url_suffix}",
3404 }
3405 },
3406 )
3407 )
3409 # The grants contain two necessary wildcard shapes. The S3 bucket ARN
3410 # uses ``/*`` for object keys within the single resolved shared bucket.
3411 # KMS uses ``Resource::*`` because the global key ARN is not exported to
3412 # this stack; ``kms:ViaService`` confines its use to S3 in the bucket's
3413 # region, while the S3 statements separately scope accessible objects.
3414 acknowledge_nag_findings(
3415 self.service_account_role,
3416 [
3417 {
3418 "id": "AwsSolutions-IAM5",
3419 "reason": (
3420 "The Cluster_Shared_Bucket grants require two wildcard shapes: "
3421 "an <arn>/* object-key suffix on the single shared bucket resolved "
3422 "from SSM, and KMS Resource::* because the global key ARN is not "
3423 "exported. KMS use is constrained by kms:ViaService to S3 in the "
3424 "bucket's region, and S3 access is separately limited to the "
3425 "allowed bucket ARNs."
3426 ),
3427 "appliesTo": [
3428 "Resource::*",
3429 "Resource::<ReadClusterSharedBucketArn4B0BD291.Parameter.Value>/*",
3430 ],
3431 },
3432 ],
3433 )
3435 def _create_mlflow_artifact_role(self, shared: SharedBucketIdentity) -> None:
3436 """Create the OIDC-only IRSA role MLflow uses for S3 artifact storage.
3438 The official mlflow chart creates a ``mlflow`` ServiceAccount in the
3439 ``monitoring`` namespace (``fullnameOverride`` keeps the bare name);
3440 the value overrides annotate it with this role's ARN so the tracking
3441 server exchanges its webhook-injected projected token for
3442 credentials, which is what feeds the server-side S3 artifact proxy
3443 (``mlflow.artifactsDestination``). The chart's default
3444 ``automountServiceAccountToken: false`` does not affect IRSA — the
3445 EKS pod identity webhook mounts its own token volume. Controller-
3446 style posture: OIDC-only (``include_pod_identity=False``), trust
3447 bound to exactly one namespace/service-account pair.
3449 Grants are deliberately narrower than the job-pod role's bucket-wide
3450 grant: object access only under the ``mlflow-artifacts/`` prefix of
3451 the cluster-shared bucket, ``ListBucket`` condition-scoped to the
3452 same prefix, and KMS confined by ``kms:ViaService`` exactly like
3453 ``_grant_cluster_shared_bucket_to_job_role``.
3454 """
3455 from gco.stacks.nag_suppressions import acknowledge_nag_findings
3457 self.mlflow_role = GCORegionalStack._create_irsa_role(
3458 self,
3459 "MlflowArtifactRole",
3460 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
3461 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
3462 service_account_names=["mlflow"],
3463 namespaces=["monitoring"],
3464 include_pod_identity=False,
3465 )
3467 self.mlflow_role.add_to_policy(
3468 iam.PolicyStatement(
3469 effect=iam.Effect.ALLOW,
3470 actions=["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
3471 resources=[f"{shared.arn}/mlflow-artifacts/*"],
3472 )
3473 )
3474 self.mlflow_role.add_to_policy(
3475 iam.PolicyStatement(
3476 effect=iam.Effect.ALLOW,
3477 actions=["s3:ListBucket"],
3478 resources=[shared.arn],
3479 conditions={
3480 "StringLike": {
3481 "s3:prefix": "mlflow-artifacts/*",
3482 }
3483 },
3484 )
3485 )
3486 # GetBucketLocation cannot share the ListBucket statement: requests
3487 # for it never carry the s3:prefix key, so the condition above would
3488 # implicitly deny it.
3489 self.mlflow_role.add_to_policy(
3490 iam.PolicyStatement(
3491 effect=iam.Effect.ALLOW,
3492 actions=["s3:GetBucketLocation"],
3493 resources=[shared.arn],
3494 )
3495 )
3496 self.mlflow_role.add_to_policy(
3497 iam.PolicyStatement(
3498 effect=iam.Effect.ALLOW,
3499 actions=["kms:Decrypt", "kms:GenerateDataKey"],
3500 resources=["*"],
3501 conditions={
3502 "StringEquals": {
3503 "kms:ViaService": f"s3.{shared.region}.{self.url_suffix}",
3504 }
3505 },
3506 )
3507 )
3509 acknowledge_nag_findings(
3510 self.mlflow_role,
3511 [
3512 {
3513 "id": "AwsSolutions-IAM5",
3514 "reason": (
3515 "The MLflow artifact grants require two wildcard shapes: a "
3516 "mlflow-artifacts/* object-key suffix within the single shared "
3517 "bucket resolved from SSM, and KMS Resource::* because the "
3518 "global key ARN is not exported to this stack. KMS use is "
3519 "constrained by kms:ViaService to S3 in the bucket's region, "
3520 "and S3 access is separately limited to the artifact prefix."
3521 ),
3522 "appliesTo": [
3523 "Resource::*",
3524 "Resource::<ReadClusterSharedBucketArn4B0BD291.Parameter.Value>/mlflow-artifacts/*",
3525 ],
3526 },
3527 ],
3528 )
3530 def _create_regional_shared_bucket(self) -> None:
3531 """Create the always-on general-purpose regional bucket for this region.
3533 Provisioned unconditionally — there is no ``cdk.json`` toggle and no
3534 feature flag that can suppress it — in addition to the central buckets
3535 owned by ``GCOGlobalStack`` (the model bucket and the cluster-shared
3536 bucket). The bucket is general purpose: any in-region workload may use
3537 it, and the per-region cold KV tier auto-targets it when cold-tier
3538 storage is requested. Its existence is independent of any endpoint's
3539 cold-tier choice.
3541 Three constructs are created, mirroring the cluster-shared bucket
3542 pattern in ``GCOGlobalStack``:
3544 1. ``regional_shared_kms_key`` — a customer-managed KMS key with annual
3545 rotation and a 7-day pending window on destroy. The key policy grants
3546 the ``s3.amazonaws.com`` and ``logs.<region>.amazonaws.com`` service
3547 principals encrypt/decrypt so S3 server-side encryption and access-log
3548 delivery work without role-side grants.
3549 2. ``regional_shared_access_logs_bucket`` — the dedicated S3 access-logs
3550 destination for the primary bucket.
3551 3. ``regional_shared_bucket`` — the primary bucket. Its physical name
3552 is CloudFormation-generated (``<stack>-regionalsharedbucket…``):
3553 S3 bucket names are a global namespace and a deleted name is not
3554 reliably reusable, so a fixed project/account/region name would
3555 make every destroy-and-redeploy (and the ``retain`` policy below)
3556 a collision hazard. Consumers never reconstruct it — they read the
3557 SSM parameters published under
3558 ``regional_shared_ssm_parameter_prefix(project_name)`` or the
3559 ``gco-regional-shared-bucket`` ConfigMap. KMS-encrypted with
3560 ``regional_shared_kms_key``, block-public-access on, SSL enforced,
3561 versioned, destroy-on-teardown.
3563 An explicit ``Deny`` for ``aws:SecureTransport=false`` is added to the
3564 bucket policy independent of ``enforce_ssl=True`` so the deny is
3565 verifiable in the synthesized template under a known SID.
3566 """
3567 # Teardown behavior is configurable (cdk.json::regional_shared_bucket.
3568 # removal_policy) because this bucket holds artifacts jobs just
3569 # produced: 'destroy' (the default, preserving historical teardown
3570 # semantics for existing deployments and validation cycles) deletes
3571 # bucket, logs, and key with the region; 'retain' lets all three
3572 # outlive a regional destroy so checkpoints survive. Invalid values
3573 # fail synthesis rather than silently choosing a side. Keep this in
3574 # sync with the tolerant CLI-side read in cli/storage.py
3575 # (_regional_shared_removal_policy) that `gco storage s3-inventory`
3576 # reports through.
3577 retention_context = self.node.try_get_context("regional_shared_bucket") or {}
3578 configured_policy = str(retention_context.get("removal_policy", "destroy")).strip().lower()
3579 if configured_policy not in ("destroy", "retain"):
3580 raise ValueError(
3581 "regional_shared_bucket.removal_policy must be 'destroy' or 'retain', "
3582 f"got {retention_context.get('removal_policy')!r}"
3583 )
3584 retain_regional_shared = configured_policy == "retain"
3585 regional_shared_removal_policy = (
3586 RemovalPolicy.RETAIN if retain_regional_shared else RemovalPolicy.DESTROY
3587 )
3589 # KMS key for the regional bucket. Matches the cluster-shared key
3590 # posture: annual rotation, 7-day pending window, destroy-on-teardown
3591 # by default. Under 'retain' the key survives with the bucket —
3592 # a retained bucket whose key was scheduled for deletion would be
3593 # undecryptable, so the two always share a fate.
3594 self.regional_shared_kms_key = kms.Key(
3595 self,
3596 "RegionalSharedKmsKey",
3597 description=(
3598 "Customer-managed KMS key for the always-on general-purpose "
3599 "regional bucket in this region's GCORegionalStack."
3600 ),
3601 enable_key_rotation=True,
3602 pending_window=Duration.days(7),
3603 removal_policy=regional_shared_removal_policy,
3604 )
3606 # Key-policy grants for the service principals that encrypt/decrypt on
3607 # behalf of the bucket (S3 server-side encryption) and the access-logs
3608 # bucket (CloudWatch/S3 log delivery).
3609 kms_actions = [
3610 "kms:Encrypt",
3611 "kms:Decrypt",
3612 "kms:ReEncrypt*",
3613 "kms:GenerateDataKey*",
3614 "kms:DescribeKey",
3615 ]
3617 self.regional_shared_kms_key.add_to_resource_policy(
3618 iam.PolicyStatement(
3619 sid="AllowS3ServiceEncryptDecrypt",
3620 effect=iam.Effect.ALLOW,
3621 principals=[iam.ServicePrincipal("s3.amazonaws.com")],
3622 actions=kms_actions,
3623 resources=["*"],
3624 )
3625 )
3627 self.regional_shared_kms_key.add_to_resource_policy(
3628 iam.PolicyStatement(
3629 sid="AllowCloudWatchLogsEncryptDecrypt",
3630 effect=iam.Effect.ALLOW,
3631 principals=[iam.ServicePrincipal(f"logs.{self.deployment_region}.amazonaws.com")],
3632 actions=kms_actions,
3633 resources=["*"],
3634 )
3635 )
3637 # Retention for the access-logs bucket honors the same `s3_access_logs`
3638 # context field used by the central buckets (default 90 days).
3639 s3_access_logs_ctx = self.node.try_get_context("s3_access_logs") or {}
3640 access_logs_retention_days = int(s3_access_logs_ctx.get("retention_days", 90))
3642 # Dedicated access-logs bucket for the regional bucket, encrypted with
3643 # the regional KMS key (its key policy grants the logs service principal
3644 # encrypt/decrypt).
3645 self.regional_shared_access_logs_bucket = s3.Bucket(
3646 self,
3647 "RegionalSharedAccessLogsBucket",
3648 encryption=s3.BucketEncryption.KMS,
3649 encryption_key=self.regional_shared_kms_key,
3650 block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
3651 enforce_ssl=True,
3652 versioned=True,
3653 removal_policy=regional_shared_removal_policy,
3654 auto_delete_objects=not retain_regional_shared,
3655 lifecycle_rules=[
3656 s3.LifecycleRule(
3657 id="ExpireAccessLogs",
3658 enabled=True,
3659 expiration=Duration.days(access_logs_retention_days),
3660 )
3661 ],
3662 )
3664 # Primary general-purpose regional bucket. No ``bucket_name``: the
3665 # physical name is CloudFormation-generated so a destroy-and-redeploy
3666 # (or a retained bucket from an earlier deployment) can never collide
3667 # in S3's global namespace; the IAM grants below reference the
3668 # construct's ARN token and consumers resolve the name from SSM.
3669 # `bucket_key_enabled=True` mirrors the central-bucket pattern to
3670 # reduce per-object KMS request costs.
3671 project_name = self.config.get_project_name()
3672 regional_shared_prefix = regional_shared_ssm_parameter_prefix(project_name)
3673 self.regional_shared_bucket = s3.Bucket(
3674 self,
3675 "RegionalSharedBucket",
3676 encryption=s3.BucketEncryption.KMS,
3677 encryption_key=self.regional_shared_kms_key,
3678 bucket_key_enabled=True,
3679 block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
3680 enforce_ssl=True,
3681 versioned=True,
3682 removal_policy=regional_shared_removal_policy,
3683 auto_delete_objects=not retain_regional_shared,
3684 server_access_logs_bucket=self.regional_shared_access_logs_bucket,
3685 server_access_logs_prefix="regional-shared/",
3686 )
3688 # Explicit Deny for insecure transport. `enforce_ssl=True` already adds
3689 # an equivalent statement, but duplicating it here makes the deny
3690 # verifiable in the synthesized template under a known SID.
3691 self.regional_shared_bucket.add_to_resource_policy(
3692 iam.PolicyStatement(
3693 sid="DenyInsecureTransport",
3694 effect=iam.Effect.DENY,
3695 principals=[iam.AnyPrincipal()],
3696 actions=["s3:*"],
3697 resources=[
3698 self.regional_shared_bucket.bucket_arn,
3699 f"{self.regional_shared_bucket.bucket_arn}/*",
3700 ],
3701 conditions={"Bool": {"aws:SecureTransport": "false"}},
3702 )
3703 )
3705 # Publish the bucket's identity as three SSM parameters in this
3706 # region's own parameter store, mirroring how the model bucket and
3707 # cluster-shared bucket publish theirs. In-region workloads and the
3708 # regional upload surface resolve the always-on regional bucket by
3709 # reading these back rather than reconstructing the name. Because the
3710 # bucket is unconditional, these parameters are always present once the
3711 # region's stack is deployed. The prefix from
3712 # ``regional_shared_ssm_parameter_prefix(project_name)`` is the single
3713 # source of truth for the namespace.
3714 self.regional_shared_bucket_name_parameter = ssm.StringParameter(
3715 self,
3716 "RegionalSharedBucketNameParam",
3717 parameter_name=f"{regional_shared_prefix}/name",
3718 string_value=self.regional_shared_bucket.bucket_name,
3719 description="Name of the always-on general-purpose regional bucket for this region.",
3720 )
3722 ssm.StringParameter(
3723 self,
3724 "RegionalSharedBucketArnParam",
3725 parameter_name=f"{regional_shared_prefix}/arn",
3726 string_value=self.regional_shared_bucket.bucket_arn,
3727 description="ARN of the always-on general-purpose regional bucket for this region.",
3728 )
3730 ssm.StringParameter(
3731 self,
3732 "RegionalSharedBucketRegionParam",
3733 parameter_name=f"{regional_shared_prefix}/region",
3734 string_value=self.deployment_region,
3735 description="Home region of the always-on general-purpose regional bucket.",
3736 )
3738 # CDK-nag suppressions scoped per-resource at the construct site,
3739 # mirroring the central bucket pattern. Every suppression carries an
3740 # explicit reason; no blanket bypasses.
3741 from gco.stacks.nag_suppressions import acknowledge_nag_findings
3743 regional_replication_reason = (
3744 "The general-purpose regional bucket is a region-local store; "
3745 "in-region workloads publish to their own region's bucket and there "
3746 "is no durability requirement that warrants cross-region "
3747 "replication. Access logs do not require replication for the same "
3748 "reason."
3749 )
3751 acknowledge_nag_findings(
3752 self.regional_shared_bucket,
3753 [
3754 {
3755 "id": "HIPAA.Security-S3BucketReplicationEnabled",
3756 "reason": regional_replication_reason,
3757 },
3758 {
3759 "id": "NIST.800.53.R5-S3BucketReplicationEnabled",
3760 "reason": regional_replication_reason,
3761 },
3762 {
3763 "id": "PCI.DSS.321-S3BucketReplicationEnabled",
3764 "reason": regional_replication_reason,
3765 },
3766 ],
3767 )
3769 access_logs_is_self_target_reason = (
3770 "This is the server access logs destination bucket for the "
3771 "general-purpose regional bucket."
3772 )
3773 acknowledge_nag_findings(
3774 self.regional_shared_access_logs_bucket,
3775 [
3776 {
3777 "id": "AwsSolutions-S1",
3778 "reason": access_logs_is_self_target_reason,
3779 },
3780 {
3781 "id": "HIPAA.Security-S3BucketLoggingEnabled",
3782 "reason": access_logs_is_self_target_reason,
3783 },
3784 {
3785 "id": "NIST.800.53.R5-S3BucketLoggingEnabled",
3786 "reason": access_logs_is_self_target_reason,
3787 },
3788 {
3789 "id": "PCI.DSS.321-S3BucketLoggingEnabled",
3790 "reason": access_logs_is_self_target_reason,
3791 },
3792 {
3793 "id": "HIPAA.Security-S3BucketReplicationEnabled",
3794 "reason": regional_replication_reason,
3795 },
3796 {
3797 "id": "NIST.800.53.R5-S3BucketReplicationEnabled",
3798 "reason": regional_replication_reason,
3799 },
3800 {
3801 "id": "PCI.DSS.321-S3BucketReplicationEnabled",
3802 "reason": regional_replication_reason,
3803 },
3804 ],
3805 )
3807 # Grant the in-region pod role read/write on this bucket and use of its
3808 # KMS key — and nothing else. The grant lives next to the bucket it
3809 # scopes to, so the role's regional-bucket access stays exactly as wide
3810 # as this one bucket and its key.
3811 self._grant_regional_shared_bucket_to_service_account()
3812 self._add_slurm_accounting_backup_lifecycle_rule()
3814 def _slurm_charts_enabled(self) -> bool:
3815 """Effective Slurm enablement: the cdk.json toggle or a run-scoped override."""
3816 helm_config = self.node.try_get_context("helm") or {}
3817 overrides = _parse_helm_enabled_overrides(
3818 self.node.try_get_context(_HELM_OVERRIDE_CONTEXT_KEY)
3819 )
3820 return _helm_chart_enabled(helm_config, overrides, "slurm")
3822 def _add_slurm_accounting_backup_lifecycle_rule(self) -> None:
3823 """Expire Slurm accounting dumps after ``helm.slurm.accounting.backup.retention_days``.
3825 The backup CronJob (post-helm-slurm-accounting-backup.yaml) writes one
3826 compressed ``mariadb-dump`` per run under
3827 ``slurm/accounting-backups/<cluster>/`` in this bucket. Retention is a
3828 prefix-scoped lifecycle rule rather than a delete loop in the job, so
3829 it holds even when the job stops running; the bucket is versioned, so
3830 the rule also clears the non-current versions a delete marker leaves
3831 behind. Added only while Slurm runs the in-cluster database with
3832 backups on; otherwise the prefix is never written.
3833 """
3834 slurm = self._slurm_config()
3835 if not (
3836 self._slurm_charts_enabled()
3837 and accounting_in_cluster(slurm)
3838 and slurm["accounting"]["backup"]["enabled"]
3839 ):
3840 return
3841 retention_days = int(slurm["accounting"]["backup"]["retention_days"])
3842 self.regional_shared_bucket.add_lifecycle_rule(
3843 id="ExpireSlurmAccountingBackups",
3844 enabled=True,
3845 prefix=SLURM_ACCOUNTING_BACKUP_PREFIX,
3846 expiration=Duration.days(retention_days),
3847 noncurrent_version_expiration=Duration.days(1),
3848 )
3850 def _grant_regional_shared_bucket_to_service_account(self) -> None:
3851 """Attach RW + KMS + discovery permissions on the regional bucket to the pod role.
3853 Three ``iam.PolicyStatement``s are added to ``self.service_account_role``
3854 (the EKS Pod Identity role used by every pod in ``gco-jobs``,
3855 ``gco-system``, and ``gco-inference``):
3857 1. S3 object + bucket-level actions (``GetObject``, ``PutObject``,
3858 ``DeleteObject``, ``ListBucket``, ``GetBucketLocation``) scoped to
3859 the literal ``regional_shared_bucket`` ARN and its ``<arn>/*``
3860 object-key space — and to no other bucket.
3861 2. KMS ``Decrypt`` / ``Encrypt`` / ``GenerateDataKey`` /
3862 ``DescribeKey`` scoped to the literal ``regional_shared_kms_key``
3863 ARN — and to no other key.
3864 3. ``ssm:GetParameter`` on the literal ARN of the bucket's ``/name``
3865 discovery parameter. The inference monitor runs under this role and
3866 resolves the Mooncake cold-tier bucket from that parameter at
3867 reconcile time (``_resolve_regional_shared_bucket``); without the
3868 grant every cold-tier endpoint reads ``AccessDenied`` and the tier
3869 is dropped.
3871 Because all three resources are local constructs in this stack, each
3872 ARN is a concrete reference rather than a wildcard, so the role gains
3873 access to precisely this bucket, this key, and this parameter. The grant
3874 runs unconditionally as part of provisioning the always-on regional
3875 bucket.
3876 """
3877 from gco.stacks.nag_suppressions import acknowledge_nag_findings
3879 self.service_account_role.add_to_policy(
3880 iam.PolicyStatement(
3881 effect=iam.Effect.ALLOW,
3882 actions=["ssm:GetParameter"],
3883 resources=[self.regional_shared_bucket_name_parameter.parameter_arn],
3884 )
3885 )
3887 self.service_account_role.add_to_policy(
3888 iam.PolicyStatement(
3889 effect=iam.Effect.ALLOW,
3890 actions=[
3891 "s3:GetObject",
3892 "s3:PutObject",
3893 "s3:DeleteObject",
3894 "s3:ListBucket",
3895 "s3:GetBucketLocation",
3896 ],
3897 resources=[
3898 self.regional_shared_bucket.bucket_arn,
3899 f"{self.regional_shared_bucket.bucket_arn}/*",
3900 ],
3901 )
3902 )
3904 self.service_account_role.add_to_policy(
3905 iam.PolicyStatement(
3906 effect=iam.Effect.ALLOW,
3907 actions=[
3908 "kms:Decrypt",
3909 "kms:Encrypt",
3910 "kms:GenerateDataKey",
3911 "kms:DescribeKey",
3912 ],
3913 resources=[self.regional_shared_kms_key.key_arn],
3914 )
3915 )
3917 # The S3 bucket-ARN resource uses a ``<arn>/*`` object-key wildcard
3918 # which cdk-nag flags as a wildcard resource. The ARN itself is the
3919 # literal regional bucket ARN created in this stack — the ``/*`` covers
3920 # all object keys inside that single bucket, which is the intended
3921 # semantic for the RW grant. The KMS statement carries no wildcard.
3922 acknowledge_nag_findings(
3923 self.service_account_role,
3924 [
3925 {
3926 "id": "AwsSolutions-IAM5",
3927 "reason": (
3928 "The regional bucket RW grant uses an <arn>/* "
3929 "object-key wildcard on the literal ARN of the "
3930 "regional-shared bucket created in this stack "
3931 "(CloudFormation-generated name). The wildcard covers object "
3932 "keys within a single bucket — this is the standard "
3933 "shape for a bucket-scoped RW grant and is what the "
3934 "allow-list assertion is written against."
3935 ),
3936 "appliesTo": [
3937 "Resource::<RegionalSharedBucket3FF19783.Arn>/*",
3938 ],
3939 },
3940 ],
3941 )
3943 def _cost_report_bucket_parameter_name(self) -> str:
3944 """SSM parameter (in the monitoring region) publishing the cost bucket name."""
3945 return f"{cost_report_ssm_parameter_prefix(self.config.get_project_name())}/name"
3947 def _grant_cost_report_bucket_discovery_to_cost_monitor(self) -> None:
3948 """Let the cost-monitor role resolve the cost report bucket it writes to.
3950 The bucket lives in ``GCOMonitoringStack`` in the monitoring region,
3951 which deploys *after* every regional stack, and it carries a
3952 CloudFormation-generated physical name — S3 bucket names are a global
3953 namespace and a deleted name is not reliably reusable, so nothing
3954 reconstructs it from project/account/region any more. That inverts
3955 the old grant direction:
3957 1. The monitoring stack publishes the bucket's identity at
3958 ``<cost_report_ssm_parameter_prefix>/{name,arn,region}`` and grants
3959 every regional cost-monitor role S3 object/bucket actions through
3960 the bucket policy and KMS use through the key policy — principal
3961 based, so this stack never needs the bucket or key ARN.
3962 2. This stack grants the role exactly one permission:
3963 ``ssm:GetParameter`` on the ``/name`` parameter, by literal ARN. The
3964 service reads it at runtime (``COST_REPORT_BUCKET_PARAMETER`` /
3965 ``COST_REPORT_BUCKET_PARAMETER_REGION`` in ``34-cost-monitor.yaml``).
3967 On a fresh ``deploy-all`` the parameter appears only once monitoring is
3968 deployed; the service re-resolves on every scheduled pass, so the
3969 pipeline self-heals without ordering hacks. No wildcard remains on the
3970 role, so no cdk-nag acknowledgement is needed here.
3971 """
3972 monitoring_region = self.config.get_monitoring_region()
3973 parameter_name = self._cost_report_bucket_parameter_name()
3974 self.cost_monitor_role.add_to_policy(
3975 iam.PolicyStatement(
3976 effect=iam.Effect.ALLOW,
3977 actions=["ssm:GetParameter"],
3978 resources=[
3979 f"arn:{self.partition}:ssm:{monitoring_region}:{self.account}:"
3980 f"parameter{parameter_name}"
3981 ],
3982 )
3983 )
3985 def _create_kubectl_lambda(self) -> None:
3986 """Create Lambda function to apply Kubernetes manifests using Python client.
3988 Note: This creates the Lambda and provider but does NOT create the custom resource.
3989 The custom resource is created in _apply_kubernetes_manifests() after ALB is created,
3990 so that target group ARNs can be passed to the manifests.
3991 """
3992 project_name = self.config.get_project_name()
3994 # Create IAM role for kubectl Lambda
3995 kubectl_lambda_role = iam.Role(
3996 self,
3997 "KubectlLambdaRole",
3998 assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
3999 managed_policies=[
4000 iam.ManagedPolicy.from_aws_managed_policy_name(
4001 "service-role/AWSLambdaVPCAccessExecutionRole"
4002 ),
4003 iam.ManagedPolicy.from_aws_managed_policy_name(
4004 "service-role/AWSLambdaBasicExecutionRole"
4005 ),
4006 ],
4007 )
4009 # Add EKS permissions
4010 kubectl_lambda_role.add_to_policy(
4011 iam.PolicyStatement(
4012 actions=[
4013 "eks:DescribeCluster",
4014 "eks:ListClusters",
4015 ],
4016 resources=[self.cluster.cluster_arn],
4017 )
4018 )
4020 # Add permissions to assume cluster admin role
4021 kubectl_lambda_role.add_to_policy(
4022 iam.PolicyStatement(actions=["sts:AssumeRole"], resources=["*"])
4023 )
4025 # Allow the convergence apply tasks to record per-phase status to SSM
4026 # (base-manifests / post-helm-manifests), mirroring the helm worker, so
4027 # `gco stacks addons status` surfaces the apply passes alongside charts.
4028 kubectl_lambda_role.add_to_policy(
4029 iam.PolicyStatement(
4030 actions=["ssm:PutParameter"],
4031 resources=[
4032 f"arn:{self.partition}:ssm:{self.deployment_region}:{self.account}:"
4033 f"parameter/{project_name}/addons/*"
4034 ],
4035 )
4036 )
4038 # Create security group for kubectl Lambda
4039 kubectl_lambda_sg = ec2.SecurityGroup(
4040 self,
4041 "KubectlLambdaSG",
4042 vpc=self.vpc,
4043 description="Security group for kubectl Lambda to access EKS cluster",
4044 security_group_name=f"{self.config.get_project_name()}-kubectl-lambda-sg-{self.deployment_region}",
4045 allow_all_outbound=True, # Lambda needs outbound access to EKS API
4046 )
4048 # Allow Lambda security group to access EKS cluster security group on port 443
4049 # The EKS cluster security group is automatically created by EKS
4050 self.cluster.cluster_security_group.add_ingress_rule(
4051 peer=kubectl_lambda_sg,
4052 connection=ec2.Port.tcp(443),
4053 description="Allow kubectl Lambda to access EKS API",
4054 )
4056 # Create Lambda function (Python-only, no Docker!)
4057 # Store function name as string attribute for cross-stack references
4058 # This avoids CDK cross-environment resolution issues when account is unresolved
4059 self.kubectl_lambda_function_name = f"{project_name}-kubectl-{self.deployment_region}"
4060 self.kubectl_lambda = lambda_.Function(
4061 self,
4062 "KubectlApplierFunction",
4063 function_name=self.kubectl_lambda_function_name,
4064 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
4065 handler="handler.lambda_handler",
4066 code=lambda_.Code.from_asset("lambda/kubectl-applier-simple-build"),
4067 timeout=Duration.minutes(15), # Max Lambda timeout
4068 memory_size=512,
4069 role=kubectl_lambda_role,
4070 vpc=self.vpc,
4071 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
4072 security_groups=[kubectl_lambda_sg], # Use the security group we created
4073 environment={
4074 "CLUSTER_NAME": self.cluster.cluster_name,
4075 "REGION": self.deployment_region,
4076 # Lets the convergence apply tasks record per-phase status to
4077 # SSM (/<project>/addons/<region>/{base,post-helm}-manifests).
4078 "PROJECT_NAME": project_name,
4079 },
4080 tracing=lambda_.Tracing.ACTIVE,
4081 )
4083 # Add EKS access entry for the Lambda role to authenticate with the cluster
4084 # This grants the Lambda role cluster admin permissions
4085 self.kubectl_lambda_access_entry = eks.AccessEntry(
4086 self,
4087 "KubectlLambdaAccessEntry",
4088 cluster=self.cluster, # type: ignore[arg-type]
4089 principal=kubectl_lambda_role.role_arn,
4090 access_policies=[
4091 eks.AccessPolicy.from_access_policy_name(
4092 "AmazonEKSClusterAdminPolicy", access_scope_type=eks.AccessScopeType.CLUSTER
4093 )
4094 ],
4095 )
4097 # No custom-resource provider needed: the kubectl-applier Lambda is now
4098 # invoked directly by the convergence state machine (the base and
4099 # post-Helm apply tasks), not through a CloudFormation custom resource.
4101 # cdk-nag suppression: the kubectl-applier Lambda requires broad
4102 # EKS and Kubernetes API access to apply arbitrary manifests.
4103 from gco.stacks.nag_suppressions import acknowledge_nag_findings
4105 acknowledge_nag_findings(
4106 kubectl_lambda_role,
4107 [
4108 {
4109 "id": "AwsSolutions-IAM5",
4110 "reason": (
4111 "The kubectl-applier Lambda requires broad EKS and Kubernetes API "
4112 "access to apply arbitrary manifests (RBAC, ServiceAccounts, "
4113 "Deployments, Jobs, NetworkPolicies) across multiple namespaces. "
4114 "Resource: * is required because the set of Kubernetes resources "
4115 "is dynamic and not known at synth time."
4116 ),
4117 "appliesTo": ["Resource::*"],
4118 },
4119 ],
4120 )
4122 def _apply_kubernetes_manifests(self) -> None:
4123 """Build the complete base/Helm/post-Helm convergence pipeline.
4125 This is called after the Gateway certificate and shared storage exist.
4126 The post-Helm pass creates the internal ALB through Gateway API only
4127 after the mandatory AWS Load Balancer Controller is installed.
4128 """
4130 # Build image replacements dict
4131 # Include one deployment token to force pod rollouts and bind live
4132 # validation to this exact asynchronous convergence execution.
4133 deployment_timestamp = _deployment_timestamp()
4134 self.addon_deployment_token = deployment_timestamp
4136 # Get resource thresholds from config
4137 thresholds = self.config.get_resource_thresholds()
4139 # Get manifest processor resource quotas.
4140 # Resource quotas and the security/image policy now live under the
4141 # shared job_validation_policy section because both the REST
4142 # manifest_processor and the SQS queue_processor read them. Service-
4143 # specific knobs (replicas, validation_enabled, max_request_body_bytes,
4144 # etc.) stay under manifest_processor. Inference TLS proxy CPU and HPA
4145 # settings live in their own optional block.
4146 mp_config = self.config.get_manifest_processor_config()
4147 inference_proxy_config = self.config.get_inference_proxy_config()
4148 job_policy = self.node.try_get_context("job_validation_policy") or {}
4149 job_quotas = _validated_manifest_caps(
4150 job_policy.get("resource_quotas", {}),
4151 _validated_resource_quota(self.node.try_get_context("resource_quota") or {}),
4152 )
4153 allowed_kinds = job_policy.get(
4154 "allowed_kinds",
4155 # Fallback mirrors manifest_processor.DEFAULT_ALLOWED_KINDS (kept
4156 # inline so CDK synth never imports service modules; lockstep is
4157 # pinned by tests/test_manifest_processor_extended.py::TestAllowedKindsLockstep).
4158 [
4159 "Job",
4160 "CronJob",
4161 "Deployment",
4162 "StatefulSet",
4163 "DaemonSet",
4164 "Service",
4165 "ConfigMap",
4166 "Pod",
4167 "TrainJob",
4168 ],
4169 )
4171 security_policy = validate_manifest_security_policy(
4172 job_policy.get("manifest_security_policy", {})
4173 )
4175 def _policy_str(value: object) -> str:
4176 if type(value) is not bool:
4177 raise ValueError("job validation policy values must be booleans")
4178 return "true" if value else "false"
4180 require_accelerator_toleration = _policy_str(
4181 job_policy.get("require_accelerator_toleration", True)
4182 )
4183 validation_enabled = _policy_str(mp_config.get("validation_enabled", True))
4185 image_replacements = {
4186 "{{BACKEND_TLS_CERTIFICATE_ARN}}": self.backend_tls_certificate_arn,
4187 "{{HEALTH_MONITOR_IMAGE}}": self.health_monitor_image.image_uri,
4188 "{{MANIFEST_PROCESSOR_IMAGE}}": self.manifest_processor_image.image_uri,
4189 "{{INFERENCE_PROXY_IMAGE}}": self.inference_proxy_image.image_uri,
4190 "{{INFERENCE_MONITOR_IMAGE}}": self.inference_monitor_image.image_uri,
4191 # External, pinned upstream image for the shared Mooncake master
4192 # (bundles the mooncake_master binary). Same default as disaggregated
4193 # role pods; per-endpoint spec.mooncake.store.master_image overrides.
4194 "{{MOONCAKE_MASTER_IMAGE}}": MOONCAKE_MASTER_DEFAULT_IMAGE,
4195 "{{CLUSTER_NAME}}": self.cluster.cluster_name,
4196 "{{REGION}}": self.deployment_region,
4197 "{{AUTH_SECRET_ARN}}": self.auth_secret_arn,
4198 "{{SERVICE_ACCOUNT_ROLE_ARN}}": self.service_account_role.role_arn,
4199 "{{MANIFEST_PROCESSOR_ROLE_ARN}}": self.manifest_processor_role.role_arn,
4200 "{{INFERENCE_PROXY_ROLE_ARN}}": self.inference_proxy_role.role_arn,
4201 "{{HEALTH_MONITOR_ROLE_ARN}}": self.health_monitor_role.role_arn,
4202 "{{EFS_FILE_SYSTEM_ID}}": self.efs_file_system.file_system_id,
4203 "{{EFS_ACCESS_POINT_ID}}": self.efs_access_point.access_point_id,
4204 "{{JOB_QUEUE_URL}}": self.job_queue.queue_url,
4205 "{{JOB_QUEUE_ARN}}": self.job_queue.queue_arn,
4206 "{{DEPLOYMENT_TIMESTAMP}}": deployment_timestamp,
4207 # Resource thresholds
4208 "{{CPU_THRESHOLD}}": str(thresholds.cpu_threshold),
4209 "{{MEMORY_THRESHOLD}}": str(thresholds.memory_threshold),
4210 "{{GPU_THRESHOLD}}": str(thresholds.gpu_threshold),
4211 "{{PENDING_PODS_THRESHOLD}}": str(thresholds.pending_pods_threshold),
4212 "{{PENDING_REQUESTED_CPU_VCPUS}}": str(thresholds.pending_requested_cpu_vcpus),
4213 "{{PENDING_REQUESTED_MEMORY_GB}}": str(thresholds.pending_requested_memory_gb),
4214 "{{PENDING_REQUESTED_GPUS}}": str(thresholds.pending_requested_gpus),
4215 # Deployment prefix (#139). Injected so in-cluster services (the
4216 # inference monitor) resolve project-scoped SSM paths
4217 # (/<project>/regional-shared-bucket/*) instead of a hardcoded
4218 # /gco/ namespace, letting two deployments share an account+region.
4219 "{{PROJECT_NAME}}": self.config.get_project_name(),
4220 # DynamoDB table names (from global stack)
4221 "{{TEMPLATES_TABLE_NAME}}": f"{self.config.get_project_name()}-job-templates",
4222 "{{WEBHOOKS_TABLE_NAME}}": f"{self.config.get_project_name()}-webhooks",
4223 "{{JOBS_TABLE_NAME}}": f"{self.config.get_project_name()}-jobs",
4224 "{{INFERENCE_ENDPOINTS_TABLE_NAME}}": (
4225 f"{self.config.get_project_name()}-inference-endpoints"
4226 ),
4227 # DynamoDB region (global stack region, may differ from cluster region)
4228 "{{DYNAMODB_REGION}}": self.config.get_global_region(),
4229 # Global region for cross-region SSM reads/writes (e.g. the health
4230 # monitor's /<project>/alb-hostname-<region> sync in the global region).
4231 "{{GLOBAL_REGION}}": self.config.get_global_region(),
4232 # Manifest processor resource quotas (sourced from shared policy).
4233 "{{MP_MAX_CPU_PER_MANIFEST}}": job_quotas["max_cpu_per_manifest"],
4234 "{{MP_MAX_MEMORY_PER_MANIFEST}}": job_quotas["max_memory_per_manifest"],
4235 "{{MP_MAX_GPU_PER_MANIFEST}}": job_quotas["max_gpu_per_manifest"],
4236 # Require accelerator (GPU/Neuron/EFA) jobs to carry a matching
4237 # toleration (shared policy). Mirrored on the SQS path via
4238 # {{QP_REQUIRE_ACCELERATOR_TOLERATION}} so neither path is a bypass.
4239 "{{MP_REQUIRE_ACCELERATOR_TOLERATION}}": require_accelerator_toleration,
4240 # Manifest processor namespace allowlist (sourced from shared policy).
4241 # Both the REST manifest processor and the SQS queue processor
4242 # read from job_validation_policy.allowed_namespaces so a single
4243 # edit takes effect on both submission paths at the next deploy.
4244 "{{MP_ALLOWED_NAMESPACES}}": ",".join(
4245 job_policy.get("allowed_namespaces", ["gco-jobs"])
4246 ),
4247 # Manifest processor Kubernetes resource kind allowlist (shared policy).
4248 "{{MP_ALLOWED_KINDS}}": ",".join(allowed_kinds),
4249 # Manifest processor image registry allowlist (sourced from shared
4250 # policy). Augmented with the project's own ECR registry hostnames
4251 # so jobs built via ``gco images build`` aren't rejected by the
4252 # REST submission path. Identical augmentation runs on the SQS
4253 # path below — see ``{{QP_TRUSTED_REGISTRIES}}``.
4254 "{{MP_TRUSTED_REGISTRIES}}": ",".join(
4255 _augment_trusted_registries_with_project_ecr(
4256 job_policy.get("trusted_registries", []),
4257 account=self.account,
4258 regions=self.config.get_regions(),
4259 global_region=self.config.get_global_region(),
4260 url_suffix=self.url_suffix,
4261 )
4262 ),
4263 "{{MP_TRUSTED_DOCKERHUB_ORGS}}": ",".join(job_policy.get("trusted_dockerhub_orgs", [])),
4264 # Manifest parsing and pod-security policy use the same values as
4265 # the queue processor, preventing REST/SQS validation drift.
4266 "{{MP_VALIDATION_ENABLED}}": validation_enabled,
4267 "{{MP_YAML_MAX_DEPTH}}": str(mp_config.get("yaml_max_depth", 50)),
4268 "{{MP_BLOCK_PRIVILEGED}}": _policy_str(security_policy["block_privileged"]),
4269 "{{MP_BLOCK_PRIVILEGE_ESCALATION}}": _policy_str(
4270 security_policy["block_privilege_escalation"]
4271 ),
4272 "{{MP_BLOCK_HOST_NETWORK}}": _policy_str(security_policy["block_host_network"]),
4273 "{{MP_BLOCK_HOST_PID}}": _policy_str(security_policy["block_host_pid"]),
4274 "{{MP_BLOCK_HOST_IPC}}": _policy_str(security_policy["block_host_ipc"]),
4275 "{{MP_BLOCK_HOST_PATH}}": _policy_str(security_policy["block_host_path"]),
4276 "{{MP_BLOCK_ADDED_CAPABILITIES}}": _policy_str(
4277 security_policy["block_added_capabilities"]
4278 ),
4279 "{{MP_BLOCK_RUN_AS_ROOT}}": _policy_str(security_policy["block_run_as_root"]),
4280 # Manifest processor request body size cap (HTTP 413 middleware).
4281 # Lives at cdk.json::manifest_processor.max_request_body_bytes.
4282 "{{MP_MAX_REQUEST_BODY_BYTES}}": str(
4283 mp_config.get("max_request_body_bytes", 1_048_576)
4284 ),
4285 # Inference request bodies use the same operator-configured cap,
4286 # but retain a service-specific placeholder for future tuning.
4287 "{{INFERENCE_PROXY_MAX_REQUEST_BODY_BYTES}}": str(
4288 mp_config.get("max_request_body_bytes", 1_048_576)
4289 ),
4290 # Shared pure renderer keeps production and Kind typed values in
4291 # lockstep (Quantity string for request, YAML integers for the HPA
4292 # target and the min/max replica bounds).
4293 **_compute_inference_proxy_tls_replacements(inference_proxy_config),
4294 # Manifest-processor sizing is an operator decision made in
4295 # cdk.json (validated in ConfigLoader): the fixed replica count and
4296 # the application container's limits render verbatim. The optional
4297 # CPU HPA (35-manifest-processor-hpa.yaml) is gated by
4298 # {{MP_HPA_ENABLED}} below; when it is on, the Deployment carries
4299 # gco.aws/hpa-controls-replicas="true" so the applier stops
4300 # re-asserting replicas and the HPA's scale value survives.
4301 "{{MP_REPLICAS}}": str(mp_config["replicas"]),
4302 "{{MP_CPU_LIMIT}}": str(mp_config["resource_limits"]["cpu"]),
4303 "{{MP_MEMORY_LIMIT}}": str(mp_config["resource_limits"]["memory"]),
4304 "{{MP_HPA_CONTROLS_REPLICAS}}": (
4305 "true" if mp_config["autoscaling"]["enabled"] else "false"
4306 ),
4307 # Regional worker for the DynamoDB-backed global queue. Multiple API
4308 # replicas are safe because JobStore claims are conditional and
4309 # lease-backed; each replica also reconciles K8s status transitions.
4310 "{{CENTRAL_QUEUE_WORKER_ENABLED}}": (
4311 "true" if mp_config.get("central_queue_worker_enabled", True) else "false"
4312 ),
4313 "{{CENTRAL_QUEUE_POLL_INTERVAL_SECONDS}}": str(
4314 mp_config.get("central_queue_poll_interval_seconds", 10)
4315 ),
4316 "{{CENTRAL_QUEUE_BATCH_SIZE}}": str(mp_config.get("central_queue_batch_size", 5)),
4317 "{{CENTRAL_QUEUE_RECONCILE_LIMIT}}": str(
4318 mp_config.get("central_queue_reconcile_limit", 100)
4319 ),
4320 "{{CENTRAL_QUEUE_LEASE_SECONDS}}": str(
4321 mp_config.get("central_queue_lease_seconds", 300)
4322 ),
4323 "{{CENTRAL_QUEUE_LEASE_RENEWAL_SECONDS}}": str(
4324 mp_config.get("central_queue_lease_renewal_seconds", 60)
4325 ),
4326 "{{QUEUE_TARGET_REGIONS}}": ",".join(self.config.get_regions()),
4327 }
4329 # Always-on Cluster_Shared_Bucket replacements. Populated from the
4330 # SharedBucketIdentity resolved in __init__ via cross-region SSM
4331 # read from GCOGlobalStack. Never gated on the analytics toggle —
4332 # the gco-cluster-shared-bucket ConfigMap is applied to every
4333 # regional cluster.
4334 image_replacements.update(
4335 _compute_kubectl_cluster_shared_replacements(self.cluster_shared_identity)
4336 )
4338 # Always-on Regional_Shared_Bucket replacements. Read straight off the
4339 # local constructs created by _create_regional_shared_bucket (which
4340 # runs earlier in __init__, before _apply_kubernetes_manifests), so no
4341 # SSM round-trip is needed — unlike the cluster-shared bucket, this one
4342 # is owned by this stack. Never gated on a toggle: the bucket is
4343 # unconditional, so the gco-regional-shared-bucket ConfigMap is applied
4344 # to every regional cluster.
4345 image_replacements.update(
4346 _compute_kubectl_regional_shared_replacements(
4347 name=self.regional_shared_bucket.bucket_name,
4348 arn=self.regional_shared_bucket.bucket_arn,
4349 region=self.deployment_region,
4350 )
4351 )
4353 # OpenTelemetry tracing of the API services (on by default). Both
4354 # placeholders are always substituted — "false" when tracing is off —
4355 # because they sit in the core service Deployments, and a leftover
4356 # placeholder would make the applier skip the whole file.
4357 image_replacements.update(_compute_kubectl_tracing_replacements(self._tracing_config()))
4359 # Cluster observability (on by default): gate the gp3 StorageClass and
4360 # the ServiceMonitors/dashboards on the toggle. When enabled the gating
4361 # placeholders resolve so those manifests apply; when disabled the keys
4362 # are absent, so the manifests keep an unreplaced placeholder and the
4363 # applier skips them (same mechanism FSx/Valkey use).
4364 _obs_config = self.config.get_cluster_observability_config()
4365 image_replacements.update(
4366 _compute_kubectl_observability_replacements(
4367 bool(_obs_config["enabled"]),
4368 grafana_admin_password_rotation_schedule=str(
4369 _obs_config["grafana"]["admin_password_rotation_schedule"]
4370 ),
4371 )
4372 )
4373 # Scheduler gates: resolve through the same enablement helper that
4374 # selects the Helm charts, so the default Kueue queue topology and the
4375 # Slinky Slurm NetworkPolicies apply exactly when their scheduler does.
4376 _helm_config = self.node.try_get_context("helm") or {}
4377 _helm_overrides = _parse_helm_enabled_overrides(
4378 self.node.try_get_context(_HELM_OVERRIDE_CONTEXT_KEY)
4379 )
4380 image_replacements.update(
4381 _compute_kubectl_scheduler_replacements(
4382 kueue_enabled=_helm_chart_enabled(_helm_config, _helm_overrides, "kueue"),
4383 slurm_enabled=_helm_chart_enabled(_helm_config, _helm_overrides, "slurm"),
4384 kubeflow_trainer_enabled=_helm_chart_enabled(
4385 _helm_config, _helm_overrides, "kubeflow_trainer"
4386 ),
4387 )
4388 )
4389 # Self-managed platform add-ons: the same enablement helper selects the
4390 # charts, so the post-Helm Argo CD fence / root Application and the
4391 # Crossplane function + RBAC apply exactly when their chart does.
4392 image_replacements.update(
4393 compute_argocd_replacements(
4394 self._argocd_config(),
4395 enabled=_helm_chart_enabled(_helm_config, _helm_overrides, "argocd"),
4396 region=self.deployment_region,
4397 # The configured (literal) cluster name renders the GitOps
4398 # path placeholders; a token would turn the path into a
4399 # deploy-time join for no gain.
4400 cluster_name=self.cluster_config.cluster_name,
4401 )
4402 )
4403 if _helm_chart_enabled(_helm_config, _helm_overrides, "crossplane"):
4404 image_replacements["{{CROSSPLANE_ENABLED}}"] = "true"
4405 # Slurm: the accounting database, its backup CronJob and the cluster
4406 # identity render only while the Slinky charts install (the same
4407 # enablement that resolves {{SLURM_ENABLED}} above).
4408 image_replacements.update(
4409 compute_slurm_replacements(
4410 self._slurm_config(),
4411 enabled=_helm_chart_enabled(_helm_config, _helm_overrides, "slurm"),
4412 )
4413 )
4415 # Cost monitoring (on by default): gate the cost-monitor Deployment
4416 # and the Grafana cost dashboard on the toggle via the same
4417 # unreplaced-placeholder mechanism. The image/role placeholders exist
4418 # only when the pipeline is active, so a disabled deployment leaves
4419 # 34-cost-monitor.yaml and the cost dashboard unapplied.
4420 if self._cost_monitoring_active():
4421 _cost_config = self.config.get_cost_monitoring_config()
4422 image_replacements.update(
4423 {
4424 "{{COST_MONITORING_ENABLED}}": "true",
4425 "{{COST_MONITOR_IMAGE}}": self.cost_monitor_image.image_uri,
4426 "{{COST_MONITOR_ROLE_ARN}}": self.cost_monitor_role.role_arn,
4427 # The bucket's CloudFormation-generated name is published by
4428 # the monitoring stack (deployed after this one); the service
4429 # resolves it from SSM at runtime instead of receiving a
4430 # reconstructed name here.
4431 "{{COST_REPORT_BUCKET_PARAMETER}}": self._cost_report_bucket_parameter_name(),
4432 "{{COST_REPORT_BUCKET_PARAMETER_REGION}}": self.config.get_monitoring_region(),
4433 "{{COST_REPORT_INTERVAL_MINUTES}}": str(
4434 _cost_config["reports"]["interval_minutes"]
4435 ),
4436 }
4437 )
4439 # Optional manifest-processor CPU autoscaler. When disabled the key is
4440 # absent, 35-manifest-processor-hpa.yaml keeps an unreplaced placeholder,
4441 # the applier skips it and prunes any HPA a previous deploy created.
4442 if mp_config["autoscaling"]["enabled"]:
4443 image_replacements.update(
4444 {
4445 "{{MP_HPA_ENABLED}}": "true",
4446 "{{MP_HPA_MAX_REPLICAS}}": str(mp_config["autoscaling"]["max_replicas"]),
4447 "{{MP_HPA_CPU_TARGET_UTILIZATION}}": str(
4448 mp_config["autoscaling"]["cpu_target_utilization_percentage"]
4449 ),
4450 }
4451 )
4453 # MLflow (on by default, requires observability): gate the server and
4454 # client NetworkPolicies (post-helm-mlflow-network.yaml) and the HTTPS
4455 # front door with its trust bundle (post-helm-mlflow-tls.yaml) on the
4456 # toggle via the same unreplaced-placeholder mechanism; a disabled
4457 # deployment leaves the files unapplied and the applier prunes what
4458 # they created plus the chart-managed metadata claim helm uninstall
4459 # leaves behind (metadata is discarded, artifacts stay in S3).
4460 if self._mlflow_active():
4461 image_replacements.update({"{{MLFLOW_ENABLED}}": "true"})
4463 # Add queue processor replacements if enabled
4464 qp_config = self.node.try_get_context("queue_processor") or {}
4466 # In-VPC ranges job pods may reach on any port (03-network-policies.yaml
4467 # allow-vpc-egress) and the MLflow server admits probes from
4468 # (post-helm-mlflow-network.yaml). Generates a YAML block of ipBlock
4469 # entries from the vpc_endpoint_cidrs array. The placeholder
4470 # {{VPC_ENDPOINT_CIDR_BLOCKS}} sits at 8-space indentation in the
4471 # manifest, so the first entry needs no leading indent (the manifest
4472 # provides it) and subsequent entries are indented to align.
4473 vpc_endpoint_cidrs = self.node.try_get_context("vpc_endpoint_cidrs") or ["10.0.0.0/16"]
4474 cidr_lines = []
4475 for i, cidr in enumerate(vpc_endpoint_cidrs):
4476 prefix = "" if i == 0 else " "
4477 cidr_lines.append(f'{prefix}- ipBlock:\n cidr: "{cidr}"')
4478 image_replacements["{{VPC_ENDPOINT_CIDR_BLOCKS}}"] = "\n".join(cidr_lines)
4480 # NetworkPolicy enforcement on EKS Auto Mode is a ConfigMap-driven
4481 # switch (06-network-policy-controller.yaml); render the operator's
4482 # choice as the literal the controller reads.
4483 image_replacements["{{NETWORK_POLICY_ENFORCEMENT}}"] = (
4484 "true"
4485 if self.config.get_eks_cluster_config().get("network_policy_enforcement", True)
4486 else "false"
4487 )
4489 # Resource governance for gco-jobs namespace: ResourceQuota caps aggregate
4490 # resource consumption across the namespace, LimitRange caps per-container
4491 # maxima. Values come from cdk.json `resource_quota` context merged
4492 # over gco.stacks.constants.DEFAULT_RESOURCE_QUOTA (per-container
4493 # maxima sized to one full accelerator-node slice) and validated at
4494 # synth: every value must parse as a Kubernetes quantity and the
4495 # container maxima must fit inside the namespace ceilings.
4496 resource_quota = _validated_resource_quota(
4497 self.node.try_get_context("resource_quota") or {}
4498 )
4499 image_replacements["{{QUOTA_MAX_CPU}}"] = resource_quota["max_cpu"]
4500 image_replacements["{{QUOTA_MAX_MEMORY}}"] = resource_quota["max_memory"]
4501 image_replacements["{{QUOTA_MAX_GPU}}"] = resource_quota["max_gpu"]
4502 image_replacements["{{QUOTA_MAX_PODS}}"] = resource_quota["max_pods"]
4503 image_replacements["{{LIMIT_MAX_CPU}}"] = resource_quota["container_max_cpu"]
4504 image_replacements["{{LIMIT_MAX_MEMORY}}"] = resource_quota["container_max_memory"]
4505 image_replacements["{{LIMIT_MAX_GPU}}"] = resource_quota["container_max_gpu"]
4507 if self.queue_processor_enabled:
4508 image_replacements["{{QUEUE_PROCESSOR_IMAGE}}"] = self.queue_processor_image.image_uri
4509 image_replacements["{{QP_POLLING_INTERVAL}}"] = str(
4510 qp_config.get("polling_interval", 10)
4511 )
4512 image_replacements["{{QP_MAX_CONCURRENT_JOBS}}"] = str(
4513 qp_config.get("max_concurrent_jobs", 10)
4514 )
4515 image_replacements["{{QP_MESSAGES_PER_JOB}}"] = str(
4516 qp_config.get("messages_per_job", 1)
4517 )
4518 image_replacements["{{QP_SUCCESSFUL_JOBS_HISTORY}}"] = str(
4519 qp_config.get("successful_jobs_history", 20)
4520 )
4521 image_replacements["{{QP_FAILED_JOBS_HISTORY}}"] = str(
4522 qp_config.get("failed_jobs_history", 10)
4523 )
4524 image_replacements["{{QP_ALLOWED_NAMESPACES}}"] = ",".join(
4525 job_policy.get("allowed_namespaces", ["gco-jobs"])
4526 )
4527 image_replacements["{{QP_ALLOWED_KINDS}}"] = ",".join(allowed_kinds)
4528 # Resource caps, image allowlist, and security policy are shared
4529 # with the REST manifest processor. Source them from the
4530 # job_validation_policy section so a single change in cdk.json
4531 # takes effect on both submission paths at the next deploy.
4532 image_replacements["{{QP_MAX_GPU_PER_MANIFEST}}"] = job_quotas["max_gpu_per_manifest"]
4533 image_replacements["{{QP_MAX_CPU_PER_MANIFEST}}"] = job_quotas["max_cpu_per_manifest"]
4534 image_replacements["{{QP_MAX_MEMORY_PER_MANIFEST}}"] = job_quotas[
4535 "max_memory_per_manifest"
4536 ]
4537 image_replacements["{{QP_TRUSTED_REGISTRIES}}"] = ",".join(
4538 _augment_trusted_registries_with_project_ecr(
4539 job_policy.get("trusted_registries", []),
4540 account=self.account,
4541 regions=self.config.get_regions(),
4542 global_region=self.config.get_global_region(),
4543 url_suffix=self.url_suffix,
4544 )
4545 )
4546 image_replacements["{{QP_TRUSTED_DOCKERHUB_ORGS}}"] = ",".join(
4547 job_policy.get("trusted_dockerhub_orgs", [])
4548 )
4550 # Security policy toggles — shared with the REST manifest_processor.
4551 # Both services read the same cdk.json section so a single policy
4552 # flip (e.g. block_run_as_root: true) takes effect on both paths.
4553 image_replacements["{{QP_BLOCK_PRIVILEGED}}"] = _policy_str(
4554 security_policy["block_privileged"]
4555 )
4556 image_replacements["{{QP_BLOCK_PRIVILEGE_ESCALATION}}"] = _policy_str(
4557 security_policy["block_privilege_escalation"]
4558 )
4559 image_replacements["{{QP_BLOCK_HOST_NETWORK}}"] = _policy_str(
4560 security_policy["block_host_network"]
4561 )
4562 image_replacements["{{QP_BLOCK_HOST_PID}}"] = _policy_str(
4563 security_policy["block_host_pid"]
4564 )
4565 image_replacements["{{QP_BLOCK_HOST_IPC}}"] = _policy_str(
4566 security_policy["block_host_ipc"]
4567 )
4568 image_replacements["{{QP_BLOCK_HOST_PATH}}"] = _policy_str(
4569 security_policy["block_host_path"]
4570 )
4571 image_replacements["{{QP_BLOCK_ADDED_CAPABILITIES}}"] = _policy_str(
4572 security_policy["block_added_capabilities"]
4573 )
4574 image_replacements["{{QP_BLOCK_RUN_AS_ROOT}}"] = _policy_str(
4575 security_policy["block_run_as_root"]
4576 )
4577 # Require accelerator (GPU/Neuron/EFA) jobs to carry a matching
4578 # toleration — shared with the REST manifest_processor via
4579 # {{MP_REQUIRE_ACCELERATOR_TOLERATION}}.
4580 image_replacements["{{QP_REQUIRE_ACCELERATOR_TOLERATION}}"] = (
4581 require_accelerator_toleration
4582 )
4584 # Add Valkey endpoint if enabled
4585 if hasattr(self, "valkey_cache") and self.valkey_cache:
4586 image_replacements["{{VALKEY_ENDPOINT}}"] = self.valkey_cache.attr_endpoint_address
4587 image_replacements["{{VALKEY_PORT}}"] = self.valkey_cache.attr_endpoint_port
4589 # Add Aurora pgvector endpoint if enabled
4590 if hasattr(self, "aurora_cluster") and self.aurora_cluster:
4591 image_replacements["{{AURORA_PGVECTOR_ENDPOINT}}"] = (
4592 self.aurora_cluster.cluster_endpoint.hostname
4593 )
4594 image_replacements["{{AURORA_PGVECTOR_READER_ENDPOINT}}"] = (
4595 self.aurora_cluster.cluster_read_endpoint.hostname
4596 )
4597 image_replacements["{{AURORA_PGVECTOR_PORT}}"] = str(
4598 self.aurora_cluster.cluster_endpoint.port
4599 )
4600 image_replacements["{{AURORA_PGVECTOR_SECRET_ARN}}"] = self.aurora_secret.secret_arn
4602 # Add vector-store discovery if enabled. Every value is deterministic
4603 # at synth time (the global stack names the table and index from the
4604 # same config), so no cross-stack reference is needed; the manifest's
4605 # {{REGION}} key points pods at their cluster's LOCAL global-table
4606 # replica. When disabled, the placeholders stay unreplaced and the
4607 # applier skips 26-storage-vector-store.yaml entirely.
4608 if self.config.get_vector_store_enabled():
4609 vector_store_config = self.config.get_vector_store_config()
4610 image_replacements["{{VECTOR_STORE_TABLE_NAME}}"] = (
4611 f"{self.config.get_project_name()}-vector-store"
4612 )
4613 image_replacements["{{VECTOR_STORE_INDEX_NAME}}"] = "corpus-embedding-index"
4614 image_replacements["{{VECTOR_STORE_EMBEDDING_MODEL_ID}}"] = str(
4615 vector_store_config["embedding_model_id"]
4616 )
4617 image_replacements["{{VECTOR_STORE_DIMENSIONS}}"] = str(
4618 vector_store_config["dimensions"]
4619 )
4621 # Add FSx replacements if enabled
4622 if self.fsx_file_system:
4623 image_replacements["{{FSX_FILE_SYSTEM_ID}}"] = self.fsx_file_system.ref
4624 image_replacements["{{FSX_DNS_NAME}}"] = self.fsx_file_system.attr_dns_name
4625 image_replacements["{{FSX_MOUNT_NAME}}"] = self.fsx_file_system.attr_lustre_mount_name
4626 image_replacements["{{PRIVATE_SUBNET_ID}}"] = self.vpc.private_subnets[0].subnet_id
4627 image_replacements["{{FSX_SECURITY_GROUP_ID}}"] = (
4628 self.fsx_security_group.security_group_id
4629 )
4631 # kro capability tenant RBAC (07-kro-tenant-access.yaml). The token is
4632 # emitted only when kro is enabled for this region; a disabled
4633 # capability leaves it unreplaced so the applier skips the file and
4634 # prunes the registered inventory. The capability acts as the
4635 # assumed-role session <role>/KRO, which is the RBAC subject.
4636 kro_role = getattr(self, "eks_capability_roles", {}).get("kro")
4637 image_replacements.update(
4638 compute_eks_capabilities_replacements(
4639 kro_username=(
4640 kro_kubernetes_username(
4641 partition=self.partition,
4642 account=self.account,
4643 role_name=kro_role.role_name,
4644 )
4645 if kro_role is not None
4646 else None
4647 )
4648 )
4649 )
4651 # ── Trigger the convergence pipeline (fire-and-forget) ───────────────
4652 # A single custom resource starts the HelmInstallStateMachine, which now
4653 # owns the WHOLE cluster convergence: apply base manifests -> install
4654 # Helm charts -> apply post-Helm (CRD-dependent) manifests -> publish
4655 # the Gateway-created ALB and optionally register it with Global
4656 # Accelerator. The resource returns
4657 # as soon as the execution is *started* (no isComplete waiter), so the
4658 # cluster's CloudFormation lifecycle is never bound to the multi-minute
4659 # add-on convergence — a slow chart can't blow CloudFormation's ~1h
4660 # custom-resource ceiling and roll back (destroy) the freshly-created
4661 # cluster. Status lives in SSM and is surfaced via `gco stacks addons
4662 # status`; re-converge out-of-band with `gco stacks addons install`.
4663 #
4664 # The execution input carries everything the state-machine tasks need:
4665 # chart selection/overrides, the manifest ImageReplacements (for the base
4666 # and post-Helm kubectl passes), the endpoint-registry identity, and the
4667 # optional Global Accelerator EndpointGroupArn.
4668 convergence_properties: dict[str, Any] = {
4669 "ClusterName": self.cluster.cluster_name,
4670 "Region": self.deployment_region,
4671 # Helm chart selection + per-chart value overrides (e.g. Volcano
4672 # image_registry redirected to the ECR mirror when enabled), the
4673 # overrides as one JSON string so their types survive CloudFormation.
4674 "EnabledCharts": self._get_enabled_helm_charts(),
4675 "Charts": self._helm_chart_value_overrides_json(),
4676 "KedaOperatorRoleArn": self.keda_operator_role.role_arn,
4677 # Template substitutions for the base + post-Helm kubectl passes.
4678 "ImageReplacements": image_replacements,
4679 # Project name lets the orchestrator persist the execution input
4680 # to SSM so `gco stacks addons install` can replay the whole
4681 # pipeline without reconstructing chart/manifest config.
4682 "ProjectName": self.config.get_project_name(),
4683 "RegistryRegion": self.config.get_global_region(),
4684 # Force re-invocation on every deployment (new charts.yaml,
4685 # manifest, or image) so convergence re-runs end to end.
4686 "DeploymentTimestamp": deployment_timestamp,
4687 }
4688 if self.global_accelerator_enabled:
4689 convergence_properties["EndpointGroupArn"] = self.endpoint_group_arn
4691 converge_trigger = CustomResource(
4692 self,
4693 "HelmInstallCharts",
4694 service_token=self.helm_installer_provider.service_token,
4695 properties=convergence_properties,
4696 )
4697 converge_trigger.node.add_dependency(self.helm_installer_provider_log_group)
4698 converge_trigger.node.add_dependency(self.aws_load_balancer_controller_policy)
4700 # The trigger (and therefore the whole convergence pipeline) must run
4701 # after the cluster, shared storage, managed-addon IRSA patches, and Pod
4702 # Identity associations exist: the base manifests reference their tokens,
4703 # and the rollout-restarts at the end of the base pass need the patched
4704 # service accounts (otherwise the mutating webhook can't inject
4705 # AWS_ROLE_ARN and the controllers fail with "no EC2 IMDS role found" —
4706 # PVCs stuck Pending, missing Container Insights metrics; see the
4707 # UpdateEfsCsiAddonRole resource in _create_efs_csi_driver_addon). These
4708 # gates previously sat on the synchronous KubectlApplyManifests custom
4709 # resource; the base apply now lives in the state machine, so the gate
4710 # moves to the trigger.
4711 converge_trigger.node.add_dependency(self.cluster)
4712 converge_trigger.node.add_dependency(self.efs_file_system)
4713 if self.fsx_file_system:
4714 converge_trigger.node.add_dependency(self.fsx_file_system)
4715 for attr in (
4716 "_efs_csi_addon_role_update",
4717 "_fsx_csi_addon_role_update",
4718 "_cloudwatch_addon_role_update",
4719 ):
4720 update_cr = getattr(self, attr, None)
4721 if update_cr is not None:
4722 converge_trigger.node.add_dependency(update_cr)
4723 # The trigger also needs both EKS access entries before it starts the
4724 # asynchronous pipeline. Keeping these explicit is essential on delete:
4725 # the ordered Helm teardown runs while its Kubernetes authentication is
4726 # still valid, then the trigger/access entries/cluster can disappear.
4727 for attr in (
4728 "kubectl_lambda_access_entry",
4729 "helm_installer_access_entry",
4730 "ga_registration_access_entry",
4731 ):
4732 access_entry = getattr(self, attr, None)
4733 if access_entry is not None:
4734 converge_trigger.node.add_dependency(access_entry)
4735 for assoc in self._pod_identity_associations:
4736 converge_trigger.node.add_dependency(assoc)
4738 # Deletion must run in the opposite safety order: synchronous Helm
4739 # teardown first (quiescing endpoint writers and removing Gateway
4740 # resources), then the unconditional endpoint deregistration guard,
4741 # then the convergence trigger and its EKS access entries. Build the
4742 # create-time chain as trigger -> endpoint guard -> Helm teardown so
4743 # CloudFormation reverses it during stack deletion.
4744 helm_teardown = getattr(self, "helm_teardown_resource", None)
4745 ga_deregistration = getattr(self, "ga_deregistration_resource", None)
4746 if helm_teardown is not None:
4747 if ga_deregistration is not None:
4748 helm_teardown.node.add_dependency(ga_deregistration)
4749 ga_deregistration.node.add_dependency(converge_trigger)
4750 else:
4751 helm_teardown.node.add_dependency(converge_trigger)
4752 elif ga_deregistration is not None:
4753 ga_deregistration.node.add_dependency(converge_trigger)
4755 def _create_ga_registration_lambda(self) -> None:
4756 """Create the exact Gateway ALB discovery and endpoint-publication Lambda.
4758 Every partition uses this function to discover ``gco-system/gco-gateway``
4759 and publish its internal ALB hostname to the regional SSM registry.
4760 Commercial partitions additionally pass an endpoint-group ARN so the
4761 same exact ALB is registered with Global Accelerator.
4762 """
4763 project_name = self.config.get_project_name()
4765 # Create Lambda function for GA registration using external handler
4766 ga_registration_lambda = lambda_.Function(
4767 self,
4768 "GaRegistrationFunction",
4769 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
4770 handler="handler.lambda_handler",
4771 code=lambda_source_code("lambda/ga-registration"),
4772 timeout=Duration.minutes(15), # Max Lambda timeout; handler uses 14 min budget
4773 memory_size=256,
4774 vpc=self.vpc,
4775 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
4776 environment={
4777 "CLUSTER_NAME": self.cluster.cluster_name,
4778 "REGION": self.deployment_region,
4779 },
4780 tracing=lambda_.Tracing.ACTIVE,
4781 )
4783 # Grant permissions
4784 ga_registration_lambda.add_to_role_policy(
4785 iam.PolicyStatement(
4786 effect=iam.Effect.ALLOW,
4787 actions=["eks:DescribeCluster"],
4788 resources=[self.cluster.cluster_arn],
4789 )
4790 )
4791 ga_registration_lambda.add_to_role_policy(
4792 iam.PolicyStatement(
4793 effect=iam.Effect.ALLOW,
4794 actions=[
4795 "elasticloadbalancing:DescribeLoadBalancers",
4796 "elasticloadbalancing:DescribeTags", # Required for tag-based ALB detection
4797 ],
4798 resources=["*"],
4799 )
4800 )
4801 if self.global_accelerator_enabled:
4802 ga_registration_lambda.add_to_role_policy(
4803 iam.PolicyStatement(
4804 effect=iam.Effect.ALLOW,
4805 actions=[
4806 "globalaccelerator:AddEndpoints",
4807 "globalaccelerator:RemoveEndpoints",
4808 "globalaccelerator:UpdateEndpointGroup",
4809 "globalaccelerator:DescribeEndpointGroup",
4810 # The teardown-time cleanup_gateway_endpoint task runs
4811 # on this Lambda and strictly waits for the accelerator
4812 # to reach DEPLOYED after endpoint removal.
4813 "globalaccelerator:DescribeAccelerator",
4814 ],
4815 resources=["*"],
4816 )
4817 )
4818 ga_registration_lambda.add_to_role_policy(
4819 iam.PolicyStatement(
4820 effect=iam.Effect.ALLOW,
4821 actions=["ssm:GetParameter", "ssm:PutParameter", "ssm:DeleteParameter"],
4822 resources=[
4823 f"arn:{self.partition}:ssm:{self.config.get_global_region()}:"
4824 f"{self.account}:parameter/{project_name}/*"
4825 ],
4826 )
4827 )
4829 # Retain the access entry so asynchronous convergence cannot start
4830 # until the endpoint publisher can read the exact Gateway object.
4831 ga_registration_role = cast(iam.IRole, ga_registration_lambda.role)
4832 self.ga_registration_access_entry = eks.AccessEntry(
4833 self,
4834 "GaRegistrationLambdaAccessEntry",
4835 cluster=self.cluster, # type: ignore[arg-type]
4836 principal=ga_registration_role.role_arn,
4837 access_policies=[
4838 eks.AccessPolicy.from_access_policy_name(
4839 "AmazonEKSClusterAdminPolicy", access_scope_type=eks.AccessScopeType.CLUSTER
4840 )
4841 ],
4842 )
4844 # Allow Lambda to access EKS API
4845 self.cluster.cluster_security_group.add_ingress_rule(
4846 peer=ec2.Peer.ipv4(self.vpc.vpc_cidr_block),
4847 connection=ec2.Port.tcp(443),
4848 description="Allow GA registration Lambda to access EKS API",
4849 )
4851 # Global Accelerator exists only in supported partitions. Resolve its
4852 # endpoint group lazily there; regional endpoint publication itself is
4853 # unconditional and needs no GA lookup.
4854 self.endpoint_group_arn: str | None = None
4855 if self.global_accelerator_enabled:
4856 global_region = self.config.get_global_region()
4857 get_endpoint_group_arn = cr.AwsCustomResource(
4858 self,
4859 "GetEndpointGroupArn",
4860 on_create=cr.AwsSdkCall(
4861 service="SSM",
4862 action="getParameter",
4863 parameters={
4864 "Name": f"/{project_name}/endpoint-group-{self.deployment_region}-arn"
4865 },
4866 region=global_region,
4867 physical_resource_id=cr.PhysicalResourceId.of(
4868 f"{project_name}-get-endpoint-group-arn-{self.deployment_region}"
4869 ),
4870 ),
4871 on_update=cr.AwsSdkCall(
4872 service="SSM",
4873 action="getParameter",
4874 parameters={
4875 "Name": f"/{project_name}/endpoint-group-{self.deployment_region}-arn"
4876 },
4877 region=global_region,
4878 ),
4879 role=self.aws_custom_resource_role,
4880 )
4881 get_endpoint_group_arn.node.add_dependency(self.aws_custom_resource_role)
4882 self.endpoint_group_arn = get_endpoint_group_arn.get_response_field("Parameter.Value")
4884 # Invoked directly by the convergence state machine's final task.
4885 self.ga_registration_lambda = ga_registration_lambda
4887 # cdk-nag suppression: the GA registration Lambda needs broad
4888 # Global Accelerator and ELB Describe access with Resource: *.
4889 from gco.stacks.nag_suppressions import acknowledge_nag_findings
4891 acknowledge_nag_findings(
4892 ga_registration_lambda,
4893 [
4894 {
4895 "id": "AwsSolutions-IAM5",
4896 "reason": (
4897 "The endpoint-publication Lambda needs ELB Describe access to "
4898 "resolve the exact Gateway-owned ALB. In partitions with Global "
4899 "Accelerator it also needs the service's endpoint-group mutation "
4900 "APIs. These APIs do not support resource-level scoping."
4901 ),
4902 "appliesTo": ["Resource::*"],
4903 },
4904 ],
4905 )
4907 # Wire the delete-time teardown guard that deregisters this region's ALB
4908 # from Global Accelerator before its VPC subnets are deleted.
4909 self._create_ga_deregistration_resource()
4911 def _create_ga_deregistration_resource(self) -> None:
4912 """Create the unconditional endpoint-registry delete guard.
4914 On stack deletion the guard always removes this region's SSM hostname.
4915 When an endpoint group exists it first deregisters the ALB and waits for
4916 Global Accelerator to release its managed ENIs. The resource therefore
4917 exists in every partition even though the GA portion is optional.
4918 """
4919 project_name = self.config.get_project_name()
4921 # Dedicated Lambda built from the SAME asset as the registration Lambda
4922 # (it reuses the shared remove/wait helpers, entry point
4923 # handler.on_delete_event). Deliberately NOT in the VPC: it only calls
4924 # the public Global Accelerator API and must not create its own ENIs in
4925 # the VPC it is helping to tear down.
4926 ga_deregistration_lambda = lambda_.Function(
4927 self,
4928 "GaDeregistrationFunction",
4929 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
4930 handler="handler.on_delete_event",
4931 code=lambda_source_code("lambda/ga-registration"),
4932 timeout=Duration.minutes(15), # covers the GA redeploy wait budget
4933 memory_size=256,
4934 tracing=lambda_.Tracing.ACTIVE,
4935 )
4936 if self.global_accelerator_enabled:
4937 ga_deregistration_lambda.add_to_role_policy(
4938 iam.PolicyStatement(
4939 effect=iam.Effect.ALLOW,
4940 actions=[
4941 "globalaccelerator:DescribeAccelerator",
4942 "globalaccelerator:DescribeEndpointGroup",
4943 "globalaccelerator:RemoveEndpoints",
4944 "globalaccelerator:UpdateEndpointGroup",
4945 ],
4946 resources=["*"],
4947 )
4948 )
4950 ga_deregistration_lambda.add_to_role_policy(
4951 iam.PolicyStatement(
4952 effect=iam.Effect.ALLOW,
4953 actions=["ssm:DeleteParameter"],
4954 resources=[
4955 f"arn:{self.partition}:ssm:{self.config.get_global_region()}:{self.account}:"
4956 f"parameter/{project_name}/alb-hostname-{self.deployment_region}"
4957 ],
4958 )
4959 )
4961 # Strict live validation retains the exact generation through the
4962 # provider's final delete invocation; its identity-fenced post-stack
4963 # cleanup removes it. Ordinary deployments retain DESTROY semantics.
4964 ga_deregistration_log_group = logs.LogGroup(
4965 self,
4966 "GaDeregistrationProviderLogGroup",
4967 retention=logs.RetentionDays.ONE_WEEK,
4968 removal_policy=self.provider_log_group_removal_policy,
4969 )
4970 ga_deregistration_provider = cr.Provider(
4971 self,
4972 "GaDeregistrationProvider",
4973 on_event_handler=ga_deregistration_lambda,
4974 log_group=ga_deregistration_log_group,
4975 )
4977 deregistration_properties: dict[str, Any] = {
4978 "Region": self.deployment_region,
4979 "RegistryRegion": self.config.get_global_region(),
4980 "ProjectName": project_name,
4981 }
4982 if self.endpoint_group_arn is not None:
4983 deregistration_properties["EndpointGroupArn"] = self.endpoint_group_arn
4985 ga_deregistration = CustomResource(
4986 self,
4987 "GaDeregistration",
4988 service_token=ga_deregistration_provider.service_token,
4989 properties=deregistration_properties,
4990 )
4992 # Teardown ordering: this deregistration must run BEFORE the VPC (and its
4993 # public subnets, where Global Accelerator pins its managed ENIs) is
4994 # deleted. Depending on the VPC means CloudFormation creates the VPC
4995 # first and — critically — deletes this custom resource first on
4996 # teardown, releasing the GA ENIs so the subnets can be removed cleanly.
4997 self.ga_deregistration_resource = ga_deregistration
4998 ga_deregistration.node.add_dependency(self.vpc)
4999 ga_deregistration.node.add_dependency(ga_deregistration_log_group)
5001 # cdk-nag: the deregistration Lambda needs globalaccelerator Describe*/
5002 # RemoveEndpoints with Resource: * (these Global Accelerator APIs do not
5003 # support resource-level IAM scoping), mirroring the registration Lambda.
5004 from gco.stacks.nag_suppressions import acknowledge_nag_findings
5006 acknowledge_nag_findings(
5007 ga_deregistration_lambda,
5008 [
5009 {
5010 "id": "AwsSolutions-IAM5",
5011 "reason": (
5012 "Where Global Accelerator is enabled, the delete guard needs its "
5013 "Describe and endpoint-group mutation APIs to release managed ENIs. "
5014 "Those APIs do not support resource-level IAM scoping; non-GA "
5015 "partitions receive no Global Accelerator actions."
5016 ),
5017 "appliesTo": ["Resource::*"],
5018 },
5019 ],
5020 )
5022 def _create_transaction_search_resource(self) -> None:
5023 """Enable CloudWatch Transaction Search in this Region, never disable it.
5025 The traced API services export OpenTelemetry spans to the X-Ray OTLP
5026 endpoint, which accepts spans only once Transaction Search is on. The
5027 setting is account-level (configured per Region) and shared with any
5028 other traced workload, so the custom resource
5029 (``lambda/transaction-search``) is one-directional: Create/Update
5030 switch the trace segment destination to CloudWatch Logs only when it
5031 is not already there, and Delete is a no-op that leaves the setting
5032 and its resource policy in place. Skipped unless both
5033 ``tracing.enabled`` and ``tracing.enable_transaction_search`` hold.
5035 Mirrors the GA deregistration provider: a plain zip Lambda outside
5036 the VPC (it only calls public AWS APIs and must not create ENIs), a
5037 ``cr.Provider`` whose explicit log group honours the live-validation
5038 retention context, and scoped cdk-nag acknowledgments. The role
5039 carries the AWS-documented Transaction Search prerequisites minus the
5040 indexing-rule APIs GCO never calls.
5041 """
5042 if not self._transaction_search_active():
5043 return
5045 region = self.deployment_region
5046 partition = self.partition
5047 account = self.account
5048 span_log_group_arns = [
5049 f"arn:{partition}:logs:{region}:{account}:log-group:aws/spans:*",
5050 f"arn:{partition}:logs:{region}:{account}:log-group:/aws/application-signals/data:*",
5051 ]
5052 application_signals_role_arn = (
5053 f"arn:{partition}:iam::{account}:role/aws-service-role/"
5054 "application-signals.cloudwatch.amazonaws.com/"
5055 "AWSServiceRoleForCloudWatchApplicationSignals"
5056 )
5057 application_signals_channel_arn = (
5058 f"arn:{partition}:cloudtrail:{region}:{account}:"
5059 "channel/aws-service-channel/application-signals/*"
5060 )
5062 transaction_search_lambda = lambda_.Function(
5063 self,
5064 "TransactionSearchFunction",
5065 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
5066 handler="handler.lambda_handler",
5067 code=lambda_source_code("lambda/transaction-search"),
5068 timeout=Duration.minutes(5), # three API calls plus throttling retries
5069 memory_size=256,
5070 tracing=lambda_.Tracing.ACTIVE,
5071 )
5072 # Read and switch the trace segment destination, and write the
5073 # X-Ray -> CloudWatch Logs resource policy. None of these APIs (nor
5074 # Application Signals discovery) support resource-level scoping.
5075 transaction_search_lambda.add_to_role_policy(
5076 iam.PolicyStatement(
5077 effect=iam.Effect.ALLOW,
5078 actions=[
5079 "xray:GetTraceSegmentDestination",
5080 "xray:UpdateTraceSegmentDestination",
5081 "logs:PutResourcePolicy",
5082 "logs:DescribeResourcePolicies",
5083 "application-signals:StartDiscovery",
5084 ],
5085 resources=["*"],
5086 )
5087 )
5088 # The two Transaction Search log groups in this Region only.
5089 transaction_search_lambda.add_to_role_policy(
5090 iam.PolicyStatement(
5091 effect=iam.Effect.ALLOW,
5092 actions=[
5093 "logs:CreateLogGroup",
5094 "logs:CreateLogStream",
5095 "logs:PutRetentionPolicy",
5096 ],
5097 resources=span_log_group_arns,
5098 )
5099 )
5100 # Application Signals' service-linked role and CloudTrail channel.
5101 transaction_search_lambda.add_to_role_policy(
5102 iam.PolicyStatement(
5103 effect=iam.Effect.ALLOW,
5104 actions=["iam:CreateServiceLinkedRole"],
5105 resources=[application_signals_role_arn],
5106 conditions={
5107 "StringEquals": {
5108 "iam:AWSServiceName": "application-signals.cloudwatch.amazonaws.com"
5109 }
5110 },
5111 )
5112 )
5113 transaction_search_lambda.add_to_role_policy(
5114 iam.PolicyStatement(
5115 effect=iam.Effect.ALLOW,
5116 actions=["iam:GetRole"],
5117 resources=[application_signals_role_arn],
5118 )
5119 )
5120 transaction_search_lambda.add_to_role_policy(
5121 iam.PolicyStatement(
5122 effect=iam.Effect.ALLOW,
5123 actions=["cloudtrail:CreateServiceLinkedChannel"],
5124 resources=[application_signals_channel_arn],
5125 )
5126 )
5128 # Strict live validation retains the provider's log group through its
5129 # final delete invocation, exactly like the GA deregistration guard.
5130 transaction_search_log_group = logs.LogGroup(
5131 self,
5132 "TransactionSearchProviderLogGroup",
5133 retention=logs.RetentionDays.ONE_WEEK,
5134 removal_policy=self.provider_log_group_removal_policy,
5135 )
5136 transaction_search_provider = cr.Provider(
5137 self,
5138 "TransactionSearchProvider",
5139 on_event_handler=transaction_search_lambda,
5140 log_group=transaction_search_log_group,
5141 )
5142 # Properties change only with the deployment identity, so the
5143 # resource re-runs only when that changes: a failing Update during an
5144 # unrelated deploy could otherwise wedge the stack's rollback.
5145 transaction_search = CustomResource(
5146 self,
5147 "TransactionSearch",
5148 service_token=transaction_search_provider.service_token,
5149 properties={
5150 "Region": region,
5151 "AccountId": account,
5152 "Partition": partition,
5153 "ProjectName": self.config.get_project_name(),
5154 },
5155 )
5156 transaction_search.node.add_dependency(transaction_search_log_group)
5157 self.transaction_search_resource = transaction_search
5159 from gco.stacks.nag_suppressions import acknowledge_nag_findings
5161 acknowledge_nag_findings(
5162 transaction_search_lambda,
5163 [
5164 {
5165 "id": "AwsSolutions-IAM5",
5166 "reason": (
5167 "The Transaction Search handler reads and switches the X-Ray "
5168 "trace segment destination, writes the X-Ray -> CloudWatch Logs "
5169 "resource policy and starts Application Signals discovery; none "
5170 "of these APIs, nor the X-Ray writes of Lambda active tracing, "
5171 "support resource-level permissions."
5172 ),
5173 "appliesTo": ["Resource::*"],
5174 },
5175 {
5176 "id": "AwsSolutions-IAM5",
5177 "reason": (
5178 "Log-group creation and retention are scoped to the two "
5179 "Transaction Search log groups of this Region; the ':*' suffix "
5180 "covers their log streams, and the CloudTrail grant is scoped to "
5181 "Application Signals' service-linked channels in this Region."
5182 ),
5183 "appliesTo": [
5184 *(f"Resource::{arn}" for arn in span_log_group_arns),
5185 f"Resource::{application_signals_channel_arn}",
5186 ],
5187 },
5188 ],
5189 )
5190 function_logical_id = self.get_logical_id(
5191 cast(lambda_.CfnFunction, transaction_search_lambda.node.default_child)
5192 )
5193 acknowledge_nag_findings(
5194 transaction_search_provider,
5195 [
5196 {
5197 "id": "AwsSolutions-IAM5",
5198 "reason": (
5199 "The cr.Provider framework role invokes the Transaction Search "
5200 "handler through the standard '<lambda-arn>:*' version "
5201 "qualifier, which cannot be narrowed at synth time."
5202 ),
5203 "appliesTo": [f"Resource::<{function_logical_id}.Arn>:*"],
5204 },
5205 ],
5206 )
5208 def _get_volcano_image_mirror_config(self) -> dict[str, Any]:
5209 """Parse the ``volcano_image_mirror`` block from cdk.json.
5211 Returns a normalized dict ``{enabled, ecr_namespace}``. Validation is
5212 strict so a misconfiguration fails at synth rather than silently leaving
5213 Volcano pointed at docker.io.
5215 - ``enabled`` (default False) — master toggle.
5216 - ``ecr_namespace`` (default ``"<project_name>/dockerhub"``, i.e.
5217 ``gco/dockerhub`` for the stock project) — the ECR repository
5218 namespace the mirrored Volcano images live under. Must start with
5219 ``<project_name>/`` so it inherits the project's existing
5220 ``<project_name>/*`` machinery (node pull access, replication rule,
5221 trusted-registry allow-list) with no extra IAM, and must be a valid
5222 (possibly nested) ECR repository path.
5223 """
5224 raw = self.node.try_get_context("volcano_image_mirror") or {}
5225 if not isinstance(raw, dict):
5226 raise ValueError(f"volcano_image_mirror must be a mapping, got {type(raw).__name__}")
5228 # The mirror namespace lives under this deployment's project prefix
5229 # (``<project_name>/``) so it inherits the project's ECR access,
5230 # replication rule, and trusted-registry allow-list (#139). Defaults to
5231 # ``<project_name>/dockerhub`` — ``gco/dockerhub`` for the stock project.
5232 project_prefix = f"{self.config.get_project_name()}/"
5233 enabled = bool(raw.get("enabled", False))
5234 ecr_namespace = (
5235 str(raw.get("ecr_namespace", f"{project_prefix}dockerhub")).strip().strip("/")
5236 )
5238 if not enabled:
5239 return {"enabled": False, "ecr_namespace": ecr_namespace}
5241 # Must live under the project prefix and be a valid nested ECR repo
5242 # path (lowercase alphanumerics + . _ - per segment, slash-separated).
5243 if not ecr_namespace.startswith(project_prefix):
5244 raise ValueError(
5245 f"volcano_image_mirror.ecr_namespace must start with {project_prefix!r} so it "
5246 f"inherits the project's {project_prefix}* ECR access/replication, got "
5247 f"{ecr_namespace!r}"
5248 )
5249 segment = r"[a-z0-9]+(?:[._-][a-z0-9]+)*"
5250 if not re.fullmatch(rf"{segment}(?:/{segment})+", ecr_namespace):
5251 raise ValueError(
5252 "volcano_image_mirror.ecr_namespace must be a valid ECR repository "
5253 f"path (lowercase alphanumerics + . _ - per slash-separated segment), "
5254 f"got {ecr_namespace!r}"
5255 )
5257 return {"enabled": True, "ecr_namespace": ecr_namespace}
5259 def _configure_volcano_image_mirror(self) -> None:
5260 """Resolve the optional Volcano image-mirror registry (no infra).
5262 Volcano is the only default chart whose images live exclusively on
5263 docker.io (``volcanosh/vc-*``). On a cold EKS Auto Mode cluster those
5264 anonymous pulls are slow / rate-limited, so Volcano's blocking
5265 ``helm --wait`` could never finish inside the installer Lambda's
5266 wall-clock guard and the whole add-on batch looped on it.
5268 The fix is to mirror Volcano's pinned images into the project's own ECR
5269 under ``gco/*`` and point Volcano's ``basic.image_registry`` there, so
5270 the cluster makes fast, same-account ECR pulls with the pull-only node
5271 role it already has — no Docker Hub credential, no pull-through cache
5272 rule, and no registry permissions policy. The mirror itself is populated
5273 out-of-band (``gco images mirror``) before the add-ons
5274 converge; this method only computes the registry override and creates no
5275 CloudFormation resources.
5277 Sets ``self.volcano_mirror_registry`` to
5278 ``<account>.dkr.ecr.<region>.<url-suffix>/<ecr_namespace>`` that
5279 ``_helm_chart_value_overrides`` feeds into Volcano's
5280 ``basic.image_registry``; left ``None`` when disabled.
5281 """
5282 # Always define the attribute so downstream code can branch on it.
5283 self.volcano_mirror_registry: str | None = None
5285 cfg = self._get_volcano_image_mirror_config()
5286 if not cfg["enabled"]:
5287 return
5289 ecr_namespace = cfg["ecr_namespace"]
5290 self.volcano_mirror_registry = (
5291 f"{self.account}.dkr.ecr.{self.deployment_region}.{self.url_suffix}/{ecr_namespace}"
5292 )
5294 def _helm_chart_value_overrides_json(self) -> str:
5295 """The chart value overrides as the JSON string both Helm custom resources carry.
5297 CloudFormation hands a custom resource's properties to its provider
5298 with every number and boolean turned into a string. Values a chart
5299 renders with ``toYaml`` then reach the API server with the wrong type:
5300 the TLS sidecars' Secret volume (``optional: "true"``,
5301 ``defaultMode: "292"``) and OpenCost's sidecar container (ports,
5302 probes, ``runAsNonRoot``) fail server-side apply, and a ``false``
5303 toggle becomes the truthy string ``"false"``. One JSON string keeps
5304 every override's type; CloudFormation still resolves the tokens inside
5305 it, and the Helm orchestrator and teardown provider decode it before
5306 building their execution input.
5307 """
5308 return str(self.to_json_string(self._helm_chart_value_overrides()))
5310 def _helm_chart_value_overrides(self) -> dict[str, Any]:
5311 """Per-chart helm value overrides injected into the install payload.
5313 Returned dict is JSON-encoded (``_helm_chart_value_overrides_json``)
5314 into the ``Charts`` property of the ``HelmInstallCharts`` and
5315 ``HelmTeardown`` custom resources; the installer deep-merges each
5316 chart's ``values`` over ``charts.yaml``. The mandatory
5317 ``aws-load-balancer-controller`` chart always receives the cluster,
5318 region, VPC, and dedicated IRSA role values. Optional overrides are:
5320 - ``volcano``: point ``basic.image_registry`` at the project's ECR
5321 image mirror when enabled, so every Volcano image (controller,
5322 scheduler, admission webhook, and the pre-install admission-init
5323 hook) resolves from ECR instead of docker.io. The upstream names
5324 (``volcanosh/vc-*``) are preserved, so each resolves to
5325 ``<mirror_registry>/volcanosh/vc-*``.
5326 - ``kube-prometheus-stack``: inject the ``cdk.json``-derived dynamic
5327 values (Grafana/Prometheus/Alertmanager persistence sizes, Prometheus
5328 retention, the gp3 ``storageClassName``, the GPU/Neuron/EFA
5329 node-exporter tolerations, and Grafana's TLS sidecar built from the
5330 manifest-processor image) over the static hardening values in
5331 ``charts.yaml`` when ``cluster_observability.enabled`` is true.
5332 - ``opencost``: the cluster identity and the OpenCost TLS sidecar
5333 built from the cost-monitor image, while cost monitoring deploys.
5334 - ``mlflow``: the artifact destination, IRSA role, claim size, the
5335 complete host allow-list and the MLflow TLS sidecar built from the
5336 manifest-processor image, while MLflow deploys
5337 (``_mlflow_chart_values``).
5338 - ``argocd``: the repo-server size and optional CPU autoscaler from
5339 ``helm.argocd.repo_server`` (``gco.argocd_config.argocd_chart_values``)
5340 whenever the chart installs, by the cdk.json toggle or a run-scoped
5341 ``helm_enabled_overrides``.
5343 The result is never empty because Gateway API requires the controller.
5344 """
5345 overrides: dict[str, Any] = {
5346 "aws-load-balancer-controller": {
5347 "values": {
5348 "clusterName": self.cluster.cluster_name,
5349 "region": self.deployment_region,
5350 "vpcId": self.vpc.vpc_id,
5351 "serviceAccount": {
5352 "annotations": {
5353 "eks.amazonaws.com/role-arn": (
5354 self.aws_load_balancer_controller_role.role_arn
5355 )
5356 }
5357 },
5358 }
5359 }
5360 }
5362 if getattr(self, "volcano_mirror_registry", None):
5363 overrides["volcano"] = {
5364 "values": {
5365 "basic": {
5366 "image_registry": self.volcano_mirror_registry,
5367 }
5368 }
5369 }
5371 if self.config.get_cluster_observability_enabled():
5372 overrides["kube-prometheus-stack"] = self._observability_chart_values()
5374 if self._cost_monitoring_active():
5375 overrides["opencost"] = self._opencost_chart_values()
5377 if self._mlflow_active():
5378 overrides["mlflow"] = self._mlflow_chart_values()
5380 # The same enablement helper that selects the chart, so a run-scoped
5381 # helm_enabled_overrides=argocd install gets the configured sizing too.
5382 helm_config = self.node.try_get_context("helm") or {}
5383 enabled_overrides = _parse_helm_enabled_overrides(
5384 self.node.try_get_context(_HELM_OVERRIDE_CONTEXT_KEY)
5385 )
5386 if _helm_chart_enabled(helm_config, enabled_overrides, "argocd"):
5387 overrides["argocd"] = {"values": argocd_chart_values(self._argocd_config())}
5389 # The Slurm cluster: every deployment-specific value (partitions as
5390 # NodeSets, accounting, storage, metrics, login, the ScaledObjects) is
5391 # rendered from helm.slurm; charts.yaml keeps only pins and sizing.
5392 # The FSx scratch mount follows this region's own fsx_lustre toggle.
5393 if _helm_chart_enabled(helm_config, enabled_overrides, "slurm"):
5394 overrides[SLURM_CLUSTER_CHART_NAME] = {
5395 "values": slurm_chart_values(
5396 self._slurm_config(), fsx_enabled=self._slurm_fsx_enabled()
5397 )
5398 }
5400 return overrides
5402 def _cost_monitoring_active(self) -> bool:
5403 """Return whether the per-region cost monitoring pipeline deploys here.
5405 Delegates to ``ConfigLoader.get_cost_monitoring_enabled``, which is
5406 already the conjunction of the ``cost_monitoring`` toggle and its
5407 ``cluster_observability`` data-source dependency — disabling either
5408 switches OpenCost, the cost-monitor service, and the cost dashboard
5409 off together.
5410 """
5411 return self.config.get_cost_monitoring_enabled()
5413 def _mlflow_active(self) -> bool:
5414 """Return whether the MLflow tracking server deploys on this cluster.
5416 Delegates to ``ConfigLoader.get_mlflow_enabled`` — the conjunction of
5417 ``cluster_observability.mlflow.enabled`` and observability itself,
5418 so the chart, its IRSA role, the gated backend PVC, and the value
5419 overrides all switch together.
5420 """
5421 return self.config.get_mlflow_enabled()
5423 def _mlflow_chart_values(self) -> dict[str, Any]:
5424 """Build the MLflow value overrides that carry deployment tokens.
5426 These things are dynamic — everything static (image pin, PVC
5427 wiring, resources, posture toggles) lives in ``charts.yaml``:
5429 - ``mlflow.artifactsDestination``: run artifacts go to the
5430 cluster-shared bucket under ``mlflow-artifacts/<region>/`` —
5431 region-suffixed because each regional tracking server numbers
5432 experiments independently, so a shared root would interleave
5433 unrelated runs' artifacts. The server proxies artifact traffic
5434 (``--serve-artifacts`` is the server default), so client pods
5435 never need S3 credentials of their own.
5436 - ``serviceAccount.annotations``: the IRSA role ARN, which is how
5437 the server-side artifact proxy gets its S3 credentials.
5438 - ``storage.size``: the metadata claim size from
5439 ``cluster_observability.mlflow.persistence_size``.
5440 - ``server.value_options.allowed_hosts``: the complete
5441 host-validation allow-list — service DNS plus wildcard patterns
5442 derived from ``vpc_endpoint_cidrs`` (see
5443 ``_mlflow_allowed_hosts``); the deep merge keeps the static
5444 ``workers`` value while replacing the charts.yaml DNS-only
5445 fallback with this full list.
5446 - ``extraContainers``/``extraVolumes``: the ``mlflow-tls-proxy``
5447 sidecar, from the manifest-processor image like Grafana's. It
5448 terminates verified TLS on 5443 (the ``mlflow-tls`` Service that
5449 gco-jobs clients call, trusting the ``gco-internal-ca`` bundle
5450 trust-manager publishes there) and forwards to the server on
5451 loopback 5000, with the ``mlflow-tls`` leaf mounted from an
5452 optional Secret volume. The server keeps its plaintext 5000 for
5453 the chart's probes, the ServiceMonitor and the tunnel. The chart
5454 renders both keys with ``toYaml``, so they are lists.
5455 - ``nodeSelector``: the service images are amd64 only, so the pod
5456 is pinned to amd64 nodes.
5457 - ``affinity``: ``_LARGE_MEMORY_NODE_AFFINITY``, so the ~1.7 GiB
5458 server lands on an instance with more than 4 GiB (see the constant).
5459 """
5460 s3_destination = (
5461 f"s3://{self.cluster_shared_identity.name}/mlflow-artifacts/{self.deployment_region}"
5462 )
5463 vpc_endpoint_cidrs = self.node.try_get_context("vpc_endpoint_cidrs") or ["10.0.0.0/16"]
5464 sidecar, tls_volume = _chart_tls_proxy_sidecar(
5465 name="mlflow-tls-proxy",
5466 image=self.manifest_processor_image.image_uri,
5467 port=_MLFLOW_TLS_PORT,
5468 upstream_port=_MLFLOW_SERVER_PORT,
5469 secret_name="mlflow-tls", # nosec B106 # Kubernetes Secret name, not a credential
5470 )
5471 return {
5472 "values": {
5473 "mlflow": {
5474 "artifactsDestination": s3_destination,
5475 },
5476 "serviceAccount": {
5477 "annotations": {
5478 "eks.amazonaws.com/role-arn": self.mlflow_role.role_arn,
5479 },
5480 },
5481 "storage": {
5482 "size": str(
5483 self.config.get_cluster_observability_config()["mlflow"]["persistence_size"]
5484 ),
5485 },
5486 "server": {
5487 "value_options": {
5488 "allowed_hosts": _mlflow_allowed_hosts(vpc_endpoint_cidrs),
5489 },
5490 },
5491 "extraContainers": [sidecar],
5492 "extraVolumes": [tls_volume],
5493 "nodeSelector": {"kubernetes.io/arch": "amd64"},
5494 "affinity": _LARGE_MEMORY_NODE_AFFINITY,
5495 }
5496 }
5498 def _opencost_chart_values(self) -> dict[str, Any]:
5499 """Build the OpenCost value overrides that carry deployment tokens.
5501 Only the cluster identity is dynamic — every static hardening value
5502 (Prometheus wiring, ServiceMonitor, resource limits, security
5503 contexts) lives in ``charts.yaml``. The identity is
5504 ``opencost.exporter.defaultClusterId`` — the value OpenCost stamps on
5505 every allocation row, which is what lets the multi-region Athena data
5506 distinguish clusters. The chart's root-level ``clusterName`` is NOT
5507 set here: that value is the Kubernetes DNS zone (``cluster.local``)
5508 used to build the Prometheus URL, and overriding it with the EKS
5509 cluster name breaks in-cluster DNS resolution.
5511 The ``opencost-tls-proxy`` sidecar is dynamic too, because it runs
5512 from the cost-monitor image: it terminates verified TLS on 9443 (the
5513 ``opencost-tls`` Service the cost-monitor calls) and forwards to the
5514 exporter API on loopback 9003, with the ``opencost-tls`` leaf mounted
5515 from an optional Secret volume (chart key ``extraVolumes``). The
5516 service images are built for amd64 only, so the pod is pinned to
5517 amd64 nodes (``opencost.nodeSelector``).
5518 """
5519 sidecar, volume = _chart_tls_proxy_sidecar(
5520 name="opencost-tls-proxy",
5521 image=self.cost_monitor_image.image_uri,
5522 port=9443,
5523 upstream_port=9003,
5524 secret_name="opencost-tls", # nosec B106 # Kubernetes Secret name, not a credential
5525 )
5526 return {
5527 "values": {
5528 "opencost": {
5529 "exporter": {
5530 "defaultClusterId": self.cluster.cluster_name,
5531 },
5532 "extraContainers": [sidecar],
5533 "nodeSelector": {"kubernetes.io/arch": "amd64"},
5534 },
5535 "extraVolumes": [volume],
5536 }
5537 }
5539 def _observability_chart_values(self) -> dict[str, Any]:
5540 """Build the kube-prometheus-stack value overrides from cdk.json.
5542 Sizes/retention come from ``cluster_observability`` in ``cdk.json``; the
5543 gp3 ``storageClassName`` is the shared ``_OBSERVABILITY_STORAGE_CLASS``
5544 (also the name of the gated StorageClass manifest), and the
5545 node-exporter tolerations reuse the shared accelerator-node tolerations
5546 so the DaemonSet schedules on tainted GPU/Neuron/EFA nodes. Deep-merged
5547 by the installer over the static hardening values in ``charts.yaml``.
5549 Grafana also gets the ``grafana-tls-proxy`` sidecar, which runs from
5550 the manifest-processor image: verified TLS on 3443 (the
5551 ``grafana-tls`` Service the admin-password rotator calls) forwarded
5552 to Grafana on loopback 3000, with the ``grafana-tls`` leaf mounted
5553 from an optional Secret volume. The Grafana subchart renders
5554 ``extraContainers`` through ``tpl`` as a YAML string, while
5555 ``extraContainerVolumes`` is a list. The service images are amd64
5556 only, so the Grafana pod is pinned to amd64 nodes. Grafana and
5557 Prometheus also carry ``_LARGE_MEMORY_NODE_AFFINITY`` (instances with
5558 more than 4 GiB; see the constant for the live evidence).
5559 """
5560 obs = self.config.get_cluster_observability_config()
5561 storage_class = _OBSERVABILITY_STORAGE_CLASS
5562 grafana_sidecar, grafana_tls_volume = _chart_tls_proxy_sidecar(
5563 name="grafana-tls-proxy",
5564 image=self.manifest_processor_image.image_uri,
5565 port=3443,
5566 upstream_port=3000,
5567 secret_name="grafana-tls", # nosec B106 # Kubernetes Secret name, not a credential
5568 )
5569 return {
5570 "values": {
5571 "grafana": {
5572 "persistence": {
5573 "storageClassName": storage_class,
5574 "size": obs["grafana"]["persistence_size"],
5575 },
5576 "extraContainers": _helm_container_list_string([grafana_sidecar]),
5577 "extraContainerVolumes": [grafana_tls_volume],
5578 "nodeSelector": {"kubernetes.io/arch": "amd64"},
5579 "affinity": _LARGE_MEMORY_NODE_AFFINITY,
5580 },
5581 "prometheus": {
5582 "prometheusSpec": {
5583 "affinity": _LARGE_MEMORY_NODE_AFFINITY,
5584 "retention": obs["prometheus"]["retention"],
5585 "storageSpec": {
5586 "volumeClaimTemplate": {
5587 "spec": {
5588 "storageClassName": storage_class,
5589 "resources": {
5590 "requests": {
5591 "storage": obs["prometheus"]["persistence_size"],
5592 },
5593 },
5594 },
5595 },
5596 },
5597 },
5598 },
5599 "alertmanager": {
5600 "enabled": obs["alertmanager"]["enabled"],
5601 "alertmanagerSpec": {
5602 "storage": {
5603 "volumeClaimTemplate": {
5604 "spec": {
5605 "storageClassName": storage_class,
5606 "resources": {
5607 "requests": {
5608 "storage": obs["alertmanager"]["persistence_size"],
5609 },
5610 },
5611 },
5612 },
5613 },
5614 },
5615 },
5616 "prometheus-node-exporter": {
5617 "tolerations": GCORegionalStack._ADDON_NODE_TOLERATIONS,
5618 },
5619 }
5620 }
5622 def _get_enabled_helm_charts(self) -> list[str]:
5623 """Return the list of Helm charts to install based on cdk.json helm config.
5625 Reads the 'helm' section from cdk.json context. Each key maps to one or
5626 more Helm chart names. Charts are returned in dependency order with Kueue
5627 last (its webhook intercepts all Job/Deployment mutations).
5628 """
5629 helm_config = self.node.try_get_context("helm") or {}
5631 # Mapping from cdk.json helm key → Helm chart name(s) in charts.yaml
5632 # Order matters: dependencies first, Kueue last
5633 chart_map: list[tuple[str, list[str]]] = [
5634 ("aws_load_balancer_controller", ["aws-load-balancer-controller"]),
5635 ("keda", ["keda"]),
5636 ("aws_efa_device_plugin", ["aws-efa-device-plugin"]),
5637 ("aws_neuron_device_plugin", ["aws-neuron-device-plugin"]),
5638 ("volcano", ["volcano"]),
5639 ("kuberay", ["kuberay-operator"]),
5640 ("cert_manager", ["cert-manager"]),
5641 ("slurm", ["slinky-slurm-operator", "slinky-slurm"]),
5642 ("yunikorn", ["yunikorn"]),
5643 ("kubeflow_trainer", ["kubeflow-trainer"]),
5644 ("argocd", ["argocd"]),
5645 # One toggle installs Crossplane and its Crossview dashboard.
5646 ("crossplane", ["crossplane", "crossview"]),
5647 ("kueue", ["kueue"]), # Must be last
5648 ]
5650 # Charts that are mandatory platform components and cannot be disabled
5651 # via cdk.json (see _MANDATORY_CHART_KEYS). KEDA is always installed:
5652 # it backs the built-in SQS queue processor (a ScaledJob) and is the
5653 # only metrics bridge that lets autoscalers consume GPU/CloudWatch
5654 # metrics (the keda-metrics-apiserver serves external.metrics.k8s.io).
5655 # Disabling it would silently break both, so the cdk.json toggle is
5656 # ignored for KEDA. A `helm_enabled_overrides` context value (see
5657 # _parse_helm_enabled_overrides) can force optional charts on for one
5658 # deploy without editing cdk.json.
5659 if {key for key, _names in chart_map} != _HELM_CHART_CONFIG_KEYS:
5660 raise RuntimeError(
5661 "chart_map keys drifted from _HELM_CHART_CONFIG_KEYS; update both together"
5662 )
5663 overrides = _parse_helm_enabled_overrides(
5664 self.node.try_get_context(_HELM_OVERRIDE_CONTEXT_KEY)
5665 )
5667 enabled_charts = []
5668 for config_key, chart_names in chart_map:
5669 if _helm_chart_enabled(helm_config, overrides, config_key):
5670 enabled_charts.extend(chart_names)
5672 # kube-prometheus-stack is driven by the separate on-by-default
5673 # cluster_observability toggle (not the helm block), so include it here
5674 # when enabled. Its install order comes from its file position in
5675 # charts.yaml (before kueue), not from where it sits in this list — the
5676 # installer runs one task per chart in charts.yaml order and skips any
5677 # task whose chart is absent from this enabled set.
5678 if self.config.get_cluster_observability_enabled():
5679 enabled_charts.append("kube-prometheus-stack")
5681 # OpenCost is driven by the on-by-default cost_monitoring toggle and
5682 # additionally requires observability (its Prometheus data source).
5683 # charts.yaml places it after kube-prometheus-stack so the Prometheus
5684 # Operator CRDs exist before its ServiceMonitor renders.
5685 if self._cost_monitoring_active():
5686 enabled_charts.append("opencost")
5688 # MLflow is driven by the on-by-default cluster_observability.mlflow
5689 # sub-toggle and requires observability itself (monitoring namespace,
5690 # gp3 StorageClass, ServiceMonitor discovery, tunnel access path).
5691 # trust-manager comes with it: it publishes the internal CA bundle
5692 # MLflow's gco-jobs clients verify the tracking server's HTTPS
5693 # listener against, and nothing else uses it. charts.yaml orders it
5694 # right after cert-manager, which issues its webhook certificate.
5695 if self._mlflow_active():
5696 enabled_charts.extend(("trust-manager", "mlflow"))
5698 return enabled_charts
5700 def _create_helm_installer_lambda(self) -> None:
5701 """Create Lambda function to install Helm charts (KEDA, NVIDIA DRA, etc.).
5703 This Lambda uses Helm to install charts that require complex setup
5704 (TLS certificates, CRDs, etc.) that are difficult to manage via raw manifests.
5706 Charts installed:
5707 - KEDA: Kubernetes Event-Driven Autoscaling (mandatory, always installed)
5708 - Volcano, KubeRay, Kueue, cert-manager, and other schedulers (toggle via cdk.json)
5709 """
5710 project_name = self.config.get_project_name()
5712 # Create IAM role for Helm installer Lambda
5713 helm_lambda_role = iam.Role(
5714 self,
5715 "HelmInstallerLambdaRole",
5716 assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
5717 managed_policies=[
5718 iam.ManagedPolicy.from_aws_managed_policy_name(
5719 "service-role/AWSLambdaVPCAccessExecutionRole"
5720 ),
5721 iam.ManagedPolicy.from_aws_managed_policy_name(
5722 "service-role/AWSLambdaBasicExecutionRole"
5723 ),
5724 ],
5725 )
5727 # Add EKS permissions
5728 helm_lambda_role.add_to_policy(
5729 iam.PolicyStatement(
5730 actions=["eks:DescribeCluster", "eks:ListClusters"],
5731 resources=[self.cluster.cluster_arn],
5732 )
5733 )
5735 # Create security group for Helm installer Lambda
5736 helm_lambda_sg = ec2.SecurityGroup(
5737 self,
5738 "HelmInstallerLambdaSG",
5739 vpc=self.vpc,
5740 description="Security group for Helm installer Lambda to access EKS cluster",
5741 security_group_name=f"{project_name}-helm-lambda-sg-{self.deployment_region}",
5742 allow_all_outbound=True,
5743 )
5745 # Allow Lambda to access EKS cluster API
5746 self.cluster.cluster_security_group.add_ingress_rule(
5747 peer=helm_lambda_sg,
5748 connection=ec2.Port.tcp(443),
5749 description="Allow Helm installer Lambda to access EKS API",
5750 )
5752 # Build Docker image for Helm installer Lambda
5753 # Points at helm-installer-build/ which is rebuilt fresh every deploy
5754 # by _build_helm_installer_lambda() in cli/stacks.py
5755 ecr_assets.DockerImageAsset(
5756 self,
5757 "HelmInstallerImage",
5758 directory="lambda/helm-installer-build",
5759 platform=ecr_assets.Platform.LINUX_AMD64,
5760 )
5762 # Create Lambda function using Docker image
5763 # Store function name as string attribute for cross-stack references
5764 # This avoids CDK cross-environment resolution issues when account is unresolved
5765 self.helm_installer_lambda_function_name = f"{project_name}-helm-{self.deployment_region}"
5766 self.helm_installer_lambda = lambda_.DockerImageFunction(
5767 self,
5768 "HelmInstallerFunction",
5769 function_name=self.helm_installer_lambda_function_name,
5770 code=lambda_.DockerImageCode.from_image_asset(
5771 directory="lambda/helm-installer-build",
5772 platform=ecr_assets.Platform.LINUX_AMD64,
5773 ),
5774 timeout=Duration.minutes(15),
5775 memory_size=1024,
5776 architecture=lambda_.Architecture.X86_64,
5777 role=helm_lambda_role,
5778 vpc=self.vpc,
5779 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
5780 security_groups=[helm_lambda_sg],
5781 environment={
5782 "CLUSTER_NAME": self.cluster.cluster_name,
5783 "REGION": self.deployment_region,
5784 "PROJECT_NAME": project_name,
5785 },
5786 tracing=lambda_.Tracing.ACTIVE,
5787 )
5789 # Allow the installer to record per-chart add-on status to SSM so the
5790 # add-on layer's health is observable out-of-band (decoupled from the
5791 # CloudFormation rollback path). Read back via `gco stacks addons-status`.
5792 helm_lambda_role.add_to_policy(
5793 iam.PolicyStatement(
5794 actions=["ssm:PutParameter"],
5795 resources=[
5796 f"arn:{self.partition}:ssm:{self.deployment_region}:{self.account}:"
5797 f"parameter/{project_name}/addons/*"
5798 ],
5799 )
5800 )
5802 # Add EKS access entry for the Lambda role
5803 self.helm_installer_access_entry = eks.AccessEntry(
5804 self,
5805 "HelmInstallerLambdaAccessEntry",
5806 cluster=self.cluster, # type: ignore[arg-type]
5807 principal=helm_lambda_role.role_arn,
5808 access_policies=[
5809 eks.AccessPolicy.from_access_policy_name(
5810 "AmazonEKSClusterAdminPolicy", access_scope_type=eks.AccessScopeType.CLUSTER
5811 )
5812 ],
5813 )
5815 # ------------------------------------------------------------------
5816 # Step Functions state machine: one task per chart, in charts.yaml
5817 # order. Each chart gets its own retry + Step Functions console
5818 # visibility, and — critically — no single Lambda invocation is bound
5819 # by the 15-minute Lambda limit, so a slow operator (e.g. a cold NVIDIA
5820 # image pull) just costs extra retries instead of failing the deploy.
5821 # ------------------------------------------------------------------
5822 chart_order = _load_helm_chart_order()
5824 def _chart_task(chart_name: str) -> sfn_tasks.LambdaInvoke:
5825 task = sfn_tasks.LambdaInvoke(
5826 self,
5827 f"HelmChart-{chart_name}",
5828 lambda_function=self.helm_installer_lambda,
5829 payload=sfn.TaskInput.from_object(
5830 {
5831 "Action": "install_chart",
5832 "Chart": chart_name,
5833 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
5834 "Region": sfn.JsonPath.string_at("$.Region"),
5835 "EnabledCharts": sfn.JsonPath.list_at("$.EnabledCharts"),
5836 "Charts": sfn.JsonPath.object_at("$.Charts"),
5837 "KedaOperatorRoleArn": sfn.JsonPath.string_at("$.KedaOperatorRoleArn"),
5838 }
5839 ),
5840 payload_response_only=True,
5841 # Keep the execution input intact so the next chart task can
5842 # still read $.ClusterName, $.EnabledCharts, etc.
5843 result_path="$.lastChart",
5844 task_timeout=sfn.Timeout.duration(Duration.minutes(16)),
5845 )
5846 # Per-chart retry with backoff. A cold image pull or a webhook race
5847 # clears on a later attempt; only after exhausting these does the
5848 # chart (and the deploy) fail.
5849 task.add_retry(
5850 errors=["States.ALL"],
5851 max_attempts=4,
5852 interval=Duration.seconds(30),
5853 backoff_rate=2.0,
5854 max_delay=Duration.minutes(5),
5855 )
5856 return task
5858 def _kubectl_task(task_id: str, *, post_helm: bool) -> sfn_tasks.LambdaInvoke:
5859 """One kubectl-apply pass (base or post-Helm) as a state-machine task.
5861 Reads ClusterName / Region / ImageReplacements from the execution
5862 input; the handler raises on any manifest failure so the task's
5863 Retry/Catch can react.
5864 """
5865 task = sfn_tasks.LambdaInvoke(
5866 self,
5867 task_id,
5868 lambda_function=self.kubectl_lambda,
5869 payload=sfn.TaskInput.from_object(
5870 {
5871 "Action": "apply_manifests",
5872 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
5873 "Region": sfn.JsonPath.string_at("$.Region"),
5874 "ImageReplacements": sfn.JsonPath.object_at("$.ImageReplacements"),
5875 "PostHelm": "true" if post_helm else "false",
5876 }
5877 ),
5878 payload_response_only=True,
5879 # Keep the execution input intact so later tasks still read
5880 # $.ClusterName, $.ImageReplacements, $.EndpointGroupArn, etc.
5881 result_path="$.lastApply",
5882 task_timeout=sfn.Timeout.duration(Duration.minutes(15)),
5883 )
5884 task.add_retry(
5885 errors=["States.ALL"],
5886 max_attempts=3,
5887 interval=Duration.seconds(30),
5888 backoff_rate=2.0,
5889 max_delay=Duration.minutes(3),
5890 )
5891 return task
5893 def _manifest_validation_task() -> sfn_tasks.LambdaInvoke:
5894 """Require every effective raw manifest object to exist and be ready."""
5895 task = sfn_tasks.LambdaInvoke(
5896 self,
5897 "ValidateKubernetesManifests",
5898 lambda_function=self.kubectl_lambda,
5899 payload=sfn.TaskInput.from_object(
5900 {
5901 "Action": "validate_manifests",
5902 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
5903 "Region": sfn.JsonPath.string_at("$.Region"),
5904 "ImageReplacements": sfn.JsonPath.object_at("$.ImageReplacements"),
5905 "DeploymentToken": sfn.JsonPath.string_at("$.DeploymentToken"),
5906 }
5907 ),
5908 payload_response_only=True,
5909 result_path="$.manifestValidation",
5910 task_timeout=sfn.Timeout.duration(Duration.minutes(15)),
5911 )
5912 # Cold EKS Auto Mode clusters can need more than the original
5913 # ~9-minute retry schedule for node capacity, cert-manager Secrets,
5914 # PDBs, and EndpointSlices to converge. Validation is read-only;
5915 # eight retries sample through ~21 minutes without replaying apply.
5916 task.add_retry(
5917 errors=["States.ALL"],
5918 max_attempts=8,
5919 interval=Duration.minutes(1),
5920 backoff_rate=2.0,
5921 max_delay=Duration.minutes(3),
5922 )
5923 return task
5925 def _helm_validation_task() -> sfn_tasks.LambdaInvoke:
5926 """Require every configured Helm release and rendered object to be ready."""
5927 task = sfn_tasks.LambdaInvoke(
5928 self,
5929 "ValidateHelmReleases",
5930 lambda_function=self.helm_installer_lambda,
5931 payload=sfn.TaskInput.from_object(
5932 {
5933 "Action": "validate_releases",
5934 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
5935 "Region": sfn.JsonPath.string_at("$.Region"),
5936 "EnabledCharts": sfn.JsonPath.list_at("$.EnabledCharts"),
5937 "Charts": sfn.JsonPath.object_at("$.Charts"),
5938 "DeploymentToken": sfn.JsonPath.string_at("$.DeploymentToken"),
5939 }
5940 ),
5941 payload_response_only=True,
5942 result_path="$.helmValidation",
5943 task_timeout=sfn.Timeout.duration(Duration.minutes(16)),
5944 )
5945 task.add_retry(
5946 errors=["States.ALL"],
5947 max_attempts=4,
5948 interval=Duration.minutes(1),
5949 backoff_rate=2.0,
5950 max_delay=Duration.minutes(3),
5951 )
5952 return task
5954 def _endpoint_publication_task() -> sfn_tasks.LambdaInvoke:
5955 """Publish the exact Gateway ALB and optionally register it with GA."""
5956 payload: dict[str, Any] = {
5957 "Action": "publish_gateway_endpoint",
5958 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
5959 "Region": sfn.JsonPath.string_at("$.Region"),
5960 "RegistryRegion": sfn.JsonPath.string_at("$.RegistryRegion"),
5961 "ProjectName": sfn.JsonPath.string_at("$.ProjectName"),
5962 }
5963 if self.global_accelerator_enabled:
5964 payload["EndpointGroupArn"] = sfn.JsonPath.string_at("$.EndpointGroupArn")
5965 # The configured health-check contract is a synth-time
5966 # constant, so it is baked into the task payload as literals
5967 # rather than threaded through the execution input: replayed
5968 # pre-upgrade inputs (`gco stacks addons install`) carry no
5969 # such keys, and a JsonPath reference to an absent input key
5970 # would fail the whole convergence pipeline. The handler's
5971 # defaults match the values it historically hardcoded.
5972 ga_config = self.config.get_global_accelerator_config()
5973 payload["GaHealthCheckPath"] = ga_config.get("health_check_path", "/api/v1/health")
5974 payload["GaHealthCheckInterval"] = ga_config.get("health_check_interval", 30)
5975 payload["GaHealthCheckThreshold"] = ga_config.get("health_check_threshold", 3)
5977 task = sfn_tasks.LambdaInvoke(
5978 self,
5979 "PublishGatewayEndpoint",
5980 lambda_function=self.ga_registration_lambda,
5981 payload=sfn.TaskInput.from_object(payload),
5982 payload_response_only=True,
5983 result_path="$.endpointPublication",
5984 task_timeout=sfn.Timeout.duration(Duration.minutes(16)),
5985 )
5986 task.add_retry(
5987 errors=["States.ALL"],
5988 max_attempts=3,
5989 interval=Duration.seconds(30),
5990 backoff_rate=2.0,
5991 max_delay=Duration.minutes(3),
5992 )
5993 return task
5995 chart_tasks = [_chart_task(name) for name in chart_order]
5997 # The state machine owns the full convergence pipeline:
5998 # base apply -> Helm charts -> post-Helm apply -> exhaustive raw
5999 # manifest validation -> exhaustive Helm/rendered-object validation
6000 # -> unconditional Gateway endpoint publication, with optional Global
6001 # Accelerator registration.
6002 #
6003 # Individual chart failures still continue so every release gets an
6004 # install attempt and diagnostic. The terminal validators then make the
6005 # overall execution fail unless every expected object and release is
6006 # present and ready. Post-Helm apply, both validators, and endpoint
6007 # publication deliberately have no catch-to-success path: topology may
6008 # trust only an exact SUCCEEDED execution for the current deployment
6009 # token.
6010 done = sfn.Succeed(self, "HelmInstallComplete")
6011 base_apply = _kubectl_task("ApplyBaseManifests", post_helm=False)
6012 post_apply = _kubectl_task("ApplyPostHelmManifests", post_helm=True)
6013 manifest_validation = _manifest_validation_task()
6014 helm_validation = _helm_validation_task()
6016 endpoint_publication = _endpoint_publication_task()
6017 post_apply.next(manifest_validation)
6018 manifest_validation.next(helm_validation)
6019 helm_validation.next(endpoint_publication)
6020 endpoint_publication.next(done)
6022 if chart_tasks:
6023 for i, task in enumerate(chart_tasks):
6024 next_state: sfn.IChainable = (
6025 chart_tasks[i + 1] if i + 1 < len(chart_tasks) else post_apply
6026 )
6027 task.add_catch(
6028 next_state,
6029 errors=["States.ALL"],
6030 result_path="$.lastChartError",
6031 )
6032 task.next(next_state)
6033 base_apply.next(chart_tasks[0])
6034 else: # pragma: no cover - charts.yaml is always present in the repo
6035 base_apply.next(post_apply)
6037 # base apply has NO catch on purpose: a persistent base-manifest failure
6038 # fails the execution rather than converging onto an incomplete base.
6039 start_state: sfn.IChainable = base_apply
6041 helm_sm_log_group = logs.LogGroup(
6042 self,
6043 "HelmInstallStateMachineLogGroup",
6044 retention=logs.RetentionDays.ONE_WEEK,
6045 removal_policy=RemovalPolicy.DESTROY,
6046 )
6048 self.helm_install_state_machine = sfn.StateMachine(
6049 self,
6050 "HelmInstallStateMachine",
6051 definition_body=sfn.DefinitionBody.from_chainable(start_state),
6052 state_machine_type=sfn.StateMachineType.STANDARD,
6053 timeout=Duration.hours(2),
6054 tracing_enabled=True,
6055 logs=sfn.LogOptions(destination=helm_sm_log_group, level=sfn.LogLevel.ALL),
6056 )
6058 # Thin fire-and-forget provider: onEvent starts the execution and
6059 # returns immediately. It does no Helm/Kubernetes work, so it never
6060 # approaches the Lambda timeout — all the heavy lifting lives in the
6061 # state machine, which converges charts in the background.
6062 helm_orchestrator_on_event = lambda_.Function(
6063 self,
6064 "HelmOrchestratorOnEvent",
6065 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
6066 handler="handler.on_event",
6067 code=lambda_source_code("lambda/helm-orchestrator"),
6068 timeout=Duration.minutes(1),
6069 memory_size=256,
6070 environment={
6071 "STATE_MACHINE_ARN": self.helm_install_state_machine.state_machine_arn,
6072 },
6073 tracing=lambda_.Tracing.ACTIVE,
6074 )
6075 self.helm_install_state_machine.grant_start_execution(helm_orchestrator_on_event)
6076 self.helm_install_state_machine.grant_execution(
6077 helm_orchestrator_on_event,
6078 "states:StopExecution",
6079 "states:DescribeExecution",
6080 )
6082 # Let on_event persist the execution input to SSM so the add-on install
6083 # can be replayed out-of-band (gco stacks addons install) without the
6084 # CLI reconstructing chart config or the KEDA role ARN.
6085 helm_orchestrator_on_event.add_to_role_policy(
6086 iam.PolicyStatement(
6087 actions=[
6088 "ssm:PutParameter",
6089 "ssm:GetParameter",
6090 "ssm:DeleteParameter",
6091 ],
6092 resources=[
6093 f"arn:{self.partition}:ssm:{self.deployment_region}:{self.account}:"
6094 f"parameter/{project_name}/addons/*"
6095 ],
6096 )
6097 )
6099 # HelmInstallCharts depends on this explicit bounded-retention group,
6100 # forcing the trigger's final provider invocation to finish before
6101 # CloudFormation removes the group.
6102 self.helm_installer_provider = cr.Provider(
6103 self,
6104 "HelmInstallerProvider",
6105 on_event_handler=helm_orchestrator_on_event,
6106 log_group=self.helm_installer_provider_log_group,
6107 )
6109 # Unlike create/update convergence, stack deletion must be synchronous:
6110 # Helm releases can own admission webhooks, load balancers, and CRs that
6111 # have to disappear while the Kubernetes API and installer AccessEntry
6112 # still exist. A delete-only provider waits on a reverse-order state
6113 # machine and fails CloudFormation if any real uninstall fails.
6114 self._create_helm_teardown(chart_order)
6116 # cdk-nag suppressions for the install path.
6117 from gco.stacks.nag_suppressions import acknowledge_nag_findings
6119 acknowledge_nag_findings(
6120 helm_lambda_role,
6121 [
6122 {
6123 "id": "AwsSolutions-IAM5",
6124 "reason": (
6125 "The Helm installer Lambda requires broad EKS and Kubernetes API "
6126 "access to install Helm charts (KEDA, NVIDIA DRA, etc.) that create "
6127 "CRDs, RBAC rules, and workloads across multiple namespaces. "
6128 "Resource: * is required because the set of Kubernetes resources "
6129 "is dynamic and not known at synth time."
6130 ),
6131 "appliesTo": ["Resource::*"],
6132 },
6133 ],
6134 )
6135 # The state machine role (auto-generated) invokes the worker Lambda
6136 # across versions using the AWS-standard ``:*`` qualifier that cannot be
6137 # enumerated at synth time.
6138 acknowledge_nag_findings(
6139 self.helm_install_state_machine,
6140 [
6141 {
6142 "id": "AwsSolutions-IAM5",
6143 "reason": (
6144 "The state machine invokes the helm worker Lambda; CDK grants "
6145 "lambda:InvokeFunction with the :* version qualifier, which is the "
6146 "standard form and cannot be narrowed at synth time."
6147 ),
6148 "appliesTo": ["Resource::*"],
6149 },
6150 ],
6151 )
6152 acknowledge_nag_findings(
6153 helm_orchestrator_on_event,
6154 [
6155 {
6156 "id": "AwsSolutions-IAM5",
6157 "reason": (
6158 "Lambda active tracing requires X-Ray write APIs against "
6159 "Resource::*, as X-Ray does not expose resource-level "
6160 "permissions for these telemetry calls."
6161 ),
6162 "appliesTo": ["Resource::*"],
6163 },
6164 {
6165 "id": "AwsSolutions-IAM5",
6166 "reason": (
6167 "If execution metadata persistence fails, the orchestrator must "
6168 "stop the just-started convergence execution before CloudFormation "
6169 "can roll back. This grant is limited to executions of the single "
6170 "regional Helm install state machine."
6171 ),
6172 "appliesTo": [
6173 "Resource::arn:<AWS::Partition>:states:<AWS::Region>:"
6174 "<AWS::AccountId>:execution:"
6175 '{"Fn::Select":[6,{"Fn::Split":[":",'
6176 '{"Ref":"HelmInstallStateMachine7DB71CDC"}]}]}:*'
6177 ],
6178 },
6179 ],
6180 )
6182 # The cr.Provider framework auto-generates a framework-onEvent Lambda and
6183 # its role (and, were an is_complete_handler set, a waiter state machine —
6184 # which this fire-and-forget provider does NOT create). None of these are
6185 # configurable by us: the framework role invokes our handler Lambda via
6186 # the standard ``<lambda-arn>:*`` version qualifier that cannot be narrowed
6187 # at synth time. Suppress the relevant rules across the whole provider
6188 # subtree; appliesTo is omitted because the findings are granted on
6189 # CDK-managed resources we do not author. The SF1/SF2/X-Ray entries are
6190 # retained defensively to cover any helper state machine the framework may
6191 # emit across CDK versions; they are harmless no-ops when none exists.
6192 acknowledge_nag_findings(
6193 self.helm_installer_provider,
6194 [
6195 {
6196 "id": "AwsSolutions-IAM5",
6197 "reason": (
6198 "CDK custom-resource provider framework roles invoke the "
6199 "orchestrator Lambdas via the standard '<lambda-arn>:*' version "
6200 "qualifier, which cannot be enumerated at synth time."
6201 ),
6202 "appliesTo": [
6203 "Resource::<HelmOrchestratorOnEventD0D51D9B.Arn>:*",
6204 ],
6205 },
6206 {
6207 "id": "AwsSolutions-SF1",
6208 "reason": (
6209 "The waiter state machine is auto-generated by the CDK "
6210 "cr.Provider framework and does not expose log configuration; "
6211 "ALL-event logging cannot be enabled on it."
6212 ),
6213 },
6214 {
6215 "id": "AwsSolutions-SF2",
6216 "reason": (
6217 "The waiter state machine is auto-generated by the CDK "
6218 "cr.Provider framework and does not expose tracing "
6219 "configuration; X-Ray cannot be enabled on it."
6220 ),
6221 },
6222 {
6223 "id": "Serverless-StepFunctionStateMachineXray",
6224 "reason": (
6225 "The waiter state machine is auto-generated by the CDK "
6226 "cr.Provider framework and does not expose tracing "
6227 "configuration; X-Ray cannot be enabled on it."
6228 ),
6229 },
6230 ],
6231 )
6233 def _create_helm_teardown(self, chart_order: list[str]) -> None:
6234 """Create the synchronous, reverse-order Helm stack-delete path.
6236 Create/update remain fire-and-forget through ``HelmInstallCharts``. This
6237 separate custom resource is a no-op for those events, but on Delete it
6238 starts a state machine whose per-chart tasks call ``uninstall_chart`` in
6239 reverse install order. The provider polls to terminal state, so a failed
6240 release blocks deletion before EKS authentication or the API disappears.
6241 """
6243 lbc_chart = "aws-load-balancer-controller"
6244 if not chart_order or chart_order[0] != lbc_chart:
6245 raise RuntimeError(
6246 "aws-load-balancer-controller must be the first Helm chart for safe teardown"
6247 )
6249 provider_code = lambda_source_code("lambda/helm-installer")
6250 drain_checker = lambda_.Function(
6251 self,
6252 "HelmTeardownDrainChecker",
6253 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
6254 handler="teardown_provider.drain_install_executions",
6255 code=provider_code,
6256 timeout=Duration.minutes(1),
6257 memory_size=256,
6258 environment={
6259 "INSTALL_STATE_MACHINE_ARN": self.helm_install_state_machine.state_machine_arn,
6260 },
6261 tracing=lambda_.Tracing.ACTIVE,
6262 )
6264 def _uninstall_task(chart_name: str) -> sfn_tasks.LambdaInvoke:
6265 timeout_minutes = (
6266 5 if chart_name == lbc_chart else _UNINSTALL_TIMEOUT_MINUTES.get(chart_name, 2)
6267 )
6268 task = sfn_tasks.LambdaInvoke(
6269 self,
6270 f"HelmUninstallChart-{chart_name}",
6271 lambda_function=self.helm_installer_lambda,
6272 payload=sfn.TaskInput.from_object(
6273 {
6274 "Action": "uninstall_chart",
6275 "Chart": chart_name,
6276 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
6277 "Region": sfn.JsonPath.string_at("$.Region"),
6278 "EnabledCharts": sfn.JsonPath.list_at("$.EnabledCharts"),
6279 "Charts": sfn.JsonPath.object_at("$.Charts"),
6280 "KedaOperatorRoleArn": sfn.JsonPath.string_at("$.KedaOperatorRoleArn"),
6281 }
6282 ),
6283 payload_response_only=True,
6284 result_path="$.lastChart",
6285 task_timeout=sfn.Timeout.duration(Duration.minutes(timeout_minutes)),
6286 )
6287 return task
6289 def _drain_check_task() -> sfn_tasks.LambdaInvoke:
6290 return sfn_tasks.LambdaInvoke(
6291 self,
6292 "CheckRunningConvergence",
6293 lambda_function=drain_checker,
6294 payload=sfn.TaskInput.from_object({}),
6295 payload_response_only=True,
6296 result_path="$.drainCheck",
6297 task_timeout=sfn.Timeout.duration(Duration.minutes(1)),
6298 )
6300 def _quiesce_task() -> sfn_tasks.LambdaInvoke:
6301 task = sfn_tasks.LambdaInvoke(
6302 self,
6303 "QuiesceHealthMonitor",
6304 lambda_function=self.helm_installer_lambda,
6305 payload=sfn.TaskInput.from_object(
6306 {
6307 "Action": "quiesce_health_monitor",
6308 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
6309 "Region": sfn.JsonPath.string_at("$.Region"),
6310 }
6311 ),
6312 payload_response_only=True,
6313 result_path="$.healthMonitorQuiesce",
6314 task_timeout=sfn.Timeout.duration(Duration.minutes(3)),
6315 )
6316 return task
6318 def _endpoint_cleanup_task() -> sfn_tasks.LambdaInvoke:
6319 """Fence SSM/GA publication after all endpoint writers are quiesced."""
6320 payload: dict[str, Any] = {
6321 "Action": "cleanup_gateway_endpoint",
6322 "Region": sfn.JsonPath.string_at("$.Region"),
6323 "RegistryRegion": sfn.JsonPath.string_at("$.RegistryRegion"),
6324 "ProjectName": sfn.JsonPath.string_at("$.ProjectName"),
6325 }
6326 if self.global_accelerator_enabled:
6327 payload["EndpointGroupArn"] = sfn.JsonPath.string_at("$.EndpointGroupArn")
6328 return sfn_tasks.LambdaInvoke(
6329 self,
6330 "CleanupGatewayEndpoint",
6331 lambda_function=self.ga_registration_lambda,
6332 payload=sfn.TaskInput.from_object(payload),
6333 payload_response_only=True,
6334 result_path="$.endpointCleanup",
6335 task_timeout=sfn.Timeout.duration(Duration.minutes(15)),
6336 )
6338 def _gateway_cleanup_task() -> sfn_tasks.LambdaInvoke:
6339 return sfn_tasks.LambdaInvoke(
6340 self,
6341 "DeleteGatewayResources",
6342 lambda_function=self.kubectl_lambda,
6343 payload=sfn.TaskInput.from_object(
6344 {
6345 "Action": "delete_gateway_resources",
6346 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
6347 "Region": sfn.JsonPath.string_at("$.Region"),
6348 }
6349 ),
6350 payload_response_only=True,
6351 result_path="$.gatewayCleanup",
6352 task_timeout=sfn.Timeout.duration(Duration.minutes(5)),
6353 )
6355 non_lbc_tasks = [
6356 _uninstall_task(name) for name in reversed(chart_order) if name != lbc_chart
6357 ]
6358 endpoint_cleanup = _endpoint_cleanup_task()
6359 pre_gateway_cleanup = sfn.Parallel(
6360 self,
6361 "CleanupEndpointAndCharts",
6362 result_path="$.preGatewayCleanup",
6363 )
6364 pre_gateway_cleanup.branch(endpoint_cleanup)
6365 if non_lbc_tasks:
6366 for index, task in enumerate(non_lbc_tasks[:-1]):
6367 task.next(non_lbc_tasks[index + 1])
6368 pre_gateway_cleanup.branch(non_lbc_tasks[0])
6370 lbc_uninstall = _uninstall_task(lbc_chart)
6371 gateway_cleanup = _gateway_cleanup_task()
6372 done = sfn.Succeed(self, "HelmTeardownComplete")
6373 quiesce = _quiesce_task()
6375 quiesce.next(pre_gateway_cleanup)
6376 pre_gateway_cleanup.next(gateway_cleanup)
6377 gateway_cleanup.next(lbc_uninstall)
6378 lbc_uninstall.next(done)
6380 # StopExecution cannot cancel a Lambda invocation already in flight,
6381 # and ListExecutions is eventually consistent. The provider stops the
6382 # initially visible executions before this unconditional 16-minute
6383 # drain. The checker then stops any late-visible work and loops through
6384 # another complete drain interval before quiescence. The SSM teardown
6385 # fence blocks supported convergence entrypoints from creating new work.
6386 drain_in_flight = sfn.Wait(
6387 self,
6388 "DrainInFlightConvergence",
6389 time=sfn.WaitTime.seconds_path("$.WaitForInFlightSeconds"),
6390 )
6391 drain_check = _drain_check_task()
6392 late_work = sfn.Choice(self, "LateConvergenceFound")
6393 drain_in_flight.next(drain_check)
6394 drain_check.next(late_work)
6395 late_work.when(
6396 sfn.Condition.number_greater_than("$.drainCheck.StoppedExecutions", 0),
6397 drain_in_flight,
6398 ).otherwise(quiesce)
6399 start_state: sfn.IChainable = drain_in_flight
6401 teardown_log_group = logs.LogGroup(
6402 self,
6403 "HelmTeardownStateMachineLogGroup",
6404 retention=logs.RetentionDays.ONE_WEEK,
6405 removal_policy=RemovalPolicy.DESTROY,
6406 )
6407 self.helm_teardown_state_machine = sfn.StateMachine(
6408 self,
6409 "HelmTeardownStateMachine",
6410 definition_body=sfn.DefinitionBody.from_chainable(start_state),
6411 state_machine_type=sfn.StateMachineType.STANDARD,
6412 # 16m drain + 3m quiesce + max(15m endpoint cleanup, the ordinary
6413 # chart chain) + 5m Gateway deletion + 5m LBC uninstall. The chain's
6414 # per-chart task timeouts (2m; _UNINSTALL_TIMEOUT_MINUTES for charts
6415 # whose custom resources are deleted first) are ceilings for a
6416 # stalled release, not expected durations: a chart that is not
6417 # installed returns in seconds, so the chain's real length is set by
6418 # the installed charts. The 56m cap leaves three minutes for the
6419 # provider's final poll inside CloudFormation's one-hour ceiling.
6420 timeout=Duration.minutes(56),
6421 tracing_enabled=True,
6422 logs=sfn.LogOptions(destination=teardown_log_group, level=sfn.LogLevel.ALL),
6423 )
6425 teardown_on_event = lambda_.Function(
6426 self,
6427 "HelmTeardownOnEvent",
6428 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
6429 handler="teardown_provider.on_event",
6430 code=provider_code,
6431 timeout=Duration.minutes(1),
6432 memory_size=256,
6433 environment={
6434 "TEARDOWN_STATE_MACHINE_ARN": self.helm_teardown_state_machine.state_machine_arn,
6435 "INSTALL_STATE_MACHINE_ARN": self.helm_install_state_machine.state_machine_arn,
6436 },
6437 tracing=lambda_.Tracing.ACTIVE,
6438 )
6439 teardown_is_complete = lambda_.Function(
6440 self,
6441 "HelmTeardownIsComplete",
6442 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
6443 handler="teardown_provider.is_complete",
6444 code=provider_code,
6445 timeout=Duration.minutes(1),
6446 memory_size=256,
6447 environment={
6448 "TEARDOWN_STATE_MACHINE_ARN": self.helm_teardown_state_machine.state_machine_arn,
6449 "INSTALL_STATE_MACHINE_ARN": self.helm_install_state_machine.state_machine_arn,
6450 },
6451 tracing=lambda_.Tracing.ACTIVE,
6452 )
6453 self.helm_teardown_state_machine.grant_start_execution(teardown_on_event)
6454 self.helm_teardown_state_machine.grant_read(teardown_is_complete)
6455 install_execution_detail = (
6456 "Resource::arn:<AWS::Partition>:states:<AWS::Region>:<AWS::AccountId>:execution:"
6457 '{"Fn::Select":[6,{"Fn::Split":[":",'
6458 '{"Ref":"HelmInstallStateMachine7DB71CDC"}]}]}:*'
6459 )
6460 teardown_execution_detail = (
6461 "Resource::arn:<AWS::Partition>:states:<AWS::Region>:<AWS::AccountId>:execution:"
6462 '{"Fn::Select":[6,{"Fn::Split":[":",'
6463 '{"Ref":"HelmTeardownStateMachine1C15895F"}]}]}:*'
6464 )
6465 for handler in (teardown_on_event, drain_checker):
6466 self.helm_install_state_machine.grant(
6467 handler,
6468 "states:ListExecutions",
6469 )
6470 self.helm_install_state_machine.grant_execution(
6471 handler,
6472 "states:StopExecution",
6473 "states:DescribeExecution",
6474 )
6475 teardown_on_event.add_to_role_policy(
6476 iam.PolicyStatement(
6477 actions=["ssm:PutParameter"],
6478 resources=[
6479 f"arn:{self.partition}:ssm:{self.deployment_region}:{self.account}:"
6480 f"parameter/{self.config.get_project_name()}/addons/"
6481 f"{self.deployment_region}/_teardown"
6482 ],
6483 )
6484 )
6486 # Strict live validation preserves this generation until exact
6487 # post-stack cleanup. Ordinary deployments retain DESTROY semantics.
6488 provider_log_group = logs.LogGroup(
6489 self,
6490 "HelmTeardownProviderLogGroup",
6491 retention=logs.RetentionDays.ONE_WEEK,
6492 removal_policy=self.provider_log_group_removal_policy,
6493 )
6494 self.helm_teardown_provider = cr.Provider(
6495 self,
6496 "HelmTeardownProvider",
6497 on_event_handler=teardown_on_event,
6498 is_complete_handler=teardown_is_complete,
6499 query_interval=Duration.seconds(15),
6500 total_timeout=Duration.minutes(59),
6501 log_group=provider_log_group,
6502 )
6503 teardown_properties: dict[str, Any] = {
6504 "ClusterName": self.cluster.cluster_name,
6505 "Region": self.deployment_region,
6506 "RegistryRegion": self.config.get_global_region(),
6507 "ProjectName": self.config.get_project_name(),
6508 "EnabledCharts": self._get_enabled_helm_charts(),
6509 "Charts": self._helm_chart_value_overrides_json(),
6510 "KedaOperatorRoleArn": self.keda_operator_role.role_arn,
6511 }
6512 if self.endpoint_group_arn is not None:
6513 teardown_properties["EndpointGroupArn"] = self.endpoint_group_arn
6514 self.helm_teardown_resource = CustomResource(
6515 self,
6516 "HelmTeardown",
6517 service_token=self.helm_teardown_provider.service_token,
6518 properties=teardown_properties,
6519 )
6520 self.helm_teardown_resource.node.add_dependency(self.cluster)
6521 self.helm_teardown_resource.node.add_dependency(self.helm_installer_access_entry)
6522 self.helm_teardown_resource.node.add_dependency(self.kubectl_lambda_access_entry)
6523 self.helm_teardown_resource.node.add_dependency(self.helm_install_state_machine)
6524 self.helm_teardown_resource.node.add_dependency(self.aws_load_balancer_controller_policy)
6525 self.helm_teardown_resource.node.add_dependency(provider_log_group)
6527 from gco.stacks.nag_suppressions import acknowledge_nag_findings
6529 acknowledge_nag_findings(
6530 self.helm_teardown_state_machine,
6531 [
6532 {
6533 "id": "AwsSolutions-IAM5",
6534 "reason": (
6535 "The teardown state machine's X-Ray integration requires "
6536 "Resource::*, and its Lambda task grants use CDK's required :* "
6537 "version qualifier. The drain-checker detail names the single "
6538 "dedicated function created by this stack."
6539 ),
6540 "appliesTo": [
6541 "Resource::*",
6542 "Resource::<HelmTeardownDrainCheckerCCF8D9D1.Arn>:*",
6543 ],
6544 },
6545 ],
6546 )
6547 acknowledge_nag_findings(
6548 drain_checker,
6549 [
6550 {
6551 "id": "AwsSolutions-IAM5",
6552 "reason": (
6553 "The drain checker uses X-Ray Resource::* APIs and stops only "
6554 "runtime-generated executions of the single regional Helm install "
6555 "state machine before Kubernetes teardown."
6556 ),
6557 "appliesTo": ["Resource::*", install_execution_detail],
6558 }
6559 ],
6560 )
6561 for handler in (teardown_on_event, teardown_is_complete):
6562 acknowledge_nag_findings(
6563 handler,
6564 [
6565 {
6566 "id": "AwsSolutions-IAM5",
6567 "reason": (
6568 "X-Ray write APIs require Resource::*. StopExecution and "
6569 "DescribeExecution are otherwise limited to runtime-generated "
6570 "execution ARNs belonging to the two regional Helm state machines."
6571 ),
6572 "appliesTo": [
6573 "Resource::*",
6574 install_execution_detail,
6575 teardown_execution_detail,
6576 ],
6577 }
6578 ],
6579 )
6580 acknowledge_nag_findings(
6581 self.helm_teardown_provider,
6582 [
6583 {
6584 "id": "AwsSolutions-IAM5",
6585 "reason": (
6586 "The CDK provider framework invokes only versioned onEvent/isComplete "
6587 "handlers and its generated waiter invokes only its versioned timeout "
6588 "and completion handlers; every wildcard is a Lambda qualifier."
6589 ),
6590 "appliesTo": [
6591 "Resource::<HelmTeardownIsComplete5ECB4605.Arn>:*",
6592 "Resource::<HelmTeardownOnEvent3DB6F756.Arn>:*",
6593 ("Resource::<HelmTeardownProviderframeworkisComplete3D7339F4.Arn>:*"),
6594 ("Resource::<HelmTeardownProviderframeworkonTimeout3415E5E9.Arn>:*"),
6595 ],
6596 },
6597 {
6598 "id": "AwsSolutions-SF1",
6599 "reason": (
6600 "The provider waiter state machine is generated by CDK and does not "
6601 "expose logging configuration."
6602 ),
6603 },
6604 {
6605 "id": "AwsSolutions-SF2",
6606 "reason": (
6607 "The provider waiter state machine is generated by CDK and does not "
6608 "expose tracing configuration."
6609 ),
6610 },
6611 {
6612 "id": "Serverless-StepFunctionStateMachineXray",
6613 "reason": (
6614 "The provider waiter state machine is generated by CDK and does not "
6615 "expose tracing configuration."
6616 ),
6617 },
6618 ],
6619 )
6621 def _create_efs(self) -> None:
6622 """Create EFS file system for shared storage across jobs.
6624 Creates an EFS file system with mount targets in each private subnet,
6625 allowing pods to share data and persist outputs. The EFS is configured
6626 with:
6627 - Encryption at rest
6628 - Automatic backups (disabled only for disposable live validation)
6629 - General Purpose performance mode (suitable for most workloads)
6630 - Bursting throughput mode
6632 Kubernetes resources (StorageClass, PV, PVC) are created via manifests.
6633 """
6634 project_name = self.config.get_project_name()
6636 # Create security group for EFS
6637 self.efs_security_group = ec2.SecurityGroup(
6638 self,
6639 "EfsSecurityGroup",
6640 vpc=self.vpc,
6641 description=f"Security group for {project_name} EFS in {self.deployment_region}",
6642 security_group_name=f"{project_name}-efs-sg-{self.deployment_region}",
6643 allow_all_outbound=False, # EFS doesn't need outbound
6644 )
6646 # Allow NFS traffic from EKS cluster security group
6647 self.efs_security_group.add_ingress_rule(
6648 peer=self.cluster.cluster_security_group,
6649 connection=ec2.Port.tcp(2049),
6650 description="Allow NFS from EKS cluster",
6651 )
6653 # Create EFS file system
6654 self.efs_file_system = efs.FileSystem(
6655 self,
6656 "GCOEfs",
6657 vpc=self.vpc,
6658 file_system_name=f"{project_name}-efs-{self.deployment_region}",
6659 security_group=self.efs_security_group,
6660 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
6661 encrypted=True,
6662 performance_mode=efs.PerformanceMode.GENERAL_PURPOSE,
6663 throughput_mode=efs.ThroughputMode.BURSTING,
6664 removal_policy=RemovalPolicy.DESTROY, # For dev/test; use RETAIN for production
6665 enable_automatic_backups=not self.disable_efs_automatic_backups,
6666 )
6667 if self.disable_efs_automatic_backups:
6668 cfn_file_system = self.efs_file_system.node.default_child
6669 if not isinstance(cfn_file_system, efs.CfnFileSystem):
6670 raise TypeError("GCOEfs default child must be AWS::EFS::FileSystem")
6671 cfn_file_system.backup_policy = efs.CfnFileSystem.BackupPolicyProperty(
6672 status="DISABLED"
6673 )
6675 # Add file system policy to allow mounting without IAM authorization
6676 # This allows any client that can reach the mount target to mount the file system
6677 self.efs_file_system.add_to_resource_policy(
6678 iam.PolicyStatement(
6679 effect=iam.Effect.ALLOW,
6680 principals=[iam.AnyPrincipal()],
6681 actions=[
6682 "elasticfilesystem:ClientMount",
6683 "elasticfilesystem:ClientWrite",
6684 "elasticfilesystem:ClientRootAccess",
6685 ],
6686 conditions={"Bool": {"elasticfilesystem:AccessedViaMountTarget": "true"}},
6687 )
6688 )
6690 # Create access point for the gco-jobs directory
6691 self.efs_access_point = self.efs_file_system.add_access_point(
6692 "JobsAccessPoint",
6693 path="/gco-jobs",
6694 create_acl=efs.Acl(owner_uid="1000", owner_gid="1000", permissions="755"),
6695 posix_user=efs.PosixUser(uid="1000", gid="1000"),
6696 )
6698 # Output EFS information
6699 CfnOutput(
6700 self,
6701 "EfsFileSystemId",
6702 value=self.efs_file_system.file_system_id,
6703 description="EFS File System ID for shared job storage",
6704 )
6706 CfnOutput(
6707 self,
6708 "EfsAccessPointId",
6709 value=self.efs_access_point.access_point_id,
6710 description="EFS Access Point ID for job outputs",
6711 )
6713 def _create_fsx_lustre(self) -> None:
6714 """Create FSx for Lustre file system for high-performance storage.
6716 FSx for Lustre provides high-performance parallel file system storage
6717 ideal for ML training workloads that require high throughput and low latency.
6719 This is optional and controlled by the fsx_lustre.enabled config setting.
6721 Supported deployment types:
6722 - SCRATCH_1: Temporary storage, no data replication
6723 - SCRATCH_2: Temporary storage with better burst performance
6724 - PERSISTENT_1: Persistent storage with data replication
6725 - PERSISTENT_2: Latest persistent storage with higher throughput
6726 """
6727 fsx_config = self.config.get_fsx_lustre_config(self.deployment_region)
6729 if not fsx_config.get("enabled", False):
6730 self.fsx_file_system = None
6731 return
6733 project_name = self.config.get_project_name()
6735 # Create security group for FSx
6736 self.fsx_security_group = ec2.SecurityGroup(
6737 self,
6738 "FsxSecurityGroup",
6739 vpc=self.vpc,
6740 description=f"Security group for {project_name} FSx Lustre in {self.deployment_region}",
6741 security_group_name=f"{project_name}-fsx-sg-{self.deployment_region}",
6742 allow_all_outbound=False,
6743 )
6745 # Allow Lustre traffic from EKS cluster security group
6746 # Lustre uses ports 988 (control) and 1021-1023 (data)
6747 self.fsx_security_group.add_ingress_rule(
6748 peer=self.cluster.cluster_security_group,
6749 connection=ec2.Port.tcp(988),
6750 description="Allow Lustre control traffic from EKS cluster",
6751 )
6752 self.fsx_security_group.add_ingress_rule(
6753 peer=self.cluster.cluster_security_group,
6754 connection=ec2.Port.tcp_range(1021, 1023),
6755 description="Allow Lustre data traffic from EKS cluster",
6756 )
6758 # Allow self-referencing traffic for FSx Lustre internal communication
6759 # FSx Lustre nodes need to communicate with each other on port 988
6760 self.fsx_security_group.add_ingress_rule(
6761 peer=self.fsx_security_group,
6762 connection=ec2.Port.tcp(988),
6763 description="Allow Lustre internal traffic on port 988",
6764 )
6765 self.fsx_security_group.add_ingress_rule(
6766 peer=self.fsx_security_group,
6767 connection=ec2.Port.tcp_range(1021, 1023),
6768 description="Allow Lustre internal traffic on ports 1021-1023",
6769 )
6771 # Get deployment type
6772 deployment_type = fsx_config.get("deployment_type", "SCRATCH_2")
6773 storage_capacity = fsx_config.get("storage_capacity_gib", 1200)
6775 # Build Lustre configuration based on deployment type
6776 lustre_config = {
6777 "deploymentType": deployment_type,
6778 "dataCompressionType": fsx_config.get("data_compression_type", "LZ4"),
6779 }
6781 # Add throughput for PERSISTENT types
6782 if deployment_type.startswith("PERSISTENT"):
6783 lustre_config["perUnitStorageThroughput"] = fsx_config.get(
6784 "per_unit_storage_throughput", 200
6785 )
6787 # Add S3 import/export if configured
6788 import_path = fsx_config.get("import_path")
6789 export_path = fsx_config.get("export_path")
6791 if import_path:
6792 lustre_config["importPath"] = import_path
6793 lustre_config["autoImportPolicy"] = fsx_config.get(
6794 "auto_import_policy", "NEW_CHANGED_DELETED"
6795 )
6797 if export_path:
6798 lustre_config["exportPath"] = export_path
6800 # Get file system type version (default to 2.15 for kernel 6.x compatibility)
6801 # IMPORTANT: Lustre 2.10 is NOT compatible with kernel 6.x (AL2023, Bottlerocket 1.19+)
6802 # See: https://docs.aws.amazon.com/fsx/latest/LustreGuide/lustre-client-matrix.html
6803 file_system_type_version = fsx_config.get("file_system_type_version", "2.15")
6805 # Create FSx for Lustre file system
6806 self.fsx_file_system = fsx.CfnFileSystem(
6807 self,
6808 "GCOFsxLustre",
6809 file_system_type="LUSTRE",
6810 file_system_type_version=file_system_type_version,
6811 storage_capacity=storage_capacity,
6812 subnet_ids=[self.vpc.private_subnets[0].subnet_id],
6813 security_group_ids=[self.fsx_security_group.security_group_id],
6814 lustre_configuration=lustre_config,
6815 tags=[
6816 {"key": "Name", "value": f"{project_name}-fsx-{self.deployment_region}"},
6817 {"key": "Project", "value": project_name},
6818 ],
6819 )
6821 # Ensure FSx file system waits for security group ingress rules to be created
6822 # This prevents "security group does not permit Lustre LNET traffic" errors
6823 self.fsx_file_system.node.add_dependency(self.fsx_security_group)
6825 # Create FSx CSI Driver add-on for Kubernetes integration
6826 self._create_fsx_csi_driver_addon()
6828 # Output FSx information
6829 CfnOutput(
6830 self,
6831 "FsxFileSystemId",
6832 value=self.fsx_file_system.ref,
6833 description="FSx for Lustre File System ID",
6834 )
6836 CfnOutput(
6837 self,
6838 "FsxDnsName",
6839 value=self.fsx_file_system.attr_dns_name,
6840 description="FSx for Lustre DNS Name",
6841 )
6843 CfnOutput(
6844 self,
6845 "FsxMountName",
6846 value=self.fsx_file_system.attr_lustre_mount_name,
6847 description="FSx for Lustre Mount Name",
6848 )
6850 def _create_valkey_cache(self) -> None:
6851 """Create an ElastiCache Serverless Valkey cache for K/V caching.
6853 Provides a low-latency key-value store that inference endpoints and
6854 jobs can use for prompt caching, session state, feature stores, or
6855 any shared state across pods. Valkey Serverless auto-scales and
6856 requires no node management.
6858 The cache is placed in the VPC private subnets and accessible from
6859 any pod via the cluster security group.
6860 """
6861 valkey_config = self.config.get_valkey_config()
6862 if not valkey_config.get("enabled", False):
6863 return
6865 from aws_cdk import aws_elasticache as elasticache
6867 # Security group for Valkey (allow access from EKS cluster)
6868 valkey_sg = ec2.SecurityGroup(
6869 self,
6870 "ValkeySG",
6871 vpc=self.vpc,
6872 description="Security group for Valkey Serverless cache",
6873 allow_all_outbound=False,
6874 )
6875 valkey_sg.add_ingress_rule(
6876 ec2.Peer.ipv4(self.vpc.vpc_cidr_block),
6877 ec2.Port.tcp(6379),
6878 "Allow Valkey access from VPC",
6879 )
6881 # The Valkey SG ingress allows 6379 from the VPC CIDR (an ``Fn::GetAtt``
6882 # token cdk-nag can't resolve), so the SG-ingress rules throw. Scope the
6883 # acknowledgment to the Valkey SG construct itself.
6884 from gco.stacks.nag_suppressions import acknowledge_security_group_cidr_findings
6886 acknowledge_security_group_cidr_findings(
6887 valkey_sg,
6888 reason=(
6889 "The Valkey Serverless cache security group allows the Valkey "
6890 "port (6379) from the VPC CIDR only, referenced via an "
6891 "``Fn::GetAtt`` token that cdk-nag cannot resolve at synth "
6892 "time. Ingress is restricted to intra-VPC traffic from the "
6893 "job pods that use the cache."
6894 ),
6895 )
6897 # ElastiCache Serverless accepts only 2-3 subnets ("Serverless Cache
6898 # should have total subnetIds between 2 and 3" — caught live by the
6899 # example-job validation run ex241-2913b044 in us-east-1, where the
6900 # VPC's span-every-AZ layout yields six private subnets). CDK orders
6901 # ``vpc.private_subnets`` deterministically by AZ, so taking the
6902 # first three keeps the selection stable across deploys; the cache
6903 # is reachable from every subnet regardless (routing, not placement).
6904 private_subnet_ids = [s.subnet_id for s in self.vpc.private_subnets[:3]]
6906 self.valkey_cache = elasticache.CfnServerlessCache(
6907 self,
6908 "ValkeyCache",
6909 engine="valkey",
6910 serverless_cache_name=f"{self.config.get_project_name()}-{self.deployment_region}",
6911 description=f"GCO K/V cache for {self.deployment_region}",
6912 major_engine_version="8",
6913 security_group_ids=[valkey_sg.security_group_id],
6914 subnet_ids=private_subnet_ids,
6915 cache_usage_limits=elasticache.CfnServerlessCache.CacheUsageLimitsProperty(
6916 data_storage=elasticache.CfnServerlessCache.DataStorageProperty(
6917 maximum=valkey_config.get("max_data_storage_gb", 5),
6918 minimum=1,
6919 unit="GB",
6920 ),
6921 ecpu_per_second=elasticache.CfnServerlessCache.ECPUPerSecondProperty(
6922 maximum=valkey_config.get("max_ecpu_per_second", 5000),
6923 minimum=1000,
6924 ),
6925 ),
6926 snapshot_retention_limit=valkey_config.get("snapshot_retention_limit", 1),
6927 tags=[
6928 CfnTag(key="Project", value=self.config.get_project_name()),
6929 CfnTag(key="gco:project", value=self.config.get_project_name()),
6930 CfnTag(key="Region", value=self.deployment_region),
6931 ],
6932 )
6934 CfnOutput(
6935 self,
6936 "ValkeyEndpoint",
6937 value=self.valkey_cache.attr_endpoint_address,
6938 description="Valkey Serverless cache endpoint",
6939 )
6940 CfnOutput(
6941 self,
6942 "ValkeyPort",
6943 value=self.valkey_cache.attr_endpoint_port,
6944 description="Valkey Serverless cache port",
6945 )
6947 # Store endpoint in SSM for discovery by pods
6948 ssm.StringParameter(
6949 self,
6950 "ValkeyEndpointParam",
6951 parameter_name=f"/{self.config.get_project_name()}/valkey-endpoint-{self.deployment_region}",
6952 string_value=self.valkey_cache.attr_endpoint_address,
6953 description=f"Valkey endpoint for {self.deployment_region}",
6954 )
6956 def _create_aurora_pgvector(self) -> None:
6957 """Create an Aurora Serverless v2 PostgreSQL cluster with pgvector.
6959 Provides a fully managed vector database that inference endpoints and
6960 jobs can use for RAG (retrieval-augmented generation), semantic search,
6961 embedding storage, and similarity queries. Aurora Serverless v2
6962 auto-scales capacity and requires no instance management.
6964 The cluster is placed in the VPC private subnets and accessible from
6965 any pod via the cluster security group. Credentials are stored in
6966 Secrets Manager and the endpoint is published to SSM + a K8s ConfigMap
6967 for automatic discovery.
6969 See: https://aws.amazon.com/blogs/database/accelerate-generative-ai-workloads-on-amazon-aurora-with-optimized-reads-and-pgvector/
6970 """
6971 aurora_config = self.config.get_aurora_pgvector_config()
6972 if not aurora_config.get("enabled", False):
6973 return
6975 from aws_cdk import aws_rds as rds
6977 project_name = self.config.get_project_name()
6979 # Security group for Aurora (allow PostgreSQL access from EKS cluster only)
6980 aurora_sg = ec2.SecurityGroup(
6981 self,
6982 "AuroraPgvectorSG",
6983 vpc=self.vpc,
6984 description="Security group for Aurora Serverless v2 pgvector",
6985 allow_all_outbound=False,
6986 )
6987 aurora_sg.add_ingress_rule(
6988 self.cluster.cluster_security_group,
6989 ec2.Port.tcp(5432),
6990 "Allow PostgreSQL access from EKS cluster",
6991 )
6993 # Subnet group for Aurora (private subnets only)
6994 subnet_group = rds.SubnetGroup(
6995 self,
6996 "AuroraPgvectorSubnetGroup",
6997 description=f"Subnet group for GCO Aurora pgvector in {self.deployment_region}",
6998 vpc=self.vpc,
6999 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
7000 )
7002 # RDS creates the exported postgresql log group itself, outside
7003 # CloudFormation, and never deletes it — live example-job validation
7004 # run ex241-edf33111-r2 found it as the only post-teardown residue.
7005 # Pre-creating the group under the exact name RDS uses
7006 # (/aws/rds/cluster/<cluster-identifier>/<export>) hands its whole
7007 # lifecycle to this stack; RDS then writes into the existing group.
7008 aurora_log_group = logs.LogGroup(
7009 self,
7010 "AuroraPgvectorPostgresqlLogs",
7011 log_group_name=(
7012 f"/aws/rds/cluster/{project_name}-pgvector-{self.deployment_region}/postgresql"
7013 ),
7014 retention=logs.RetentionDays.ONE_MONTH,
7015 removal_policy=RemovalPolicy.DESTROY,
7016 )
7018 # Aurora Serverless v2 cluster with PostgreSQL 16 + pgvector
7019 self.aurora_cluster = rds.DatabaseCluster(
7020 self,
7021 "AuroraPgvectorCluster",
7022 engine=rds.DatabaseClusterEngine.aurora_postgres(
7023 # ``of()`` rather than a ``VER_X_Y`` enum member: the pin in
7024 # constants.py is a plain version string so an Aurora minor
7025 # bump never has to wait for an aws-cdk-lib enum release.
7026 version=rds.AuroraPostgresEngineVersion.of(
7027 AURORA_POSTGRES_VERSION,
7028 AURORA_POSTGRES_VERSION.split(".", 1)[0],
7029 ),
7030 ),
7031 serverless_v2_min_capacity=aurora_config.get("min_acu", 0),
7032 serverless_v2_max_capacity=aurora_config.get("max_acu", 16),
7033 writer=rds.ClusterInstance.serverless_v2(
7034 "Writer",
7035 auto_minor_version_upgrade=True,
7036 ),
7037 readers=[
7038 rds.ClusterInstance.serverless_v2(
7039 "Reader",
7040 auto_minor_version_upgrade=True,
7041 scale_with_writer=True,
7042 ),
7043 ],
7044 vpc=self.vpc,
7045 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
7046 subnet_group=subnet_group,
7047 security_groups=[aurora_sg],
7048 default_database_name="gco_vectors",
7049 backup=rds.BackupProps(
7050 retention=Duration.days(aurora_config.get("backup_retention_days", 7)),
7051 ),
7052 deletion_protection=aurora_config.get("deletion_protection", False),
7053 removal_policy=RemovalPolicy.DESTROY,
7054 storage_encrypted=True,
7055 iam_authentication=True,
7056 cloudwatch_logs_exports=["postgresql"],
7057 monitoring_interval=Duration.seconds(60),
7058 cluster_identifier=f"{project_name}-pgvector-{self.deployment_region}",
7059 )
7060 self.aurora_secret = cast(secretsmanager.ISecret, self.aurora_cluster.secret)
7061 # The group must exist before the cluster starts exporting, and must
7062 # outlive it on delete (reverse order) so late writes cannot recreate
7063 # an unowned group.
7064 self.aurora_cluster.node.add_dependency(aurora_log_group)
7066 # aws-cdk-lib >= 2.262 ships a built-in "CloudFormation Validate"
7067 # pack whose W9008 wants StorageEncrypted on every CfnDBInstance. The
7068 # cluster above sets storage_encrypted=True, and Aurora cluster
7069 # members inherit the cluster's storage encryption — the
7070 # instance-level property is not applicable to Aurora members, so the
7071 # finding cannot be satisfied at the instance. ``Validations.acknowledge``
7072 # is the API that feeds the validation report's suppression pass
7073 # (``Annotations.acknowledge_warning`` only silences the console
7074 # annotation). Note: the current CDK implementation collects these
7075 # acknowledgments app-wide per rule ID, so this quiets W9008
7076 # everywhere — attaching it here records this cluster as the
7077 # provenance in the report, and the five cdk-nag packs' own
7078 # RDS storage-encryption rules remain scoped and would still fail
7079 # a genuinely unencrypted instance elsewhere.
7080 Validations.of(self.aurora_cluster).acknowledge(
7081 Acknowledgment(
7082 id="CloudFormation-Validate::W9008",
7083 reason=(
7084 "Aurora cluster members inherit the cluster's "
7085 "storage_encrypted=True; StorageEncrypted is not applicable "
7086 "on Aurora member DBInstances."
7087 ),
7088 )
7089 )
7091 # E9006 checks EngineVersion against the enum embedded in the CDK's
7092 # bundled CloudFormation resource spec, which lags new Aurora minor
7093 # releases (at 17.10's release the spec listed 17.9 and even 18.3,
7094 # but not 17.10). The pin in constants.py is validated against the
7095 # authoritative source instead: the monthly dependency scan compares
7096 # it with live ``rds describe-db-engine-versions`` output, and the
7097 # live release validation deploys it for real. Same app-wide
7098 # collection caveat as W9008 above; the compensating controls are
7099 # those live checks, which a stale spec enum cannot see.
7100 Validations.of(self.aurora_cluster).acknowledge(
7101 Acknowledgment(
7102 id="CloudFormation-Validate::E9006",
7103 reason=(
7104 "EngineVersion is validated against live RDS (monthly "
7105 "dependency scan + live release validation); the CDK's "
7106 "embedded CloudFormation spec enum lags new Aurora minors."
7107 ),
7108 )
7109 )
7111 # Construct-level cdk-nag suppressions for Aurora pgvector
7112 from gco.stacks.nag_suppressions import NagSuppression, acknowledge_nag_findings
7114 acknowledge_nag_findings(
7115 self.aurora_cluster,
7116 [
7117 NagSuppression(
7118 id="AwsSolutions-RDS10",
7119 reason=(
7120 "Deletion protection is intentionally disabled for dev/test deployments. "
7121 "Production deployments should set aurora_pgvector.deletion_protection=true "
7122 "in cdk.json."
7123 ),
7124 ),
7125 NagSuppression(
7126 id="AwsSolutions-SMG4",
7127 reason=(
7128 "Aurora manages credential rotation via the RDS integration with Secrets "
7129 "Manager. Manual Secrets Manager rotation is not required. "
7130 "See: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/rds-secrets-manager.html"
7131 ),
7132 ),
7133 NagSuppression(
7134 id="HIPAA.Security-RDSInstanceDeletionProtectionEnabled",
7135 reason=(
7136 "Deletion protection is intentionally disabled for dev/test deployments. "
7137 "Production deployments should set aurora_pgvector.deletion_protection=true "
7138 "in cdk.json."
7139 ),
7140 ),
7141 NagSuppression(
7142 id="NIST.800.53.R5-RDSInstanceDeletionProtectionEnabled",
7143 reason=(
7144 "Deletion protection is intentionally disabled for dev/test deployments. "
7145 "Production deployments should set aurora_pgvector.deletion_protection=true "
7146 "in cdk.json."
7147 ),
7148 ),
7149 NagSuppression(
7150 id="PCI.DSS.321-SecretsManagerUsingKMSKey",
7151 reason=(
7152 "Aurora Serverless v2 credentials in Secrets Manager are encrypted with "
7153 "AWS-managed keys by default. Customer-managed KMS can be enabled if "
7154 "required for PCI compliance."
7155 ),
7156 ),
7157 ],
7158 )
7160 # Outputs
7161 CfnOutput(
7162 self,
7163 "AuroraPgvectorEndpoint",
7164 value=self.aurora_cluster.cluster_endpoint.hostname,
7165 description="Aurora pgvector cluster writer endpoint",
7166 )
7167 CfnOutput(
7168 self,
7169 "AuroraPgvectorReaderEndpoint",
7170 value=self.aurora_cluster.cluster_read_endpoint.hostname,
7171 description="Aurora pgvector cluster reader endpoint",
7172 )
7173 CfnOutput(
7174 self,
7175 "AuroraPgvectorPort",
7176 value=str(self.aurora_cluster.cluster_endpoint.port),
7177 description="Aurora pgvector cluster port",
7178 )
7179 CfnOutput(
7180 self,
7181 "AuroraPgvectorSecretArn",
7182 value=self.aurora_secret.secret_arn,
7183 description="Aurora pgvector credentials secret ARN",
7184 )
7186 # Store endpoint in SSM for discovery by pods and external tools
7187 ssm.StringParameter(
7188 self,
7189 "AuroraPgvectorEndpointParam",
7190 parameter_name=f"/{project_name}/aurora-pgvector-endpoint-{self.deployment_region}",
7191 string_value=self.aurora_cluster.cluster_endpoint.hostname,
7192 description=f"Aurora pgvector endpoint for {self.deployment_region}",
7193 )
7195 # Grant the ServiceAccountRole read access to the Aurora secret
7196 # so pods can retrieve credentials via the ConfigMap + Secrets Manager.
7197 self.aurora_secret.grant_read(self.service_account_role)
7199 def _create_fsx_csi_driver_addon(self) -> None:
7200 """Create FSx CSI Driver add-on for Kubernetes integration.
7202 The FSx CSI driver enables Kubernetes pods to mount FSx for Lustre
7203 file systems as persistent volumes.
7204 """
7205 # Create IAM role for FSx CSI Driver using IRSA + Pod Identity
7206 self.fsx_csi_role = GCORegionalStack._create_irsa_role(
7207 self,
7208 "FsxCsiDriverRole",
7209 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
7210 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
7211 service_account_names=["fsx-csi-controller-sa"],
7212 namespaces=["kube-system"],
7213 )
7215 # Add FSx CSI driver permissions
7216 self.fsx_csi_role.add_to_policy(
7217 iam.PolicyStatement(
7218 effect=iam.Effect.ALLOW,
7219 actions=[
7220 "fsx:DescribeFileSystems",
7221 "fsx:DescribeVolumes",
7222 "fsx:CreateVolume",
7223 "fsx:DeleteVolume",
7224 "fsx:TagResource",
7225 ],
7226 resources=["*"],
7227 )
7228 )
7230 self.fsx_csi_role.add_to_policy(
7231 iam.PolicyStatement(
7232 effect=iam.Effect.ALLOW,
7233 actions=[
7234 "ec2:DescribeInstances",
7235 "ec2:DescribeVolumes",
7236 "ec2:DescribeVpcs",
7237 "ec2:DescribeSubnets",
7238 "ec2:DescribeSecurityGroups",
7239 ],
7240 resources=["*"],
7241 )
7242 )
7244 # cdk-nag suppression: the FSx CSI driver role grants
7245 # ec2:Describe* APIs that don't support resource-level scoping.
7246 from gco.stacks.nag_suppressions import acknowledge_nag_findings
7248 acknowledge_nag_findings(
7249 self.fsx_csi_role,
7250 [
7251 {
7252 "id": "AwsSolutions-IAM5",
7253 "reason": (
7254 "The FSx CSI driver role grants ec2:Describe* for volume "
7255 "and network discovery. These AWS APIs do not support "
7256 "resource-level IAM scoping — Resource: * is the only "
7257 "valid form."
7258 ),
7259 "appliesTo": ["Resource::*"],
7260 },
7261 ],
7262 )
7264 # Create FSx CSI Driver add-on
7265 fsx_addon = eks.Addon(
7266 self,
7267 "FsxCsiDriverAddon",
7268 cluster=self.cluster, # type: ignore[arg-type]
7269 addon_name="aws-fsx-csi-driver",
7270 addon_version=EKS_ADDON_FSX_CSI_DRIVER,
7271 preserve_on_delete=False,
7272 configuration_values={
7273 # DaemonSet node agent must run on every node type; the
7274 # Deployment-shaped controller deliberately carries no
7275 # accelerator tolerations (see _ADDON_NODE_TOLERATIONS).
7276 "node": {
7277 "tolerations": self._ADDON_NODE_TOLERATIONS,
7278 },
7279 },
7280 )
7282 # Append the PassRole statement for the FSx CSI role to the shared
7283 # AwsCustomResource execution role. See
7284 # _create_aws_custom_resource_role for the full rationale.
7285 self.aws_custom_resource_role.add_to_policy(
7286 iam.PolicyStatement(
7287 effect=iam.Effect.ALLOW,
7288 actions=["iam:PassRole"],
7289 resources=[self.fsx_csi_role.role_arn],
7290 )
7291 )
7293 # Update the add-on to use the IRSA role
7294 update_fsx_addon = cr.AwsCustomResource(
7295 self,
7296 "UpdateFsxCsiAddonRole",
7297 on_create=cr.AwsSdkCall(
7298 service="EKS",
7299 action="updateAddon",
7300 parameters={
7301 "clusterName": self.cluster.cluster_name,
7302 "addonName": "aws-fsx-csi-driver",
7303 "serviceAccountRoleArn": self.fsx_csi_role.role_arn,
7304 },
7305 physical_resource_id=cr.PhysicalResourceId.of(
7306 f"{self.cluster.cluster_name}-fsx-csi-role-update"
7307 ),
7308 ),
7309 on_update=cr.AwsSdkCall(
7310 service="EKS",
7311 action="updateAddon",
7312 parameters={
7313 "clusterName": self.cluster.cluster_name,
7314 "addonName": "aws-fsx-csi-driver",
7315 "serviceAccountRoleArn": self.fsx_csi_role.role_arn,
7316 },
7317 ),
7318 role=self.aws_custom_resource_role,
7319 )
7321 update_fsx_addon.node.add_dependency(fsx_addon)
7322 update_fsx_addon.node.add_dependency(self.fsx_csi_role)
7323 update_fsx_addon.node.add_dependency(self.aws_custom_resource_role)
7325 # Expose the update-addon resource so _apply_kubernetes_manifests can
7326 # make the kubectl Lambda wait for the IRSA annotation patch to land
7327 # before it rollout-restarts the fsx-csi-controller. See the EFS CSI
7328 # equivalent for the full rationale — same race, same fix, same
7329 # symptom (PVCs stuck Pending with "no EC2 IMDS role found").
7330 self._fsx_csi_addon_role_update = update_fsx_addon
7332 # Create Pod Identity Association for FSx CSI driver
7333 eks_l1.CfnPodIdentityAssociation(
7334 self,
7335 "PodIdentity-fsx-csi",
7336 cluster_name=self.cluster.cluster_name,
7337 namespace="kube-system",
7338 service_account="fsx-csi-controller-sa",
7339 role_arn=self.fsx_csi_role.role_arn,
7340 )
7342 def _create_drift_detection(self) -> None:
7343 """Create CloudFormation drift detection on a daily schedule.
7345 Creates:
7346 - SNS topic (KMS-encrypted) for drift alerts
7347 - Lambda function that initiates drift detection on this stack, polls
7348 until detection completes, and publishes to SNS if drift is found
7349 - EventBridge rule on a daily schedule (configurable via cdk.json
7350 ``drift_detection.schedule_hours``) that invokes the Lambda
7352 Operators can disable drift detection entirely by setting
7353 ``drift_detection.enabled`` to ``false`` in cdk.json. When disabled,
7354 no resources are created.
7355 """
7356 drift_config = self.node.try_get_context("drift_detection") or {}
7357 if not drift_config.get("enabled", True):
7358 return
7360 schedule_hours = int(drift_config.get("schedule_hours", 24))
7362 # KMS key for SNS topic encryption. SNS with AWS-managed keys doesn't
7363 # allow CloudFormation/Lambda to publish, so we use a customer-managed
7364 # key we can grant publish access on.
7365 drift_topic_key = kms.Key(
7366 self,
7367 "DriftDetectionTopicKey",
7368 description="KMS key for GCO drift detection SNS topic",
7369 enable_key_rotation=True,
7370 removal_policy=RemovalPolicy.DESTROY,
7371 )
7373 self.drift_detection_topic = sns.Topic(
7374 self,
7375 "DriftDetectionTopic",
7376 display_name="GCO CloudFormation Drift Alerts",
7377 master_key=drift_topic_key,
7378 )
7380 # IAM role for the drift detection Lambda
7381 drift_lambda_role = iam.Role(
7382 self,
7383 "DriftDetectionLambdaRole",
7384 assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
7385 managed_policies=[
7386 iam.ManagedPolicy.from_aws_managed_policy_name(
7387 "service-role/AWSLambdaBasicExecutionRole"
7388 ),
7389 ],
7390 )
7392 # CloudFormation drift APIs operate at the stack level; the API does
7393 # not support resource-level ARN scoping for these actions, so we scope
7394 # to this stack's ARN where supported and accept "*" where not.
7395 drift_lambda_role.add_to_policy(
7396 iam.PolicyStatement(
7397 effect=iam.Effect.ALLOW,
7398 actions=[
7399 "cloudformation:DetectStackDrift",
7400 "cloudformation:DescribeStackDriftDetectionStatus",
7401 "cloudformation:DescribeStackResourceDrifts",
7402 "cloudformation:DescribeStackResource",
7403 "cloudformation:DescribeStackResources",
7404 ],
7405 resources=["*"],
7406 )
7407 )
7409 self.drift_detection_topic.grant_publish(drift_lambda_role)
7411 # Lambda function — one per stack; stack name is baked into env vars
7412 drift_lambda = lambda_.Function(
7413 self,
7414 "DriftDetectionFunction",
7415 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
7416 handler="handler.lambda_handler",
7417 code=lambda_source_code("lambda/drift-detection"),
7418 timeout=Duration.minutes(14), # Leave headroom under Lambda 15-min cap
7419 memory_size=256,
7420 role=drift_lambda_role,
7421 environment={
7422 "STACK_NAME": self.stack_name,
7423 "SNS_TOPIC_ARN": self.drift_detection_topic.topic_arn,
7424 "REGION": self.deployment_region,
7425 },
7426 tracing=lambda_.Tracing.ACTIVE,
7427 )
7429 # Dead-letter queue for EventBridge → Lambda target failures.
7430 # Captures events that fail to reach the Lambda (e.g. due to
7431 # throttling or permission issues) so operators can retry or
7432 # investigate. Required by Serverless-EventBusDLQ cdk-nag rule.
7433 drift_rule_dlq = sqs.Queue(
7434 self,
7435 "DriftDetectionRuleDlq",
7436 retention_period=Duration.days(14),
7437 enforce_ssl=True,
7438 encryption=sqs.QueueEncryption.SQS_MANAGED,
7439 removal_policy=RemovalPolicy.DESTROY,
7440 )
7442 # DLQs themselves are terminal — they don't need their own DLQ.
7443 # Suppress the circular AwsSolutions-SQS3 nag finding.
7444 from gco.stacks.nag_suppressions import acknowledge_nag_findings
7446 acknowledge_nag_findings(
7447 drift_rule_dlq,
7448 [
7449 {
7450 "id": "AwsSolutions-SQS3",
7451 "reason": (
7452 "This queue IS the dead-letter queue for the "
7453 "DriftDetectionSchedule EventBridge rule. A DLQ for a "
7454 "DLQ is circular; if events fail to reach this queue "
7455 "they are captured by EventBridge's own retry metrics "
7456 "(CloudWatch FailedInvocations)."
7457 ),
7458 },
7459 ],
7460 )
7462 # EventBridge rule — daily schedule by default
7463 events.Rule(
7464 self,
7465 "DriftDetectionSchedule",
7466 description=(f"Daily CloudFormation drift detection for {self.stack_name}"),
7467 schedule=events.Schedule.rate(Duration.hours(schedule_hours)),
7468 targets=[
7469 events_targets.LambdaFunction(
7470 drift_lambda,
7471 dead_letter_queue=drift_rule_dlq,
7472 retry_attempts=2,
7473 )
7474 ],
7475 )
7477 # Outputs for operators to subscribe to the topic
7478 CfnOutput(
7479 self,
7480 "DriftDetectionTopicArn",
7481 value=self.drift_detection_topic.topic_arn,
7482 description=(
7483 f"SNS topic ARN for CloudFormation drift alerts in "
7484 f"{self.deployment_region}. Subscribe an endpoint (email, "
7485 f"Slack, PagerDuty) to receive drift notifications."
7486 ),
7487 )
7489 # cdk-nag suppressions for this component
7490 from gco.stacks.nag_suppressions import acknowledge_nag_findings
7492 acknowledge_nag_findings(
7493 drift_lambda_role,
7494 [
7495 {
7496 "id": "AwsSolutions-IAM4",
7497 "reason": (
7498 "AWSLambdaBasicExecutionRole provides standard "
7499 "CloudWatch Logs permissions required for Lambda "
7500 "logging. This is the AWS-recommended managed policy."
7501 ),
7502 },
7503 {
7504 "id": "AwsSolutions-IAM5",
7505 "reason": (
7506 "CloudFormation drift detection APIs (DetectStackDrift, "
7507 "DescribeStackDriftDetectionStatus, "
7508 "DescribeStackResourceDrifts) cannot be scoped to a "
7509 "specific stack resource via IAM; the action-level "
7510 "scoping requires wildcard resources. The Lambda's "
7511 "environment pins it to a single stack name, so the "
7512 "effective blast radius is limited. The "
7513 "``kms:GenerateDataKey*`` action wildcard is the "
7514 "AWS-recommended grant for publishing to the "
7515 "KMS-encrypted drift-detection SNS topic."
7516 ),
7517 "appliesTo": [
7518 "Resource::*",
7519 "Action::kms:GenerateDataKey*",
7520 ],
7521 },
7522 ],
7523 )
7525 def _create_mcp_role(self) -> None:
7526 """Create dedicated IAM role for the MCP server.
7528 The MCP server exposes GCO CLI tools to LLM agents. Without a dedicated
7529 role, the server would inherit the full ambient credentials of the user
7530 who launches it (often an administrator). This method creates a
7531 least-privilege role that the MCP server can assume at startup via
7532 ``GCO_MCP_ROLE_ARN``.
7534 Permissions are scoped to the minimum needed by the tools exposed:
7536 - ``eks:DescribeCluster`` on this regional EKS cluster ARN only.
7537 - ``s3:GetObject`` on model weights buckets. The model bucket lives in
7538 the global stack, so we scope to the same name pattern used by the
7539 service account role (``{project_name}-*``). This is a deliberate
7540 compromise: a precise cross-stack ARN export would force a tight
7541 dependency on the global stack, and cdk-nag will flag it anyway
7542 because the bucket name is auto-generated.
7543 - ``cloudwatch:GetMetricData`` / ``cloudwatch:ListMetrics``. These APIs
7544 do not support resource-level IAM, so wildcard is required. Read-only.
7545 - ``sqs:SendMessage`` scoped to this region's job queue ARN only.
7547 The trust policy uses ``AccountRootPrincipal`` so any IAM user/role in
7548 the account can assume it (gated by an explicit sts:AssumeRole
7549 permission on the caller — standard AWS behavior). Operators who want
7550 to restrict assumption further should add an external-id or principal
7551 condition to the trust policy after deployment.
7553 Operators can disable this component entirely by setting
7554 ``mcp_server.enabled`` to ``false`` in cdk.json.
7555 """
7556 mcp_config = self.node.try_get_context("mcp_server") or {}
7557 if not mcp_config.get("enabled", True):
7558 return
7560 project_name = self.config.get_project_name()
7562 self.mcp_server_role = iam.Role(
7563 self,
7564 "McpServerRole",
7565 assumed_by=iam.AccountRootPrincipal(),
7566 description=(
7567 "Least-privilege role assumed by the GCO MCP server at startup. "
7568 "Grants only the permissions needed by MCP tools: eks:DescribeCluster, "
7569 "s3:GetObject on model buckets, cloudwatch read-only metrics, and "
7570 "sqs:SendMessage to the regional job queue."
7571 ),
7572 max_session_duration=Duration.hours(12),
7573 )
7575 # eks:DescribeCluster on this region's cluster only
7576 self.mcp_server_role.add_to_policy(
7577 iam.PolicyStatement(
7578 effect=iam.Effect.ALLOW,
7579 actions=["eks:DescribeCluster"],
7580 resources=[self.cluster.cluster_arn],
7581 )
7582 )
7584 # s3:GetObject on model weights buckets. Bucket name is auto-generated
7585 # in the global stack, so we match the same prefix pattern used by the
7586 # service account role.
7587 self.mcp_server_role.add_to_policy(
7588 iam.PolicyStatement(
7589 effect=iam.Effect.ALLOW,
7590 actions=["s3:GetObject", "s3:ListBucket"],
7591 resources=[
7592 f"arn:{self.partition}:s3:::{project_name}-*",
7593 f"arn:{self.partition}:s3:::{project_name}-*/*",
7594 ],
7595 )
7596 )
7598 # CloudWatch read-only metrics APIs. These APIs do not support
7599 # resource-level IAM so wildcard is required.
7600 self.mcp_server_role.add_to_policy(
7601 iam.PolicyStatement(
7602 effect=iam.Effect.ALLOW,
7603 actions=[
7604 "cloudwatch:GetMetricData",
7605 "cloudwatch:GetMetricStatistics",
7606 "cloudwatch:ListMetrics",
7607 ],
7608 resources=["*"],
7609 )
7610 )
7612 # sqs:SendMessage scoped to the regional job queue only
7613 self.mcp_server_role.add_to_policy(
7614 iam.PolicyStatement(
7615 effect=iam.Effect.ALLOW,
7616 actions=["sqs:SendMessage", "sqs:GetQueueUrl", "sqs:GetQueueAttributes"],
7617 resources=[self.job_queue.queue_arn],
7618 )
7619 )
7621 # Export the role ARN so operators can set GCO_MCP_ROLE_ARN in their
7622 # MCP server environment.
7623 CfnOutput(
7624 self,
7625 "McpServerRoleArn",
7626 value=self.mcp_server_role.role_arn,
7627 description=(
7628 "IAM role ARN for the GCO MCP server. Set GCO_MCP_ROLE_ARN to "
7629 "this value when launching the MCP server so it assumes a "
7630 "least-privilege role instead of ambient credentials."
7631 ),
7632 export_name=f"{project_name}-mcp-server-role-arn-{self.deployment_region}",
7633 )
7635 # cdk-nag suppressions: CloudWatch metrics APIs cannot be scoped.
7636 from gco.stacks.nag_suppressions import acknowledge_nag_findings
7638 acknowledge_nag_findings(
7639 self.mcp_server_role,
7640 [
7641 {
7642 "id": "AwsSolutions-IAM5",
7643 "reason": (
7644 "The CloudWatch metrics APIs (GetMetricData, "
7645 "GetMetricStatistics, ListMetrics) do not support "
7646 "resource-level IAM; wildcard resource is required. "
7647 "The S3 permissions use the {project_name}-* prefix "
7648 "pattern because the model weights bucket name is "
7649 "auto-generated by CDK in the global stack and a "
7650 "cross-stack ARN export would create tight stack "
7651 "coupling. All actions are read-only or scoped "
7652 "send-only (SQS)."
7653 ),
7654 "appliesTo": [
7655 "Resource::*",
7656 ],
7657 },
7658 ],
7659 )
7661 def _create_outputs(self) -> None:
7662 """Create CloudFormation outputs for cluster information"""
7663 project_name = self.config.get_project_name()
7665 # Export cluster information
7666 CfnOutput(
7667 self,
7668 "ClusterName",
7669 value=self.cluster.cluster_name,
7670 description=f"EKS cluster name for {self.deployment_region}",
7671 export_name=f"{project_name}-cluster-name-{self.deployment_region}",
7672 )
7674 CfnOutput(
7675 self,
7676 "AddonDeploymentToken",
7677 value=self.addon_deployment_token,
7678 description=(
7679 "Exact token for the asynchronous Kubernetes and Helm convergence execution"
7680 ),
7681 )
7683 CfnOutput(
7684 self,
7685 "ClusterArn",
7686 value=self.cluster.cluster_arn,
7687 description=f"EKS cluster ARN for {self.deployment_region}",
7688 export_name=f"{project_name}-cluster-arn-{self.deployment_region}",
7689 )
7691 CfnOutput(
7692 self,
7693 "ClusterEndpoint",
7694 value=self.cluster.cluster_endpoint,
7695 description=f"EKS cluster endpoint for {self.deployment_region}",
7696 export_name=f"{project_name}-cluster-endpoint-{self.deployment_region}",
7697 )
7699 CfnOutput(
7700 self,
7701 "ClusterSecurityGroupId",
7702 value=self.cluster.cluster_security_group_id,
7703 description=f"EKS cluster security group ID for {self.deployment_region}",
7704 export_name=f"{project_name}-cluster-sg-{self.deployment_region}",
7705 )
7707 CfnOutput(
7708 self,
7709 "VpcId",
7710 value=self.vpc.vpc_id,
7711 description=f"VPC ID for {self.deployment_region}",
7712 export_name=f"{project_name}-vpc-id-{self.deployment_region}",
7713 )
7715 # Export public subnet IDs for ALB
7716 public_subnet_ids = [subnet.subnet_id for subnet in self.vpc.public_subnets]
7717 CfnOutput(
7718 self,
7719 "PublicSubnetIds",
7720 value=Fn.join(",", public_subnet_ids),
7721 description=f"Public subnet IDs for ALB in {self.deployment_region}",
7722 export_name=f"{project_name}-public-subnets-{self.deployment_region}",
7723 )
7725 # Note: the ALB is created by the AWS Load Balancer Controller from the
7726 # gco-system/gco-gateway Gateway API resources; the GA registration
7727 # Lambda registers its ARN with Global Accelerator
7729 def get_cluster(self) -> eks.Cluster:
7730 """Get the EKS cluster"""
7731 return self.cluster
7733 def get_vpc(self) -> ec2.Vpc:
7734 """Get the VPC"""
7735 return self.vpc