Coverage for gco / stacks / regional_stack.py: 100.00%
1052 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-09-14 22:07 +0000
1"""
2Regional stack for GCO (Global Capacity Orchestrator on AWS) - EKS cluster and ALB per region.
4This is the largest stack in the project (~3200 lines) and creates all regional
5resources for a single AWS region. One instance is deployed per region defined
6in cdk.json.
8Resources Created:
9 VPC & Networking:
10 - VPC spanning every AZ in the region, public subnets (NAT), private subnets (EKS and ALB)
11 - 2 NAT Gateways for high availability
12 - VPC endpoints for ECR, S3, STS, Secrets Manager, SSM, CloudWatch
13 - VPC Flow Logs (CloudWatch Logs, 30-day retention)
15 EKS Cluster (Auto Mode):
16 - Managed control plane with full logging (API, Audit, Authenticator, Controller Manager, Scheduler)
17 - Built-in NodePools: system, general-purpose
18 - Custom NodePools: gpu-x86-pool, gpu-arm-pool, gpu-inference-pool,
19 gpu-efa-pool, mooncake-efa-pool, neuron-pool, cpu-general-pool
20 - IRSA roles for service accounts (Secrets Manager, SQS, DynamoDB, CloudWatch, S3, EFS)
22 Load Balancing:
23 - Internal ALB created from Gateway API resources by the self-managed
24 AWS Load Balancer Controller
25 - Always-deployed regional API bridge reaches the ALB through a VPC Lambda;
26 direct caller access is optional in ``aws`` and required elsewhere
27 - Global Accelerator endpoint registration in commercial ``aws`` only
29 Storage:
30 - EFS with dynamic provisioning (CSI driver, access points, encryption at rest + in transit)
31 - FSx for Lustre (optional, toggled via cdk.json)
32 - Valkey Serverless cache (optional)
33 - Aurora Serverless v2 with pgvector (optional)
35 Lambda Functions:
36 - kubectl-applier: applies K8s manifests during deployment
37 - helm-installer: installs Helm charts (KEDA, Volcano, KubeRay, etc.)
38 - ga-registration: registers the ALB with Global Accelerator in ``aws``
39 - regional-api-proxy: separate-stack VPC proxy used by the always-on
40 aggregation bridge and by optional direct callers in ``aws`` or the
41 required regional workload ingress in other partitions
43 Container Images:
44 - Docker image assets (CDK bootstrap asset repository, content-hash
45 tags) for health-monitor, manifest-processor, inference-proxy,
46 inference-monitor, queue-processor, cost-monitor
48 SQS:
49 - Regional job queue + dead letter queue (for gco jobs submit-sqs)
51Key Design Decisions:
52 - EKS Auto Mode handles node provisioning — no managed node groups or Karpenter provisioners
53 - NodePools use WhenEmpty consolidation for inference to avoid disrupting long-running pods
54 - IRSA (IAM Roles for Service Accounts) for least-privilege pod-level AWS access
55 - All optional features (FSx, Valkey, Aurora) are toggled via cdk.json context variables
56 - Template variables in K8s manifests ({{PLACEHOLDER}}) are replaced at deploy time
58Dependencies:
59 - GCOGlobalStack (partition-wide state and, in ``aws``, Global Accelerator endpoint groups)
60 - GCOApiGatewayGlobalStack (for auth secret ARN)
62Modification Guide:
63 - To add a new NodePool: add a YAML manifest in lambda/kubectl-applier-simple/manifests/ (40-49 range)
64 - To add a new service: add ECR image build here, Dockerfile in dockerfiles/, manifest in manifests/
65 - To add a new optional feature: add a cdk.json context toggle, guard with if/else in this file
66 - To change EKS version: update KUBERNETES_VERSION in constants.py
67"""
69from __future__ import annotations
71import ipaddress
72import os
73import re
74from collections.abc import Mapping
75from dataclasses import dataclass
76from datetime import UTC, datetime
77from pathlib import Path
78from typing import Any, cast
80import aws_cdk.aws_eks_v2 as eks
81import yaml
82from aws_cdk import (
83 Acknowledgment,
84 Annotations,
85 CfnJson,
86 CfnOutput,
87 CfnTag,
88 CustomResource,
89 Duration,
90 Fn,
91 RemovalPolicy,
92 Stack,
93 Validations,
94)
95from aws_cdk import aws_ec2 as ec2
96from aws_cdk import aws_ecr_assets as ecr_assets
97from aws_cdk import aws_efs as efs
98from aws_cdk import aws_eks as eks_l1 # L1 constructs (CfnPodIdentityAssociation)
99from aws_cdk import aws_events as events
100from aws_cdk import aws_events_targets as events_targets
101from aws_cdk import aws_fsx as fsx
102from aws_cdk import aws_iam as iam
103from aws_cdk import aws_kms as kms
104from aws_cdk import aws_lambda as lambda_
105from aws_cdk import aws_logs as logs
106from aws_cdk import aws_s3 as s3
107from aws_cdk import aws_secretsmanager as secretsmanager
108from aws_cdk import aws_sns as sns
109from aws_cdk import aws_sqs as sqs
110from aws_cdk import aws_ssm as ssm
111from aws_cdk import aws_stepfunctions as sfn
112from aws_cdk import aws_stepfunctions_tasks as sfn_tasks
113from aws_cdk import custom_resources as cr
114from constructs import Construct
116from gco.config.config_loader import ConfigLoader
117from gco.inference_proxy_config import (
118 compute_inference_proxy_tls_replacements as _compute_inference_proxy_tls_replacements,
119)
120from gco.manifest_security_policy import validate_manifest_security_policy
121from gco.stacks.aws_load_balancer_controller_policy import (
122 aws_load_balancer_controller_policy_document,
123)
124from gco.stacks.constants import (
125 AURORA_POSTGRES_VERSION,
126 DEFAULT_MANIFEST_RESOURCE_CAPS,
127 DEFAULT_RESOURCE_QUOTA,
128 EKS_ADDON_CLOUDWATCH_OBSERVABILITY,
129 EKS_ADDON_EFS_CSI_DRIVER,
130 EKS_ADDON_FSX_CSI_DRIVER,
131 EKS_ADDON_METRICS_SERVER,
132 EKS_ADDON_POD_IDENTITY_AGENT,
133 EKS_UNSUPPORTED_AZ_IDS,
134 LAMBDA_PYTHON_RUNTIME,
135 MOONCAKE_MASTER_DEFAULT_IMAGE,
136 api_gateway_auth_secret_name,
137 backend_tls_certificate_arn_parameter_name,
138 cluster_shared_ssm_parameter_prefix,
139 cost_report_ssm_parameter_prefix,
140 parse_k8s_quantity,
141 regional_shared_ssm_parameter_prefix,
142)
144# <pyflowchart-code-diagram> BEGIN - auto-inserted, do not edit
145# Generated at (UTC): 2026-09-12T06:04:03Z
146# Generated from Git commit: e96e2c39c3626a5088651f43873dfade6a346850
147# Flowchart(s) generated from this file:
148# * ``GCORegionalStack.__init__`` -> ``diagrams/code_diagrams/gco/stacks/regional_stack.GCORegionalStack___init__.html``
149# (PNG: ``diagrams/code_diagrams/gco/stacks/regional_stack.GCORegionalStack___init__.png``)
150# * ``GCORegionalStack._get_volcano_image_mirror_config`` -> ``diagrams/code_diagrams/gco/stacks/regional_stack.GCORegionalStack__get_volcano_image_mirror_config.html``
151# (PNG: ``diagrams/code_diagrams/gco/stacks/regional_stack.GCORegionalStack__get_volcano_image_mirror_config.png``)
152# Regenerate with ``SOURCE_DATE_EPOCH=<unix-seconds> GCO_DIAGRAM_SOURCE_COMMIT=<40-char-sha> python diagrams/generate.py --code-only``.
153# <pyflowchart-code-diagram> END
156_LIVE_VALIDATION_PROVIDER_LOG_CONTEXT = "gco_live_validation_retain_provider_log_groups"
159@dataclass(frozen=True)
160class SharedBucketIdentity:
161 """Identity of the always-on ``Cluster_Shared_Bucket`` owned by ``GCOGlobalStack``.
163 Every regional stack resolves this identity from the three SSM parameters
164 ``/gco/cluster-shared-bucket/{name,arn,region}`` published by
165 ``GCOGlobalStack`` in the global region. The three values are used to
166 grant IAM permissions on the bucket to the regional job-pod role and to
167 populate the ``gco-cluster-shared-bucket`` ConfigMap applied to every
168 regional EKS cluster. Frozen so it can be safely shared across helper
169 methods without accidental mutation.
170 """
172 name: str
173 arn: str
174 region: str
177def _compute_kubectl_cluster_shared_replacements(
178 shared: SharedBucketIdentity,
179) -> dict[str, str]:
180 """Build the ``{{CLUSTER_SHARED_BUCKET*}}`` kubectl-applier replacements.
182 Pure helper kept at module scope so property and presence tests can
183 inspect the output without synthesizing a full regional stack. The
184 three keys are always populated — there is no feature toggle — because
185 the ``gco-cluster-shared-bucket`` ConfigMap is applied unconditionally
186 on every regional cluster.
187 """
188 return {
189 "{{CLUSTER_SHARED_BUCKET}}": shared.name,
190 "{{CLUSTER_SHARED_BUCKET_ARN}}": shared.arn,
191 "{{CLUSTER_SHARED_BUCKET_REGION}}": shared.region,
192 }
195def _compute_kubectl_regional_shared_replacements(
196 name: str,
197 arn: str,
198 region: str,
199) -> dict[str, str]:
200 """Build the ``{{REGIONAL_SHARED_BUCKET*}}`` kubectl-applier replacements.
202 Pure helper kept at module scope so property and presence tests can
203 inspect the output without synthesizing a full regional stack, mirroring
204 :func:`_compute_kubectl_cluster_shared_replacements`.
206 The three keys are always populated — there is no feature toggle —
207 because ``_create_regional_shared_bucket`` provisions the bucket
208 unconditionally, so the ``gco-regional-shared-bucket`` ConfigMap is
209 applied on every regional cluster and is never gated out of the
210 applier by an unresolved placeholder.
212 Unlike the cluster-shared helper this takes the three values directly
213 rather than a :class:`SharedBucketIdentity`: the regional bucket is a
214 local construct in this stack, so its name/ARN are CDK tokens resolved
215 at deploy time instead of values read back from cross-region SSM.
216 """
217 return {
218 "{{REGIONAL_SHARED_BUCKET}}": name,
219 "{{REGIONAL_SHARED_BUCKET_ARN}}": arn,
220 "{{REGIONAL_SHARED_BUCKET_REGION}}": region,
221 }
224#: StorageClass name for in-cluster observability PVCs (Prometheus, Grafana,
225#: Alertmanager). The value overrides reference this name, and the gated gp3
226#: StorageClass manifest (25-storage-observability-gp3.yaml) declares it. A
227#: synth test asserts the two stay in lockstep. The manifest keeps this name
228#: static (a placeholder in ``metadata.name`` would fail k8s schema
229#: validation), so the toggle gate lives in an annotation value instead.
230_OBSERVABILITY_STORAGE_CLASS = "gco-observability-gp3"
233#: In-cluster names clients use to reach the MLflow tracking server. MLflow
234#: 3.x's host-validation middleware matches the raw Host header (port
235#: included — that is why both spellings are listed), and setting
236#: ``allowed-hosts`` REPLACES its built-in localhost/private-IP allowance
237#: rather than extending it, so the value override must carry the complete
238#: list (see ``_mlflow_allowed_hosts``).
239_MLFLOW_SERVICE_HOSTS = ("mlflow.monitoring", "mlflow.monitoring:5000")
241#: Loopback spellings a browser sends through the access tunnel.
242#:
243#: ``gco monitoring open --service mlflow`` is the ONLY human path to this
244#: server (ClusterIP, no Ingress), and it port-forwards to localhost — so the
245#: browser sends ``Host: localhost:5000`` or ``Host: 127.0.0.1:5000``. Because
246#: the flag replaces MLflow's built-in loopback allowance instead of extending
247#: it, omitting these makes the documented UI path answer 403 "possible DNS
248#: rebinding attack detected" while the server is perfectly healthy — the
249#: in-cluster DNS spellings return 200 through the very same tunnel (caught
250#: live 2026-08-15, taking the release screenshot).
251#:
252#: This restores upstream's own loopback posture rather than widening it: the
253#: rebinding attack these checks exist for is an external DNS name resolving
254#: to an internal address, which loopback literals cannot express. Reaching
255#: the port at all still requires the authenticated SSM tunnel, and arbitrary
256#: DNS names stay rejected.
257_MLFLOW_TUNNEL_HOSTS = ("localhost", "localhost:5000", "127.0.0.1", "127.0.0.1:5000")
260def _mlflow_allowed_hosts(vpc_endpoint_cidrs: list[str]) -> str:
261 """Compose the MLflow ``allowed-hosts`` list from the deployment's CIDRs.
263 The service-DNS and loopback spellings are static; the IP tail derives
264 from ``vpc_endpoint_cidrs`` (single source of truth — the same context key
265 the NetworkPolicy egress rules render) so widening the VPC range never
266 needs a matching charts.yaml edit. Every group is load-bearing:
267 Prometheus scrapes the pod IP directly, so dropping the pod-IP allowance
268 403s every ServiceMonitor scrape (caught live 2026-08-14); the tunnel
269 forwards to loopback, so dropping those 403s the only human UI path
270 (caught live 2026-08-15).
272 MLflow's allow-list is glob-based, so only octet-aligned prefixes convert
273 exactly; other masks WIDEN to the containing octet boundary (capped at
274 /24 granularity so the trailing ``.*`` still matches ``host:port``
275 Host headers). Widening is the safe direction — this is Host-header
276 hygiene layered over NetworkPolicies and a private ALB, and
277 under-matching is what breaks scrapes.
278 """
279 patterns: list[str] = []
280 for cidr in vpc_endpoint_cidrs:
281 network = ipaddress.ip_network(cidr, strict=False)
282 if network.version != 4:
283 raise ValueError(
284 f"vpc_endpoint_cidrs entry {cidr!r} is not IPv4; the MLflow "
285 "allowed-hosts derivation only understands IPv4 globs"
286 )
287 octets = str(network.network_address).split(".")
288 kept = min(max(network.prefixlen // 8, 1), 3)
289 patterns.append(".".join(octets[:kept]) + ".*")
290 return ",".join(dict.fromkeys([*_MLFLOW_SERVICE_HOSTS, *_MLFLOW_TUNNEL_HOSTS, *patterns]))
293_SERVICE_IMAGE_BUILD_INPUTS = (
294 "dockerfiles/health-monitor-dockerfile",
295 "dockerfiles/manifest-processor-dockerfile",
296 "dockerfiles/inference-proxy-dockerfile",
297 "dockerfiles/inference-monitor-dockerfile",
298 "dockerfiles/queue-processor-dockerfile",
299 "dockerfiles/cost-monitor-dockerfile",
300)
301_SERVICE_IMAGE_COMMON_EXCLUDES = (
302 "cli/**",
303 "gco/stacks/**",
304 "dockerfiles/README.md",
305)
308def _service_image_asset_excludes(*included_paths: str) -> list[str]:
309 """Exclude inputs that cannot affect one production service image."""
310 included = set(included_paths)
311 return list(_SERVICE_IMAGE_COMMON_EXCLUDES) + [
312 path for path in _SERVICE_IMAGE_BUILD_INPUTS if path not in included
313 ]
316#: cdk.json ``vpc_endpoints`` service keys → CDK endpoint services. The key
317#: sets are pinned to ``ConfigLoader``'s validation lists by
318#: ``tests/test_regional_stack.py`` so a name the loader accepts always maps.
319_GATEWAY_ENDPOINT_SERVICES: dict[str, ec2.GatewayVpcEndpointAwsService] = {
320 "s3": ec2.GatewayVpcEndpointAwsService.S3,
321 "dynamodb": ec2.GatewayVpcEndpointAwsService.DYNAMODB,
322}
323_INTERFACE_ENDPOINT_SERVICES: dict[str, ec2.InterfaceVpcEndpointAwsService] = {
324 "sts": ec2.InterfaceVpcEndpointAwsService.STS,
325 "ecr.api": ec2.InterfaceVpcEndpointAwsService.ECR,
326 "ecr.dkr": ec2.InterfaceVpcEndpointAwsService.ECR_DOCKER,
327 "logs": ec2.InterfaceVpcEndpointAwsService.CLOUDWATCH_LOGS,
328 "monitoring": ec2.InterfaceVpcEndpointAwsService.CLOUDWATCH_MONITORING,
329 "sqs": ec2.InterfaceVpcEndpointAwsService.SQS,
330 "ssm": ec2.InterfaceVpcEndpointAwsService.SSM,
331 "secretsmanager": ec2.InterfaceVpcEndpointAwsService.SECRETS_MANAGER,
332 "kms": ec2.InterfaceVpcEndpointAwsService.KMS,
333 "eks": ec2.InterfaceVpcEndpointAwsService.EKS,
334 "elasticfilesystem": ec2.InterfaceVpcEndpointAwsService.ELASTIC_FILESYSTEM,
335 "bedrock-runtime": ec2.InterfaceVpcEndpointAwsService.BEDROCK_RUNTIME,
336}
339#: CDK context key that force-enables optional Helm charts for one deploy
340#: without editing cdk.json (comma-separated cdk.json helm-block key names,
341#: e.g. ``--context helm_enabled_overrides=yunikorn,slurm``). The live release
342#: validation harness uses this to exercise off-by-default schedulers against
343#: an otherwise pristine checkout; it is equally useful for trying one
344#: scheduler ahead of a config change. Overrides can only ENABLE — a chart
345#: disabled by an operator stays disabled unless named here.
346_HELM_OVERRIDE_CONTEXT_KEY = "helm_enabled_overrides"
348#: Live-validation-only context that prevents AWS-managed EFS automatic backups
349#: from outliving a disposable stack for the service's fixed retention window.
350#: Normal deployments omit this key and preserve automatic backups.
351_LIVE_VALIDATION_DISABLE_EFS_BACKUPS_CONTEXT = "gco_live_validation_disable_efs_automatic_backups"
354def _explicit_context_bool(raw: object, *, key: str) -> bool:
355 """Parse an optional CDK context boolean without truthy-string ambiguity."""
356 if raw is None:
357 return False
358 if isinstance(raw, bool):
359 return raw
360 if isinstance(raw, str):
361 normalized = raw.strip().casefold()
362 if normalized == "true":
363 return True
364 if normalized == "false":
365 return False
366 raise ValueError(f"{key} must be true or false")
369#: Every cdk.json helm-block key _get_enabled_charts understands. Kept in
370#: lockstep with its chart_map so an override typo fails the synth loudly
371#: instead of silently deploying without the requested chart.
372_HELM_CHART_CONFIG_KEYS = frozenset(
373 {
374 "aws_load_balancer_controller",
375 "keda",
376 "aws_efa_device_plugin",
377 "aws_neuron_device_plugin",
378 "volcano",
379 "kuberay",
380 "cert_manager",
381 "slurm",
382 "yunikorn",
383 "kubeflow_trainer",
384 "kueue",
385 }
386)
388#: Charts that are mandatory platform components; the cdk.json toggle is
389#: ignored for these (see _get_enabled_charts for the rationale).
390_MANDATORY_CHART_KEYS = frozenset({"aws_load_balancer_controller", "keda"})
393#: (container ceiling, namespace ceiling) pairs the resource-quota invariant
394#: compares; every dimension a container can request must fit the namespace.
395_RESOURCE_QUOTA_INVARIANTS = (
396 ("container_max_cpu", "max_cpu"),
397 ("container_max_memory", "max_memory"),
398 ("container_max_gpu", "max_gpu"),
399)
402def _validated_resource_quota(raw: object) -> dict[str, str]:
403 """Merge the ``resource_quota`` context over defaults and validate it.
405 The values are substituted verbatim into the gco-jobs ResourceQuota and
406 LimitRange manifests, where a typo or an incoherent pair (a per-container
407 ceiling that exceeds the namespace ceiling) previously deployed silently
408 and only surfaced as pods being forbidden at admission — with the reason
409 visible in namespace events alone. Fail the synth instead.
411 Raises:
412 ValueError: If the context is not a mapping, carries an unknown key,
413 a value that does not parse as a Kubernetes quantity, or a
414 per-container ceiling exceeding its namespace ceiling.
415 """
416 if not isinstance(raw, dict):
417 raise ValueError(
418 f"cdk.json context 'resource_quota' must be an object, got {type(raw).__name__}"
419 )
420 unknown = sorted(set(raw) - set(DEFAULT_RESOURCE_QUOTA))
421 if unknown:
422 allowed = ", ".join(sorted(DEFAULT_RESOURCE_QUOTA))
423 raise ValueError(
424 f"cdk.json context 'resource_quota' has unknown key(s) {unknown}; allowed: {allowed}"
425 )
426 merged = {key: str(raw.get(key, default)) for key, default in DEFAULT_RESOURCE_QUOTA.items()}
427 parsed: dict[str, float] = {}
428 for key, value in merged.items():
429 try:
430 parsed[key] = parse_k8s_quantity(value)
431 except ValueError as exc:
432 raise ValueError(
433 f"resource_quota.{key}={value!r} is not a valid Kubernetes quantity"
434 ) from exc
435 if parsed[key] < 0:
436 raise ValueError(f"resource_quota.{key}={value!r} must not be negative")
437 for container_key, namespace_key in _RESOURCE_QUOTA_INVARIANTS:
438 if parsed[container_key] > parsed[namespace_key]:
439 raise ValueError(
440 f"resource_quota.{container_key}={merged[container_key]!r} exceeds "
441 f"resource_quota.{namespace_key}={merged[namespace_key]!r}: a container "
442 "that passes the LimitRange could never be admitted by the namespace "
443 "ResourceQuota"
444 )
445 return merged
448#: (per-manifest cap, container ceiling, namespace ceiling) triples for the
449#: cross-layer invariant: container_max_* <= *_per_manifest <= max_*.
450_MANIFEST_CAP_INVARIANTS = (
451 ("max_cpu_per_manifest", "container_max_cpu", "max_cpu"),
452 ("max_memory_per_manifest", "container_max_memory", "max_memory"),
453 ("max_gpu_per_manifest", "container_max_gpu", "max_gpu"),
454)
457def _validated_manifest_caps(raw: object, resource_quota: dict[str, str]) -> dict[str, str]:
458 """Merge ``job_validation_policy.resource_quotas`` over defaults; validate.
460 Three layers govern job resources and must tell one story: the
461 manifest/queue processors cap what a single submitted manifest may total
462 (these values), the LimitRange caps each container, and the namespace
463 ResourceQuota caps the aggregate. Enforce
464 ``container_max_* <= *_per_manifest <= max_*`` at synth so the front door
465 never rejects a manifest whose pods the namespace would admit and never
466 accepts one that can never run — previously the defaults disagreed
467 (per-manifest 4 GPUs vs the platform's own 16-GPU EFA training example).
469 Raises:
470 ValueError: On unknown keys, unparseable quantities, or a violated
471 layering invariant.
472 """
473 if not isinstance(raw, dict):
474 raise ValueError(
475 "cdk.json context 'job_validation_policy.resource_quotas' must be an "
476 f"object, got {type(raw).__name__}"
477 )
478 unknown = sorted(set(raw) - set(DEFAULT_MANIFEST_RESOURCE_CAPS))
479 if unknown:
480 allowed = ", ".join(sorted(DEFAULT_MANIFEST_RESOURCE_CAPS))
481 raise ValueError(
482 "cdk.json context 'job_validation_policy.resource_quotas' has unknown "
483 f"key(s) {unknown}; allowed: {allowed}"
484 )
485 merged = {
486 key: str(raw.get(key, default)) for key, default in DEFAULT_MANIFEST_RESOURCE_CAPS.items()
487 }
488 parsed: dict[str, float] = {}
489 for key, value in merged.items():
490 try:
491 parsed[key] = parse_k8s_quantity(value)
492 except ValueError as exc:
493 raise ValueError(
494 f"job_validation_policy.resource_quotas.{key}={value!r} is not a "
495 "valid Kubernetes quantity"
496 ) from exc
497 for manifest_key, container_key, namespace_key in _MANIFEST_CAP_INVARIANTS:
498 container_value = parse_k8s_quantity(resource_quota[container_key])
499 namespace_value = parse_k8s_quantity(resource_quota[namespace_key])
500 if parsed[manifest_key] < container_value:
501 raise ValueError(
502 f"job_validation_policy.resource_quotas.{manifest_key}="
503 f"{merged[manifest_key]!r} is below resource_quota.{container_key}="
504 f"{resource_quota[container_key]!r}: the front door would reject a "
505 "manifest whose single container the LimitRange admits"
506 )
507 if parsed[manifest_key] > namespace_value:
508 raise ValueError(
509 f"job_validation_policy.resource_quotas.{manifest_key}="
510 f"{merged[manifest_key]!r} exceeds resource_quota.{namespace_key}="
511 f"{resource_quota[namespace_key]!r}: the front door would accept a "
512 "manifest the namespace ResourceQuota can never admit"
513 )
514 return merged
517def _parse_helm_enabled_overrides(raw: object) -> frozenset[str]:
518 """Parse and validate the ``helm_enabled_overrides`` context value.
520 Accepts a comma-separated string (the only shape the CDK CLI can pass
521 with ``--context``) or a list of strings (cdk.json-style), returning the
522 validated set of helm-block keys to force-enable. Unknown names raise at
523 synth time with the valid list.
524 """
525 if raw is None:
526 return frozenset()
527 if isinstance(raw, str):
528 names = [part.strip() for part in raw.split(",") if part.strip()]
529 elif isinstance(raw, list) and all(isinstance(part, str) for part in raw):
530 names = [part.strip() for part in raw if part.strip()]
531 else:
532 raise ValueError(
533 f"{_HELM_OVERRIDE_CONTEXT_KEY} must be a comma-separated string or string list"
534 )
535 unknown = sorted(set(names) - _HELM_CHART_CONFIG_KEYS)
536 if unknown:
537 valid = ", ".join(sorted(_HELM_CHART_CONFIG_KEYS))
538 raise ValueError(
539 f"Unknown {_HELM_OVERRIDE_CONTEXT_KEY} name(s): {', '.join(unknown)}. Valid: {valid}"
540 )
541 return frozenset(names)
544def _helm_chart_enabled(
545 helm_config: Mapping[str, Any],
546 overrides: frozenset[str],
547 config_key: str,
548) -> bool:
549 """Resolve one helm-block key's effective enablement.
551 Single source of truth shared by _get_enabled_charts and the
552 kubectl-applier gate replacements so the installed chart set and the
553 gated manifests can never disagree: mandatory charts are always on, a
554 context override forces on, and otherwise the cdk.json toggle decides
555 (missing key defaults to enabled, matching the historical behavior).
556 """
557 if config_key in _MANDATORY_CHART_KEYS or config_key in overrides:
558 return True
559 chart_config = helm_config.get(config_key, {})
560 return bool(chart_config.get("enabled", True)) if isinstance(chart_config, dict) else True
563def _compute_kubectl_scheduler_replacements(
564 *, kueue_enabled: bool, slurm_enabled: bool, kubeflow_trainer_enabled: bool = False
565) -> dict[str, str]:
566 """Build the kubectl-applier replacements that gate scheduler manifests.
568 When Kueue is enabled the ``{{KUEUE_ENABLED}}`` gate resolves so the
569 default queue topology (post-helm-kueue-default-queues.yaml) applies;
570 when Slurm is enabled the ``{{SLURM_ENABLED}}`` gate resolves so the
571 Slinky NetworkPolicies (post-helm-slurm-network.yaml) apply; when the
572 Kubeflow Trainer is enabled the ``{{KUBEFLOW_TRAINER_ENABLED}}`` gate
573 resolves so the built-in ClusterTrainingRuntime blueprints
574 (post-helm-kubeflow-trainer-runtimes.yaml) apply. A disabled
575 scheduler leaves its placeholder unreplaced, the applier skips the file,
576 and _FEATURE_RESOURCE_INVENTORY prunes previously applied objects — the
577 same optional-feature gating observability, FSx, and Valkey use.
578 """
579 replacements: dict[str, str] = {}
580 if kueue_enabled:
581 replacements["{{KUEUE_ENABLED}}"] = "true"
582 if slurm_enabled:
583 replacements["{{SLURM_ENABLED}}"] = "true"
584 if kubeflow_trainer_enabled:
585 replacements["{{KUBEFLOW_TRAINER_ENABLED}}"] = "true"
586 return replacements
589def _compute_kubectl_observability_replacements(
590 enabled: bool, *, grafana_admin_password_rotation_schedule: str = ""
591) -> dict[str, str]:
592 """Build the kubectl-applier replacements that gate the observability manifests.
594 Pure helper kept at module scope so presence/absence can be asserted
595 without synthesizing a full regional stack. When observability is enabled
596 the ``{{CLUSTER_OBSERVABILITY_ENABLED}}`` gate resolves to ``"true"`` so the
597 gp3 StorageClass, ServiceMonitors, dashboards, and credential-rotation
598 CronJob render and apply, and ``{{GRAFANA_ADMIN_PASSWORD_ROTATION_SCHEDULE}}``
599 resolves to the configured cron. When disabled the dict is empty, so those
600 manifests keep an unreplaced ``{{...}}`` token and the applier skips them —
601 the same optional-feature gating FSx and Valkey already rely on.
602 """
603 if not enabled:
604 return {}
605 return {
606 "{{CLUSTER_OBSERVABILITY_ENABLED}}": "true",
607 "{{GRAFANA_ADMIN_PASSWORD_ROTATION_SCHEDULE}}": grafana_admin_password_rotation_schedule,
608 }
611def _augment_trusted_registries_with_project_ecr(
612 base: list[str],
613 *,
614 account: str,
615 regions: list[str],
616 global_region: str,
617 url_suffix: str,
618) -> list[str]:
619 """Return the configured trusted registries plus the project's own ECR.
621 The new ``gco images build`` flow pushes images to a per-account ECR
622 registry under ``<account>.dkr.ecr.<region>.<url-suffix>/gco/<name>``.
623 Without this augmentation the queue/manifest validators would treat
624 those URIs as untrusted and reject every job that uses one — which
625 defeats the whole point of the image registry feature.
627 Returns the unique union of the operator-configured ``base`` list
628 plus the per-region project ECR hostnames (one per deployed region,
629 plus the global region where ``gco-global`` provisions the source
630 repo). Order is stable so the rendered ConfigMap doesn't churn
631 between deploys.
632 """
633 augmented: list[str] = list(base)
634 seen = set(augmented)
635 targets = list(dict.fromkeys([global_region, *regions]))
636 if account:
637 for region in targets:
638 host = f"{account}.dkr.ecr.{region}.{url_suffix}"
639 if host not in seen:
640 augmented.append(host)
641 seen.add(host)
642 return augmented
645def _deployment_timestamp() -> str:
646 """Return the synth-time token that deliberately retriggers convergence."""
647 return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
650def _load_helm_chart_order() -> list[str]:
651 """Return helm chart names in their canonical install order.
653 Reads ``lambda/helm-installer/charts.yaml`` (the source of truth, in file
654 order) so the Step Functions state machine has exactly one task per chart,
655 in the same order every deploy — kueue stays last because its mutating
656 webhook intercepts every Job/Deployment. Missing or malformed chart data
657 aborts synthesis rather than silently omitting every Helm install/uninstall.
658 """
659 charts_path = Path(__file__).resolve().parents[2] / "lambda" / "helm-installer" / "charts.yaml"
660 try:
661 with open(charts_path, encoding="utf-8") as f:
662 data = yaml.safe_load(f)
663 except (OSError, yaml.YAMLError) as exc:
664 raise RuntimeError(f"Unable to load Helm chart order from {charts_path}: {exc}") from exc
665 if not isinstance(data, dict):
666 raise RuntimeError(f"Helm chart config {charts_path} must be an object")
667 charts = data.get("charts")
668 if not isinstance(charts, dict) or not charts:
669 raise RuntimeError(
670 f"Helm chart config {charts_path} must contain a non-empty charts object"
671 )
672 if any(not isinstance(name, str) or not name for name in charts):
673 raise RuntimeError(f"Helm chart config {charts_path} contains an invalid chart name")
674 return list(charts)
677class GCORegionalStack(Stack):
678 """
679 Regional resources stack for a single AWS region.
681 Creates EKS cluster, load balancers, and supporting infrastructure
682 for running GCO services in a specific region.
684 Attributes:
685 vpc: VPC with public/private subnets
686 cluster: EKS Auto Mode cluster
687 """
689 @staticmethod
690 def _create_irsa_role(
691 scope: GCORegionalStack,
692 id: str,
693 oidc_provider_arn: str,
694 oidc_issuer_url: str,
695 service_account_names: list[str],
696 namespaces: list[str],
697 *,
698 include_pod_identity: bool = True,
699 ) -> iam.Role:
700 """Create an OIDC IRSA role, optionally trusted by EKS Pod Identity.
702 IRSA is the primary credential mechanism — it works reliably on EKS Auto
703 Mode by projecting a service-account token that the AWS SDK exchanges for
704 temporary credentials via the OIDC provider. General platform roles retain
705 Pod Identity as a secondary path; controller roles can disable it to keep
706 their trust policy bound to one exact Kubernetes service account.
708 Uses CfnJson to defer OIDC condition key resolution to deploy time,
709 because the issuer URL is a CloudFormation token that can't be used
710 as a Python dict key at synth time.
711 """
712 # Strip https:// from issuer URL for the OIDC condition
713 issuer = Fn.select(1, Fn.split("//", oidc_issuer_url))
715 # Build OIDC conditions using CfnJson to defer token resolution
716 # The issuer URL is a CFN token — can't be used as a dict key at synth time
717 aud_key = Fn.join("", [issuer, ":aud"])
718 sub_key = Fn.join("", [issuer, ":sub"])
720 conditions_json = CfnJson(
721 scope,
722 f"{id}OidcConditions",
723 value={
724 aud_key: "sts.amazonaws.com",
725 sub_key: [
726 f"system:serviceaccount:{ns}:{sa}"
727 for ns in namespaces
728 for sa in service_account_names
729 ],
730 },
731 )
733 role = iam.Role(
734 scope,
735 id,
736 assumed_by=iam.FederatedPrincipal(
737 federated=oidc_provider_arn,
738 conditions={
739 "StringEquals": conditions_json,
740 },
741 assume_role_action="sts:AssumeRoleWithWebIdentity",
742 ),
743 )
745 if include_pod_identity:
746 # Secondary credential path for platform workloads. Dedicated
747 # controllers such as LBC deliberately remain OIDC-only.
748 assert role.assume_role_policy is not None
749 role.assume_role_policy.add_statements(
750 iam.PolicyStatement(
751 effect=iam.Effect.ALLOW,
752 principals=[iam.ServicePrincipal("pods.eks.amazonaws.com")],
753 actions=["sts:AssumeRole", "sts:TagSession"],
754 )
755 )
756 return role
758 def __init__(
759 self,
760 scope: Construct,
761 construct_id: str,
762 config: ConfigLoader,
763 region: str,
764 auth_secret_arn: str,
765 **kwargs: Any,
766 ) -> None:
767 super().__init__(scope, construct_id, **kwargs)
769 self.config = config
770 self.deployment_region = region
771 self.auth_secret_arn = auth_secret_arn
772 self.alb_arn: str | None = None
773 self.disable_efs_automatic_backups = _explicit_context_bool(
774 self.node.try_get_context(_LIVE_VALIDATION_DISABLE_EFS_BACKUPS_CONTEXT),
775 key=_LIVE_VALIDATION_DISABLE_EFS_BACKUPS_CONTEXT,
776 )
777 retain_provider_logs = self.node.try_get_context(_LIVE_VALIDATION_PROVIDER_LOG_CONTEXT)
778 self.provider_log_group_removal_policy = (
779 RemovalPolicy.RETAIN
780 if retain_provider_logs is True
781 or (
782 isinstance(retain_provider_logs, str)
783 and retain_provider_logs.strip().casefold() == "true"
784 )
785 else RemovalPolicy.DESTROY
786 )
787 supports_global_accelerator = getattr(config, "supports_global_accelerator", None)
788 self.global_accelerator_enabled = (
789 bool(supports_global_accelerator()) if callable(supports_global_accelerator) else True
790 )
792 # Get cluster configuration for this region
793 cluster_config = self.config.get_cluster_config(region)
794 self.cluster_config = cluster_config
796 # Create VPC for the EKS cluster.
797 #
798 # ``max_azs=99`` is the CDK idiom for "span every Availability Zone the
799 # region offers" — CDK caps the value at the number of AZs actually
800 # returned for this account+region, so each AZ gets one public and one
801 # private subnet. This only enumerates the *real* AZ list when the stack
802 # is environment-specific (account + region both resolved); app.py sets
803 # the account from CDK_DEFAULT_ACCOUNT for exactly this reason. In an
804 # environment-agnostic synth (no account, e.g. some CI paths) CDK falls
805 # back to a fixed placeholder AZ list rather than the full set.
806 self.vpc = ec2.Vpc(
807 self,
808 "GCOVpc",
809 # vpc_name intentionally omitted - let CDK generate unique name
810 max_azs=99, # use every AZ in the region (each AZ gets 1 public + 1 private subnet)
811 nat_gateways=2, # For high availability
812 subnet_configuration=[
813 ec2.SubnetConfiguration(
814 name="PublicSubnet", subnet_type=ec2.SubnetType.PUBLIC, cidr_mask=24
815 ),
816 ec2.SubnetConfiguration(
817 name="PrivateSubnet",
818 subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS,
819 cidr_mask=24,
820 ),
821 ],
822 )
824 # Enable VPC Flow Logs for network traffic analysis and security monitoring
825 self._create_vpc_flow_logs()
827 # Keep AWS API traffic inside the VPC where the operator asked for it
828 self._create_vpc_endpoints()
830 # Create SQS queue for job ingestion
831 self._create_sqs_queue()
833 # Build the platform service images as CDK Docker image assets
834 self._create_container_images()
836 # Pre-create the execution role shared by every ``cr.AwsCustomResource``
837 # in this stack. See ``_create_aws_custom_resource_role`` for the full
838 # rationale — in short, CDK's default behavior of auto-generating a
839 # Lambda role per ``AwsCustomResource`` (and then merging all the
840 # ``policy=`` statements onto it during deploy) triggers an IAM
841 # propagation race on cold creates. We sidestep the race by creating
842 # a single long-lived role up front and attaching policies to it as
843 # each consumer is built; every ``AwsCustomResource`` then passes
844 # ``role=self.aws_custom_resource_role`` instead of ``policy=``, so
845 # the singleton Lambda runs against a role whose inline policy has
846 # already replicated globally.
847 self._create_aws_custom_resource_role()
849 # Resolve this region's fixed ACM certificate ARN from the global
850 # backend-TLS registry before rendering the HTTPS-only Gateway.
851 self.backend_tls_certificate_arn = self._resolve_backend_tls_certificate_arn()
853 # Create EKS cluster
854 self._create_eks_cluster(cluster_config)
856 # Optional Volcano image mirror (cdk.json ``volcano_image_mirror``).
857 # When enabled this resolves ``self.volcano_mirror_registry`` — the
858 # gco/* ECR namespace that Volcano's ``basic.image_registry`` is
859 # redirected to, so its docker.io-only images are pulled from the
860 # project's own ECR (populated out-of-band by
861 # ``gco images mirror``) instead of rate-limited Docker
862 # Hub. Creates no CloudFormation resources; must run before
863 # ``_apply_kubernetes_manifests`` builds the ``HelmInstallCharts`` custom
864 # resource, which reads the override via ``_helm_chart_value_overrides()``.
865 self._configure_volcano_image_mirror()
867 # Resolve the always-on Cluster_Shared_Bucket identity from SSM
868 # (owned by GCOGlobalStack) and attach RW + KMS grants to the
869 # job-pod role. Runs unconditionally — the ConfigMap and IAM
870 # statements are always present on every regional cluster. Must
871 # run after
872 # _create_pod_identity_associations (which created service_account_role)
873 # and before _apply_kubernetes_manifests (which consumes the
874 # replacements in the KubectlApplyManifests CustomResource).
875 self.cluster_shared_identity = self._resolve_cluster_shared_bucket_from_ssm()
876 self._grant_cluster_shared_bucket_to_job_role(self.cluster_shared_identity)
878 # MLflow artifact storage: a dedicated OIDC-only IRSA role for the
879 # tracking server's service account, scoped to the mlflow-artifacts/
880 # prefix of the same shared bucket. Created here (not with the other
881 # IRSA roles) because it needs the resolved bucket identity above;
882 # must precede _apply_kubernetes_manifests, whose value overrides
883 # inject the role ARN into the chart's service-account annotation.
884 if self._mlflow_active():
885 self._create_mlflow_artifact_role(self.cluster_shared_identity)
887 # Create the always-on general-purpose regional bucket (KMS key +
888 # access-logs bucket + primary bucket). Provisioned unconditionally —
889 # there is no cdk.json toggle and no feature flag gating its existence —
890 # in addition to the central buckets owned by GCOGlobalStack.
891 self._create_regional_shared_bucket()
893 # Create EFS for shared storage
894 self._create_efs()
896 # Create FSx for Lustre (if enabled) for high-performance storage
897 self._create_fsx_lustre()
899 # Create Valkey Serverless cache (if enabled) for K/V caching
900 self._create_valkey_cache()
902 # Create Aurora Serverless v2 + pgvector (if enabled) for vector DB
903 self._create_aurora_pgvector()
905 # Discover and publish the Gateway ALB in every partition. Global
906 # Accelerator registration is an optional extension of this same exact
907 # ownership path where the service is available.
908 self._create_ga_registration_lambda()
910 # Provider framework Lambdas can emit their final delete-event log after
911 # CloudFormation has otherwise finished the custom resource. Strict live
912 # validation retains this explicit group through stack deletion so the
913 # harness can remove the same checkpointed generation after every target
914 # stack is absent. Ordinary deployments keep DESTROY semantics and do not
915 # accumulate retained groups.
916 self.helm_installer_provider_log_group = logs.LogGroup(
917 self,
918 "HelmInstallerProviderLogGroup",
919 retention=logs.RetentionDays.ONE_WEEK,
920 removal_policy=self.provider_log_group_removal_policy,
921 )
923 # Create Helm installer Lambda for KEDA and other Helm-based installations
924 self._create_helm_installer_lambda()
926 # Apply Kubernetes manifests (after EFS so IDs are available)
927 self._apply_kubernetes_manifests()
929 # Create CloudFormation drift detection (daily schedule + SNS alerts)
930 self._create_drift_detection()
932 # Create dedicated IAM role for MCP server
933 self._create_mcp_role()
935 # Export cluster information
936 self._create_outputs()
938 # Apply cdk-nag suppressions for this stack
939 self._apply_nag_suppressions()
941 def _create_vpc_flow_logs(self) -> None:
942 """Create VPC Flow Logs for network traffic monitoring.
944 Flow logs capture information about IP traffic going to and from
945 network interfaces in the VPC. This is required for security
946 monitoring and compliance (HIPAA, SOC2, etc.).
947 """
948 # Create CloudWatch Log Group for flow logs
949 flow_log_group = logs.LogGroup(
950 self,
951 "VpcFlowLogGroup",
952 # log_group_name intentionally omitted - let CDK generate unique name
953 retention=logs.RetentionDays.ONE_MONTH,
954 removal_policy=RemovalPolicy.DESTROY,
955 )
957 # Create IAM role for VPC Flow Logs
958 flow_log_role = iam.Role(
959 self,
960 "VpcFlowLogRole",
961 assumed_by=iam.ServicePrincipal("vpc-flow-logs.amazonaws.com"),
962 )
964 flow_log_role.add_to_policy(
965 iam.PolicyStatement(
966 actions=[
967 "logs:CreateLogStream",
968 "logs:PutLogEvents",
969 "logs:DescribeLogGroups",
970 "logs:DescribeLogStreams",
971 ],
972 resources=[flow_log_group.log_group_arn, f"{flow_log_group.log_group_arn}:*"],
973 )
974 )
976 # Create VPC Flow Log
977 ec2.FlowLog(
978 self,
979 "VpcFlowLog",
980 resource_type=ec2.FlowLogResourceType.from_vpc(self.vpc),
981 destination=ec2.FlowLogDestination.to_cloud_watch_logs(flow_log_group, flow_log_role),
982 traffic_type=ec2.FlowLogTrafficType.ALL,
983 )
985 def _create_vpc_endpoints(self) -> None:
986 """Create the VPC endpoints selected by ``cdk.json`` ``vpc_endpoints``.
988 Gateway endpoints (S3, DynamoDB) are free route-table entries: with
989 them, the platform's largest data path — model and dataset pulls,
990 checkpoints, MLflow artifacts, cost reports — stops flowing through the
991 NAT gateways' per-GB metering and never leaves the VPC. Interface
992 endpoints are PrivateLink ENIs billed per AZ-hour, so they are opt-in;
993 CDK gives each one a security group admitting HTTPS from the VPC CIDR
994 and private DNS, so callers keep using the public service hostnames.
996 NetworkPolicy egress is unaffected either way: S3 traffic still
997 resolves to public S3 addresses that the route table steers into the
998 endpoint, which is why 03-network-policies.yaml allows HTTPS by port
999 rather than by destination.
1000 """
1001 selection = self.config.get_vpc_endpoints_config()
1002 self.vpc_gateway_endpoints: dict[str, ec2.GatewayVpcEndpoint] = {}
1003 self.vpc_interface_endpoints: dict[str, ec2.InterfaceVpcEndpoint] = {}
1004 for service in selection["gateway"]:
1005 self.vpc_gateway_endpoints[service] = self.vpc.add_gateway_endpoint(
1006 f"VpcEndpoint-{service}",
1007 service=_GATEWAY_ENDPOINT_SERVICES[service],
1008 )
1009 for service in selection["interface"]:
1010 self.vpc_interface_endpoints[service] = self.vpc.add_interface_endpoint(
1011 f"VpcEndpoint-{service.replace('.', '-')}",
1012 service=_INTERFACE_ENDPOINT_SERVICES[service],
1013 subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
1014 )
1016 def _apply_nag_suppressions(self) -> None:
1017 """Apply cdk-nag suppressions for this stack."""
1018 from gco.stacks.nag_suppressions import apply_all_suppressions
1020 apply_all_suppressions(
1021 self,
1022 stack_type="regional",
1023 regions=self.config.get_regions(),
1024 global_region=self.config.get_global_region(),
1025 api_gateway_region=self.config.get_api_gateway_region(),
1026 project_name=self.config.get_project_name(),
1027 )
1029 def _create_sqs_queue(self) -> None:
1030 """Create SQS queue for job ingestion.
1032 Creates an SQS queue that serves as the default job ingestion point
1033 for this region. Jobs submitted to this queue are processed by the
1034 manifest processor and KEDA scales based on queue depth.
1036 Also creates a dead-letter queue for failed messages.
1037 Both queues use server-side encryption with AWS managed keys.
1038 """
1039 project_name = self.config.get_project_name()
1041 # Create dead-letter queue for failed messages
1042 self.job_dlq = sqs.Queue(
1043 self,
1044 "JobDeadLetterQueue",
1045 queue_name=f"{project_name}-jobs-dlq-{self.deployment_region}",
1046 retention_period=Duration.days(14),
1047 removal_policy=RemovalPolicy.DESTROY,
1048 enforce_ssl=True, # Require SSL for all requests
1049 encryption=sqs.QueueEncryption.SQS_MANAGED, # Server-side encryption
1050 )
1052 # Create main job queue
1053 self.job_queue = sqs.Queue(
1054 self,
1055 "JobQueue",
1056 queue_name=f"{project_name}-jobs-{self.deployment_region}",
1057 visibility_timeout=Duration.minutes(5), # Match Lambda timeout
1058 retention_period=Duration.days(7),
1059 dead_letter_queue=sqs.DeadLetterQueue(
1060 max_receive_count=3, # Move to DLQ after 3 failed attempts
1061 queue=self.job_dlq,
1062 ),
1063 removal_policy=RemovalPolicy.DESTROY,
1064 enforce_ssl=True, # Require SSL for all requests
1065 encryption=sqs.QueueEncryption.SQS_MANAGED, # Server-side encryption
1066 )
1068 # Output queue information
1069 CfnOutput(
1070 self,
1071 "JobQueueUrl",
1072 value=self.job_queue.queue_url,
1073 description=f"SQS Job Queue URL for {self.deployment_region}",
1074 export_name=f"{project_name}-job-queue-url-{self.deployment_region}",
1075 )
1077 CfnOutput(
1078 self,
1079 "JobQueueArn",
1080 value=self.job_queue.queue_arn,
1081 description=f"SQS Job Queue ARN for {self.deployment_region}",
1082 export_name=f"{project_name}-job-queue-arn-{self.deployment_region}",
1083 )
1085 CfnOutput(
1086 self,
1087 "JobDlqUrl",
1088 value=self.job_dlq.queue_url,
1089 description=f"SQS Dead Letter Queue URL for {self.deployment_region}",
1090 export_name=f"{project_name}-job-dlq-url-{self.deployment_region}",
1091 )
1093 def _create_aws_custom_resource_role(self) -> None:
1094 """Pre-create the execution role shared by every ``AwsCustomResource``.
1096 CDK's ``cr.AwsCustomResource`` defaults to auto-generating a per-
1097 construct Lambda execution role from the ``policy=`` parameter.
1098 Internally, CDK deduplicates those auto-generated roles onto a
1099 single *singleton* provider Lambda (logical id prefix
1100 ``AWS679f53fac002430cb0da5b7982bd22872``), and merges each custom
1101 resource's policy statements onto that Lambda's role at stack
1102 create time. On cold deploys, CloudFormation invokes the Lambda
1103 within 2-3 seconds of attaching a new policy statement, which is
1104 faster than IAM's global propagation window. The symptom is a
1105 ``iam:PassRole NOT authorized`` failure on whichever addon role
1106 update happens to run right after its ``iam:PassRole`` policy
1107 statement was attached but before it had replicated.
1109 The fix is to create the role up front, attach every policy
1110 statement the stack will need during stack creation, and pass
1111 ``role=self.aws_custom_resource_role`` to every
1112 ``AwsCustomResource`` instead of ``policy=``. Because the role
1113 already exists — and its inline policy has had minutes to
1114 replicate by the time any ``AwsCustomResource`` actually fires —
1115 the race disappears entirely.
1117 This method creates the role with the statements we can compute
1118 without a cluster reference (EKS ``UpdateAddon`` / ``DescribeAddon``
1119 scoped to this cluster, and SSM ``GetParameter`` for the endpoint
1120 group ARN). ``iam:PassRole`` statements for individual addon
1121 roles (EFS CSI, FSx CSI, CloudWatch Observability) are appended
1122 by each ``_create_*_addon`` method after the corresponding IRSA
1123 role has been created, so every PassRole ``resources=`` list
1124 stays precise (no wildcards) and cdk-nag stays happy.
1125 """
1126 project_name = self.config.get_project_name()
1127 global_region = self.config.get_global_region()
1129 self.aws_custom_resource_role = iam.Role(
1130 self,
1131 "AwsCustomResourceRole",
1132 assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
1133 description=(
1134 "Shared execution role for every cr.AwsCustomResource in this "
1135 "stack. Pre-created to avoid the IAM policy propagation race "
1136 "that occurs when CDK auto-generates per-CR roles and the "
1137 "singleton provider Lambda fires before the freshly-attached "
1138 "policy has replicated globally."
1139 ),
1140 managed_policies=[
1141 iam.ManagedPolicy.from_aws_managed_policy_name(
1142 "service-role/AWSLambdaBasicExecutionRole"
1143 ),
1144 ],
1145 )
1147 # EKS UpdateAddon / DescribeAddon — used by the three updateAddon
1148 # custom resources (EFS CSI, FSx CSI, CloudWatch Observability).
1149 # Scoped to this cluster's addons by ARN.
1150 self.aws_custom_resource_role.add_to_policy(
1151 iam.PolicyStatement(
1152 effect=iam.Effect.ALLOW,
1153 actions=["eks:UpdateAddon", "eks:DescribeAddon"],
1154 resources=[
1155 f"arn:{self.partition}:eks:{self.deployment_region}:{self.account}"
1156 f":addon/{self.cluster_config.cluster_name}/*"
1157 ],
1158 )
1159 )
1161 # SSM GetParameter — used by the GetEndpointGroupArn custom
1162 # resource in _create_ga_registration_lambda to read the ARN of
1163 # the Global Accelerator endpoint group published by the global
1164 # stack during its deploy.
1165 self.aws_custom_resource_role.add_to_policy(
1166 iam.PolicyStatement(
1167 effect=iam.Effect.ALLOW,
1168 actions=["ssm:GetParameter"],
1169 resources=[
1170 f"arn:{self.partition}:ssm:{global_region}:{self.account}:"
1171 f"parameter/{project_name}/*"
1172 ],
1173 )
1174 )
1176 # cdk-nag suppressions: the two wildcard-bearing ARNs above are
1177 # intentional and both scoped as tightly as AWS IAM permits.
1178 #
1179 # - The ``eks:UpdateAddon`` / ``eks:DescribeAddon`` statement uses
1180 # ``addon/<cluster>/*`` as its resource because the same shared
1181 # role is consumed by three different updateAddon custom
1182 # resources (EFS CSI, FSx CSI, CloudWatch Observability). Each
1183 # addon has its own ARN and we'd otherwise need three separate
1184 # statements that each grant access to a known addon name. The
1185 # wildcard is scoped to a single cluster in a single region in
1186 # a single account — it cannot be used against any addon
1187 # belonging to a different cluster or a different service.
1188 #
1189 # - The ``ssm:GetParameter`` statement uses
1190 # ``parameter/<project>/*`` because the exact parameter name
1191 # (``endpoint-group-<region>-arn``) is only known at Global
1192 # Accelerator registration time and the endpoint path
1193 # structure is ``<project>/<parameter>``. Scoping to the
1194 # project prefix restricts access to parameters owned by this
1195 # project only.
1196 from gco.stacks.nag_suppressions import acknowledge_nag_findings
1198 acknowledge_nag_findings(
1199 self.aws_custom_resource_role,
1200 [
1201 {
1202 "id": "AwsSolutions-IAM5",
1203 "reason": (
1204 "Scoped to a single EKS cluster's addons "
1205 "(addon/<cluster>/*) and this project's SSM "
1206 "parameters (parameter/<project>/*). Both wildcards "
1207 "are as tight as AWS IAM permits: addon names and "
1208 "parameter names are not known at stack synthesis "
1209 "time because the addons are created later in the "
1210 "same stack and the GA endpoint group ARN is "
1211 "published by a separate stack during deploy. The "
1212 "shared role pattern itself is deliberate — see "
1213 "_create_aws_custom_resource_role docstring for why "
1214 "we pre-create instead of letting CDK auto-generate "
1215 "per-CR roles."
1216 ),
1217 "appliesTo": [
1218 f"Resource::arn:<AWS::Partition>:eks:{self.deployment_region}"
1219 f":<AWS::AccountId>:addon/{self.cluster_config.cluster_name}/*",
1220 f"Resource::arn:<AWS::Partition>:ssm:{global_region}"
1221 f":<AWS::AccountId>:parameter/{project_name}/*",
1222 ],
1223 },
1224 ],
1225 )
1227 def _resolve_backend_tls_certificate_arn(self) -> str:
1228 """Read this region's stable imported ACM ARN from global-region SSM.
1230 The certificate manager publishes one fixed ARN per workload region.
1231 The regional Ingress consumes the token directly, which creates a
1232 CloudFormation dependency ensuring the certificate exists before the
1233 HTTPS listener is reconciled. The shared custom-resource role is
1234 already restricted to this project's SSM namespace.
1235 """
1236 project_name = self.config.get_project_name()
1237 parameter_name = backend_tls_certificate_arn_parameter_name(
1238 project_name, self.deployment_region
1239 )
1240 reader = cr.AwsCustomResource(
1241 self,
1242 "GetBackendTlsCertificateArn",
1243 on_create=cr.AwsSdkCall(
1244 service="SSM",
1245 action="getParameter",
1246 parameters={"Name": parameter_name},
1247 region=self.config.get_global_region(),
1248 physical_resource_id=cr.PhysicalResourceId.of(
1249 f"{project_name}-backend-tls-certificate-{self.deployment_region}"
1250 ),
1251 ),
1252 on_update=cr.AwsSdkCall(
1253 service="SSM",
1254 action="getParameter",
1255 parameters={"Name": parameter_name},
1256 region=self.config.get_global_region(),
1257 ),
1258 role=self.aws_custom_resource_role,
1259 )
1260 reader.node.add_dependency(self.aws_custom_resource_role)
1261 return str(reader.get_response_field("Parameter.Value"))
1263 def _create_container_images(self) -> None:
1264 """Build the platform service images as CDK Docker image assets.
1266 Every image is a ``DockerImageAsset``: CDK builds it at synth, pushes
1267 it to the bootstrap asset repository under a content-hash tag, and the
1268 manifests receive that URI through ``{{*_IMAGE}}`` replacements. The
1269 stack creates no ECR repositories of its own — three empty
1270 per-service repositories used to be declared here and were never
1271 pushed to or referenced; they only added resources to create, scan,
1272 and delete on every deploy.
1273 """
1275 # All Docker images target AMD64 (x86_64) to match EKS Auto Mode's
1276 # default system nodepool.
1278 # Build and push health monitor Docker image
1279 self.health_monitor_image = ecr_assets.DockerImageAsset(
1280 self,
1281 "HealthMonitorImage",
1282 directory=".", # Root directory
1283 file="dockerfiles/health-monitor-dockerfile",
1284 platform=ecr_assets.Platform.LINUX_AMD64,
1285 exclude=_service_image_asset_excludes(
1286 "dockerfiles/health-monitor-dockerfile",
1287 ),
1288 )
1290 # Build and push manifest processor Docker image
1291 self.manifest_processor_image = ecr_assets.DockerImageAsset(
1292 self,
1293 "ManifestProcessorImage",
1294 directory=".",
1295 file="dockerfiles/manifest-processor-dockerfile",
1296 platform=ecr_assets.Platform.LINUX_AMD64,
1297 exclude=_service_image_asset_excludes(
1298 "dockerfiles/manifest-processor-dockerfile",
1299 ),
1300 )
1302 # Build the inference-only data-plane proxy image. Keeping this
1303 # separate from manifest-processor prevents model traffic from
1304 # sharing its Kubernetes API/RBAC and queue-worker process surface.
1305 self.inference_proxy_image = ecr_assets.DockerImageAsset(
1306 self,
1307 "InferenceProxyImage",
1308 directory=".",
1309 file="dockerfiles/inference-proxy-dockerfile",
1310 platform=ecr_assets.Platform.LINUX_AMD64,
1311 exclude=_service_image_asset_excludes(
1312 "dockerfiles/inference-proxy-dockerfile",
1313 ),
1314 )
1316 # Output image URIs for reference
1317 CfnOutput(
1318 self,
1319 "HealthMonitorImageUri",
1320 value=self.health_monitor_image.image_uri,
1321 description="Health Monitor Docker image URI",
1322 )
1324 CfnOutput(
1325 self,
1326 "ManifestProcessorImageUri",
1327 value=self.manifest_processor_image.image_uri,
1328 description="Manifest Processor Docker image URI",
1329 )
1331 CfnOutput(
1332 self,
1333 "InferenceProxyImageUri",
1334 value=self.inference_proxy_image.image_uri,
1335 description="Inference Proxy Docker image URI",
1336 )
1338 # Build and push inference monitor Docker image
1339 self.inference_monitor_image = ecr_assets.DockerImageAsset(
1340 self,
1341 "InferenceMonitorImage",
1342 directory=".",
1343 file="dockerfiles/inference-monitor-dockerfile",
1344 platform=ecr_assets.Platform.LINUX_AMD64,
1345 exclude=_service_image_asset_excludes(
1346 "dockerfiles/inference-monitor-dockerfile",
1347 ),
1348 )
1350 CfnOutput(
1351 self,
1352 "InferenceMonitorImageUri",
1353 value=self.inference_monitor_image.image_uri,
1354 description="Inference Monitor Docker image URI",
1355 )
1357 # Build and push queue processor Docker image (if enabled).
1358 # The queue processor is a KEDA ScaledJob that consumes manifests from
1359 # the regional SQS queue. It can be disabled in cdk.json if users want
1360 # to implement their own consumer. When disabled, the post-helm-sqs-consumer.yaml
1361 # manifest is skipped (unreplaced template variables cause it to be skipped).
1362 queue_processor_config = self.node.try_get_context("queue_processor") or {}
1363 self.queue_processor_enabled = queue_processor_config.get("enabled", True)
1365 if self.queue_processor_enabled:
1366 self.queue_processor_image = ecr_assets.DockerImageAsset(
1367 self,
1368 "QueueProcessorImage",
1369 directory=".",
1370 file="dockerfiles/queue-processor-dockerfile",
1371 platform=ecr_assets.Platform.LINUX_AMD64,
1372 exclude=_service_image_asset_excludes(
1373 "dockerfiles/queue-processor-dockerfile",
1374 ),
1375 )
1377 CfnOutput(
1378 self,
1379 "QueueProcessorImageUri",
1380 value=self.queue_processor_image.image_uri,
1381 description="Queue Processor Docker image URI",
1382 )
1384 # Build and push the cost-monitor image only when the cost monitoring
1385 # pipeline deploys to this region — skipping the build keeps opted-out
1386 # deployments' synth/deploy time unchanged (same gating rationale as
1387 # the queue processor above).
1388 if self._cost_monitoring_active():
1389 self.cost_monitor_image = ecr_assets.DockerImageAsset(
1390 self,
1391 "CostMonitorImage",
1392 directory=".",
1393 file="dockerfiles/cost-monitor-dockerfile",
1394 platform=ecr_assets.Platform.LINUX_AMD64,
1395 exclude=_service_image_asset_excludes(
1396 "dockerfiles/cost-monitor-dockerfile",
1397 ),
1398 )
1400 CfnOutput(
1401 self,
1402 "CostMonitorImageUri",
1403 value=self.cost_monitor_image.image_uri,
1404 description="Cost Monitor Docker image URI",
1405 )
1407 def _resolve_unsupported_az_names(self) -> list[str]:
1408 """Resolve this region's EKS-unsupported AZ *IDs* to this account's AZ *names*.
1410 EKS rejects cluster subnets in a small set of Availability Zones,
1411 published by AZ ID (``EKS_UNSUPPORTED_AZ_IDS``). AZ *names* are
1412 randomized per account, so the disallowed ``use1-az3`` may be
1413 ``us-east-1e`` in one account and a different name in another — we must
1414 map ID -> name for the deploy account.
1416 Returns an empty list when the region has no restriction (the common
1417 case) or when the deploy account is not resolved. A credentialed,
1418 environment-specific synth fails closed if EC2 cannot resolve every
1419 restricted AZ ID; selecting all private subnets in that case could hand
1420 EKS a known-unsupported control-plane subnet.
1421 """
1422 unsupported_ids = EKS_UNSUPPORTED_AZ_IDS.get(self.deployment_region, ())
1423 if not unsupported_ids:
1424 return []
1425 # Only reach EC2 during a credentialed, environment-specific synth or
1426 # deploy. The CDK CLI exports CDK_DEFAULT_ACCOUNT from the active
1427 # identity; unit tests and agnostic synth don't, so we never call AWS
1428 # (nor block synthesis on missing credentials) there.
1429 if not os.environ.get("CDK_DEFAULT_ACCOUNT"):
1430 return []
1431 try:
1432 import boto3
1433 from botocore.config import Config
1435 ec2_client = boto3.client(
1436 "ec2",
1437 region_name=self.deployment_region,
1438 config=Config(connect_timeout=5, read_timeout=5, retries={"max_attempts": 2}),
1439 )
1440 response = ec2_client.describe_availability_zones(
1441 Filters=[{"Name": "zone-id", "Values": list(unsupported_ids)}]
1442 )
1443 except Exception as exc:
1444 raise RuntimeError(
1445 f"Unable to resolve EKS-unsupported Availability Zones in "
1446 f"{self.deployment_region}: {exc}"
1447 ) from exc
1449 zones = response.get("AvailabilityZones")
1450 if not isinstance(zones, list):
1451 raise RuntimeError(
1452 f"EC2 returned malformed Availability Zone data for {self.deployment_region}"
1453 )
1454 names_by_id = {
1455 zone_id: zone_name
1456 for zone in zones
1457 if isinstance(zone, dict)
1458 and isinstance((zone_id := zone.get("ZoneId")), str)
1459 and isinstance((zone_name := zone.get("ZoneName")), str)
1460 and zone_id in unsupported_ids
1461 and zone_name
1462 }
1463 missing_ids = [zone_id for zone_id in unsupported_ids if zone_id not in names_by_id]
1464 if missing_ids:
1465 raise RuntimeError(
1466 f"EC2 did not resolve EKS-unsupported Availability Zone IDs in "
1467 f"{self.deployment_region}: {', '.join(missing_ids)}"
1468 )
1469 return [names_by_id[zone_id] for zone_id in unsupported_ids]
1471 def _eks_control_plane_subnets(self) -> ec2.SubnetSelection:
1472 """Private-subnet selection for the EKS control plane, excluding any AZ
1473 EKS does not support for cluster subnets.
1475 Records the outcome on ``self`` (``eks_unsupported_az_names`` and
1476 ``eks_control_plane_subnets``) so tests and operators can introspect
1477 exactly which subnets the cluster was given.
1478 """
1479 unsupported = set(self._resolve_unsupported_az_names())
1480 usable = [
1481 subnet
1482 for subnet in self.vpc.private_subnets
1483 if subnet.availability_zone not in unsupported
1484 ]
1485 self.eks_unsupported_az_names = sorted(unsupported)
1486 self.eks_control_plane_subnets = usable
1487 if not unsupported:
1488 # No restricted AZ in this region: keep the subnet-type selection so
1489 # the synthesized template is identical to before for the common case.
1490 return ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS)
1491 return ec2.SubnetSelection(subnets=usable)
1493 def _create_eks_cluster(self, cluster_config: Any) -> None:
1494 """Create the EKS cluster with auto mode and GPU node groups"""
1496 # Create cluster admin role
1497 # role_name intentionally omitted - let CDK generate unique name
1498 cluster_admin_role = iam.Role(
1499 self,
1500 "ClusterAdminRole",
1501 assumed_by=iam.ServicePrincipal("eks.amazonaws.com"),
1502 managed_policies=[
1503 iam.ManagedPolicy.from_aws_managed_policy_name("AmazonEKSClusterPolicy")
1504 ],
1505 )
1507 # Create node group role
1508 # role_name intentionally omitted - let CDK generate unique name
1509 iam.Role(
1510 self,
1511 "NodeGroupRole",
1512 assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"),
1513 managed_policies=[
1514 iam.ManagedPolicy.from_aws_managed_policy_name("AmazonEKSWorkerNodePolicy"),
1515 iam.ManagedPolicy.from_aws_managed_policy_name("AmazonEKS_CNI_Policy"),
1516 iam.ManagedPolicy.from_aws_managed_policy_name(
1517 "AmazonEC2ContainerRegistryReadOnly"
1518 ),
1519 ],
1520 )
1522 # Create EKS Auto Mode cluster with built-in system and general-purpose nodepools
1523 # Auto Mode automatically manages compute resources and comes with essential addons
1524 # Get endpoint access configuration
1525 eks_config = self.config.get_eks_cluster_config()
1526 endpoint_access_mode = eks_config.get("endpoint_access", "PRIVATE")
1528 # Map config string to EKS EndpointAccess enum. When the public
1529 # endpoint is enabled, eks_cluster.public_access_cidrs restricts who
1530 # can reach it; enabling it with no allowlist is a deliberate,
1531 # loudly-announced 0.0.0.0/0 exposure rather than a silent default.
1532 public_access_cidrs = [str(cidr) for cidr in (eks_config.get("public_access_cidrs") or [])]
1533 if endpoint_access_mode == "PRIVATE":
1534 endpoint_access = eks.EndpointAccess.PRIVATE
1535 elif public_access_cidrs:
1536 endpoint_access = eks.EndpointAccess.PUBLIC_AND_PRIVATE.only_from(*public_access_cidrs)
1537 else:
1538 Annotations.of(self).add_warning(
1539 "eks_cluster.endpoint_access is PUBLIC_AND_PRIVATE with no "
1540 "public_access_cidrs allowlist: the EKS API server accepts "
1541 "connections from 0.0.0.0/0 (authentication still applies). "
1542 "Set eks_cluster.public_access_cidrs to your egress CIDRs "
1543 "(gco stacks eks endpoint set PUBLIC_AND_PRIVATE --cidr <cidr>) "
1544 "or use PRIVATE with `gco cluster tunnel`."
1545 )
1546 endpoint_access = eks.EndpointAccess.PUBLIC_AND_PRIVATE
1548 # Create KMS key for EKS secrets encryption
1549 self.eks_encryption_key = kms.Key(
1550 self,
1551 "EksSecretsEncryptionKey",
1552 description="KMS key for EKS Kubernetes secrets encryption",
1553 enable_key_rotation=True,
1554 removal_policy=RemovalPolicy.RETAIN,
1555 )
1557 # Get Kubernetes version - use custom version if not available in CDK enum
1558 k8s_version_str = cluster_config.kubernetes_version
1559 try:
1560 k8s_version = getattr(eks.KubernetesVersion, f"V{k8s_version_str.replace('.', '_')}")
1561 except AttributeError:
1562 # Version not in CDK enum yet, use custom version
1563 k8s_version = eks.KubernetesVersion.of(k8s_version_str)
1565 self.cluster = eks.Cluster(
1566 self,
1567 "GCOEksCluster",
1568 cluster_name=cluster_config.cluster_name,
1569 version=k8s_version, # Use configured version for Auto Mode with DRA support
1570 vpc=self.vpc,
1571 compute=eks.ComputeConfig(
1572 # Enable both built-in node pools - Auto Mode manages these automatically
1573 node_pools=["system", "general-purpose"]
1574 ),
1575 # SECURITY: Endpoint access controlled via cdk.json eks_cluster.endpoint_access
1576 # PRIVATE (default): EKS API accessible only from within VPC - most secure
1577 # Job submission works via API Gateway → Lambda (in VPC) or SQS
1578 # For kubectl access, use a bastion host, VPN, or AWS SSM Session Manager
1579 # PUBLIC_AND_PRIVATE: EKS API accessible from internet and VPC
1580 # Allows direct kubectl access but less secure
1581 endpoint_access=endpoint_access,
1582 role=cluster_admin_role,
1583 # The VPC spans every AZ, but EKS refuses control-plane subnets in a
1584 # few AZs (by stable AZ ID; see EKS_UNSUPPORTED_AZ_IDS). Select the
1585 # private subnets in supported AZs only — worker/other subnets in the
1586 # excluded AZs still exist in the VPC.
1587 vpc_subnets=[self._eks_control_plane_subnets()],
1588 # Enable all control plane logging for security and compliance
1589 cluster_logging=[
1590 eks.ClusterLoggingTypes.API,
1591 eks.ClusterLoggingTypes.AUDIT,
1592 eks.ClusterLoggingTypes.AUTHENTICATOR,
1593 eks.ClusterLoggingTypes.CONTROLLER_MANAGER,
1594 eks.ClusterLoggingTypes.SCHEDULER,
1595 ],
1596 # SECURITY: Enable envelope encryption for Kubernetes secrets using KMS
1597 secrets_encryption_key=self.eks_encryption_key,
1598 )
1600 # The EKS cluster security group's auto-generated ingress rule allows
1601 # 443 from the VPC CIDR, expressed as a CloudFormation token. cdk-nag's
1602 # SG-ingress rules can't resolve the token and throw; scope the
1603 # acknowledgment to the cluster construct so it can't mask an
1604 # open-ingress finding elsewhere in the stack.
1605 from gco.stacks.nag_suppressions import acknowledge_security_group_cidr_findings
1607 acknowledge_security_group_cidr_findings(
1608 self.cluster,
1609 reason=(
1610 "The EKS Auto Mode cluster security group allows HTTPS (443) "
1611 "ingress from the VPC CIDR only, referenced via an "
1612 "``Fn::GetAtt`` token that cdk-nag cannot resolve at synth "
1613 "time. Ingress is restricted to intra-VPC traffic, the "
1614 "tightest possible source for the Kubernetes API and webhook "
1615 "endpoints."
1616 ),
1617 )
1619 # Auto Mode comes with essential addons pre-configured:
1620 # - AWS Load Balancer Controller (for ALB/NLB integration)
1621 # - CoreDNS, kube-proxy, VPC CNI (standard Kubernetes components)
1623 # OIDC provider for IRSA — the primary credential injection mechanism.
1624 # IRSA uses projected service-account tokens exchanged via the OIDC provider
1625 # for temporary AWS credentials. This works reliably on EKS Auto Mode.
1626 self.oidc_provider = eks.OidcProviderNative(
1627 self,
1628 "OidcProvider",
1629 url=self.cluster.cluster_open_id_connect_issuer_url,
1630 )
1632 # Pod Identity Agent add-on — registers the admission webhook that injects
1633 # Pod Identity credentials. On Auto Mode the DaemonSet schedules 0 pods
1634 # (the agent is built into the node), but the add-on registration is still
1635 # needed for the control-plane webhook. Kept as a secondary credential path.
1636 self._create_pod_identity_agent_addon()
1638 # Add Metrics Server add-on for HPA and resource monitoring
1639 self._create_metrics_server_addon()
1641 # Add EFS CSI Driver add-on for shared storage
1642 self._create_efs_csi_driver_addon()
1644 # Add CloudWatch Observability add-on for Container Insights metrics
1645 self._create_cloudwatch_observability_addon()
1647 # NOTE: GPU compute is configured via Karpenter NodePools (not managed node groups)
1648 # NodePool manifests are located in lambda/kubectl-applier-simple/manifests/:
1649 # - 40-nodepool-gpu-x86.yaml: active x86_64 general GPU instances (g4dn, g5, g6/g6e/g6f/gr6/gr6f, g7/g7e; deprecated p3/p3dn excluded)
1650 # - 41-nodepool-gpu-arm.yaml: ARM64 GPU instances (g5g)
1651 # - 42-nodepool-inference.yaml: inference-optimized GPU instances
1652 # - 43-nodepool-efa.yaml: EFA-enabled instances (p4d, p4de, p5/p5e/p5en, p6-b200/p6-b300/p6e-gb200)
1653 # - 44-nodepool-neuron.yaml: Trainium/Inferentia instances
1654 # These will be applied by the kubectl Lambda custom resource (created below)
1656 # Developer access entries (eks_cluster.developer_access). The cluster
1657 # authenticates through EKS access entries only, and until now no human
1658 # principal could be granted one at deploy time — every access entry
1659 # belonged to a platform Lambda. Absent or empty config synthesizes
1660 # exactly today's entries.
1661 self._create_developer_access_entries(eks_config)
1663 # Create IRSA role for service account to access secrets
1664 self._create_service_account_role()
1666 # Create kubectl Lambda for applying Kubernetes manifests
1667 self._create_kubectl_lambda()
1669 def _create_developer_access_entries(self, eks_config: dict[str, Any]) -> None:
1670 """Synthesize one EKS access entry per configured developer principal.
1672 Each ``eks_cluster.developer_access`` element is
1673 ``{principal_arn, scope, namespaces}``. The default is deliberately
1674 the narrow grant — AmazonEKSEditPolicy scoped to the ``gco-jobs``
1675 namespace, enough to submit and inspect jobs — not cluster-admin.
1676 ``scope: cluster`` opts into AmazonEKSClusterAdminPolicy explicitly.
1677 Config errors fail synthesis: a typo'd access grant must never
1678 silently synthesize as nothing.
1679 """
1680 entries = eks_config.get("developer_access") or []
1681 if not isinstance(entries, list):
1682 raise ValueError("eks_cluster.developer_access must be a list of entries")
1683 for index, entry in enumerate(entries):
1684 if not isinstance(entry, dict):
1685 raise ValueError(
1686 f"eks_cluster.developer_access[{index}] must be an object with "
1687 "principal_arn, and optionally scope and namespaces"
1688 )
1689 principal_arn = entry.get("principal_arn")
1690 if not isinstance(principal_arn, str) or not principal_arn.startswith("arn:"):
1691 raise ValueError(
1692 f"eks_cluster.developer_access[{index}].principal_arn must be an "
1693 f"IAM principal ARN, got {principal_arn!r}"
1694 )
1695 scope = str(entry.get("scope", "namespace")).strip().lower()
1696 if scope == "cluster":
1697 access_policy = eks.AccessPolicy.from_access_policy_name(
1698 "AmazonEKSClusterAdminPolicy",
1699 access_scope_type=eks.AccessScopeType.CLUSTER,
1700 )
1701 elif scope == "namespace":
1702 namespaces = entry.get("namespaces") or ["gco-jobs"]
1703 if not isinstance(namespaces, list) or not all(
1704 isinstance(namespace, str) and namespace for namespace in namespaces
1705 ):
1706 raise ValueError(
1707 f"eks_cluster.developer_access[{index}].namespaces must be a "
1708 "list of namespace names"
1709 )
1710 access_policy = eks.AccessPolicy.from_access_policy_name(
1711 "AmazonEKSEditPolicy",
1712 access_scope_type=eks.AccessScopeType.NAMESPACE,
1713 namespaces=namespaces,
1714 )
1715 else:
1716 raise ValueError(
1717 f"eks_cluster.developer_access[{index}].scope must be 'namespace' "
1718 f"or 'cluster', got {entry.get('scope')!r}"
1719 )
1720 eks.AccessEntry(
1721 self,
1722 f"DeveloperAccessEntry{index}",
1723 cluster=self.cluster, # type: ignore[arg-type]
1724 principal=principal_arn,
1725 access_policies=[access_policy],
1726 )
1728 # ── Shared toleration config for EKS add-ons ──────────────────────────
1729 # All GCO nodepools apply taints (nvidia.com/gpu, aws.amazon.com/neuron,
1730 # vpc.amazonaws.com/efa) that prevent DaemonSet pods from scheduling.
1731 # Every add-on component that runs as a DaemonSet (storage drivers' node
1732 # agents, metrics/log agents, node exporters) must tolerate these taints
1733 # so infrastructure works on every node type.
1734 #
1735 # Deployment-shaped add-on components (metrics-server, the CSI
1736 # *controllers*) must NOT carry these tolerations: a toleration makes
1737 # EKS Auto Mode consider the tainted accelerator pools for them, and
1738 # during a deploy's pod surge it will happily launch GPU instances for
1739 # zero-GPU pods and then consolidation-flap them — live release
1740 # validation run sched241-350ffc7d caught exactly that (two g4dn.xlarge
1741 # NodeClaims requesting ``nvidia.com/gpu: "0"``, churned mid-install,
1742 # failing GPU DaemonSet convergence checks).
1743 _ADDON_NODE_TOLERATIONS = [
1744 {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"},
1745 {"key": "aws.amazon.com/neuron", "operator": "Exists", "effect": "NoSchedule"},
1746 {"key": "vpc.amazonaws.com/efa", "operator": "Exists", "effect": "NoSchedule"},
1747 ]
1749 def _create_pod_identity_agent_addon(self) -> None:
1750 """Create EKS Pod Identity Agent add-on.
1752 On Auto Mode the DaemonSet schedules 0 pods (the agent is built into
1753 the node runtime), but the add-on registration is still required for
1754 the control-plane admission webhook that injects Pod Identity tokens.
1755 """
1756 eks.Addon(
1757 self,
1758 "PodIdentityAgentAddon",
1759 cluster=self.cluster, # type: ignore[arg-type]
1760 addon_name="eks-pod-identity-agent",
1761 addon_version=EKS_ADDON_POD_IDENTITY_AGENT,
1762 preserve_on_delete=False,
1763 configuration_values={
1764 "tolerations": self._ADDON_NODE_TOLERATIONS,
1765 },
1766 )
1768 def _create_metrics_server_addon(self) -> None:
1769 """Create Metrics Server add-on for resource metrics.
1771 The Metrics Server collects resource metrics from kubelets and exposes
1772 them via the Kubernetes API server. This is required for:
1773 - Horizontal Pod Autoscaler (HPA)
1774 - Vertical Pod Autoscaler (VPA)
1775 - kubectl top commands
1776 - Resource monitoring dashboards
1778 Note: Metrics Server doesn't require an IRSA role as it only needs
1779 in-cluster permissions which are handled by its service account.
1780 """
1781 # Deployment-shaped: no accelerator tolerations on purpose (see
1782 # _ADDON_NODE_TOLERATIONS) — metrics-server runs fine on the default
1783 # CPU pool and a toleration invites Auto Mode to launch GPU nodes
1784 # for it during deploy pod surges.
1785 eks.Addon(
1786 self,
1787 "MetricsServerAddon",
1788 cluster=self.cluster, # type: ignore[arg-type]
1789 addon_name="metrics-server",
1790 addon_version=EKS_ADDON_METRICS_SERVER,
1791 preserve_on_delete=False,
1792 )
1794 def _create_efs_csi_driver_addon(self) -> None:
1795 """Create EFS CSI Driver add-on for shared storage support.
1797 The EFS CSI driver enables Kubernetes pods to mount EFS file systems
1798 as persistent volumes. This is required for the shared storage feature.
1800 We create a Pod Identity role for the EFS CSI driver and update the add-on
1801 to use it via a custom resource after the add-on is created.
1802 """
1803 # Create IAM role for EFS CSI Driver using IRSA + Pod Identity
1804 self.efs_csi_role = GCORegionalStack._create_irsa_role(
1805 self,
1806 "EfsCsiDriverRole",
1807 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
1808 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
1809 service_account_names=["efs-csi-controller-sa"],
1810 namespaces=["kube-system"],
1811 )
1813 # Add EFS CSI driver permissions
1814 self.efs_csi_role.add_managed_policy(
1815 iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AmazonEFSCSIDriverPolicy")
1816 )
1818 # Create EFS CSI Driver add-on
1819 efs_addon = eks.Addon(
1820 self,
1821 "EfsCsiDriverAddon",
1822 cluster=self.cluster, # type: ignore[arg-type]
1823 addon_name="aws-efs-csi-driver",
1824 addon_version=EKS_ADDON_EFS_CSI_DRIVER,
1825 preserve_on_delete=False,
1826 configuration_values={
1827 # DaemonSet node agent must run on every node type; the
1828 # Deployment-shaped controller deliberately carries no
1829 # accelerator tolerations (see _ADDON_NODE_TOLERATIONS).
1830 "node": {
1831 "tolerations": self._ADDON_NODE_TOLERATIONS,
1832 },
1833 },
1834 )
1836 # Append the PassRole statement for the EFS CSI role to the shared
1837 # AwsCustomResource execution role. See the role's creation in
1838 # _create_aws_custom_resource_role for the full rationale on why
1839 # we pre-create + attach up-front instead of letting CDK
1840 # auto-generate per-CR roles.
1841 self.aws_custom_resource_role.add_to_policy(
1842 iam.PolicyStatement(
1843 effect=iam.Effect.ALLOW,
1844 actions=["iam:PassRole"],
1845 resources=[self.efs_csi_role.role_arn],
1846 )
1847 )
1849 # Update the add-on to use the IRSA role via custom resource
1850 # This is needed because the eks v2 alpha Addon doesn't support service_account_role directly
1851 update_addon = cr.AwsCustomResource(
1852 self,
1853 "UpdateEfsCsiAddonRole",
1854 on_create=cr.AwsSdkCall(
1855 service="EKS",
1856 action="updateAddon",
1857 parameters={
1858 "clusterName": self.cluster.cluster_name,
1859 "addonName": "aws-efs-csi-driver",
1860 "serviceAccountRoleArn": self.efs_csi_role.role_arn,
1861 },
1862 physical_resource_id=cr.PhysicalResourceId.of(
1863 f"{self.cluster.cluster_name}-efs-csi-role-update"
1864 ),
1865 ),
1866 on_update=cr.AwsSdkCall(
1867 service="EKS",
1868 action="updateAddon",
1869 parameters={
1870 "clusterName": self.cluster.cluster_name,
1871 "addonName": "aws-efs-csi-driver",
1872 "serviceAccountRoleArn": self.efs_csi_role.role_arn,
1873 },
1874 ),
1875 role=self.aws_custom_resource_role,
1876 )
1878 # Ensure the update happens after the add-on is created. We also
1879 # depend on the shared execution role so CloudFormation has fully
1880 # attached + replicated its inline policy before the Lambda fires.
1881 update_addon.node.add_dependency(efs_addon)
1882 update_addon.node.add_dependency(self.efs_csi_role)
1883 update_addon.node.add_dependency(self.aws_custom_resource_role)
1885 # Expose the update-addon resource so _apply_kubernetes_manifests can
1886 # make the kubectl Lambda wait for the IRSA annotation patch to land
1887 # before it tries to rollout-restart the efs-csi-controller. Without
1888 # this ordering, the restart could fire before EKS has re-attached
1889 # the role ARN, leaving the new pods just as credential-less as the
1890 # old ones and causing every EFS CreateAccessPoint to fail with a
1891 # 401 from IMDS.
1892 self._efs_csi_addon_role_update = update_addon
1894 def _create_cloudwatch_observability_addon(self) -> None:
1895 """Create CloudWatch Observability add-on for Container Insights.
1897 The CloudWatch Observability add-on enables Container Insights metrics
1898 for the EKS cluster, providing visibility into:
1899 - Cluster CPU and memory utilization
1900 - Node-level metrics
1901 - Pod and container metrics
1902 - Application logs (optional)
1904 These metrics are used by the monitoring dashboard to display
1905 cluster health and resource utilization.
1906 """
1908 # Create IAM role for CloudWatch agent using IRSA + Pod Identity
1909 self.cloudwatch_role = GCORegionalStack._create_irsa_role(
1910 self,
1911 "CloudWatchObservabilityRole",
1912 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
1913 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
1914 service_account_names=["cloudwatch-agent"],
1915 namespaces=["amazon-cloudwatch"],
1916 )
1918 # Add CloudWatch agent permissions
1919 self.cloudwatch_role.add_managed_policy(
1920 iam.ManagedPolicy.from_aws_managed_policy_name("CloudWatchAgentServerPolicy")
1921 )
1922 self.cloudwatch_role.add_managed_policy(
1923 iam.ManagedPolicy.from_aws_managed_policy_name("AWSXrayWriteOnlyAccess")
1924 )
1926 # Create CloudWatch Observability add-on
1927 cw_addon = eks.Addon(
1928 self,
1929 "CloudWatchObservabilityAddon",
1930 cluster=self.cluster, # type: ignore[arg-type]
1931 addon_name="amazon-cloudwatch-observability",
1932 addon_version=EKS_ADDON_CLOUDWATCH_OBSERVABILITY,
1933 preserve_on_delete=False,
1934 configuration_values={
1935 "tolerations": self._ADDON_NODE_TOLERATIONS,
1936 # Enable Container Insights with application log collection
1937 # Logs are sent to /aws/containerinsights/{cluster}/application
1938 "containerLogs": {
1939 "enabled": True,
1940 },
1941 },
1942 )
1944 # Append the PassRole statement for the CloudWatch Observability
1945 # role to the shared AwsCustomResource execution role. See
1946 # _create_aws_custom_resource_role for the full rationale.
1947 self.aws_custom_resource_role.add_to_policy(
1948 iam.PolicyStatement(
1949 effect=iam.Effect.ALLOW,
1950 actions=["iam:PassRole"],
1951 resources=[self.cloudwatch_role.role_arn],
1952 )
1953 )
1955 # Update the add-on to use the IRSA role via custom resource
1956 update_cw_addon = cr.AwsCustomResource(
1957 self,
1958 "UpdateCloudWatchAddonRole",
1959 on_create=cr.AwsSdkCall(
1960 service="EKS",
1961 action="updateAddon",
1962 parameters={
1963 "clusterName": self.cluster.cluster_name,
1964 "addonName": "amazon-cloudwatch-observability",
1965 "serviceAccountRoleArn": self.cloudwatch_role.role_arn,
1966 },
1967 physical_resource_id=cr.PhysicalResourceId.of(
1968 f"{self.cluster.cluster_name}-cw-obs-role-update"
1969 ),
1970 ),
1971 on_update=cr.AwsSdkCall(
1972 service="EKS",
1973 action="updateAddon",
1974 parameters={
1975 "clusterName": self.cluster.cluster_name,
1976 "addonName": "amazon-cloudwatch-observability",
1977 "serviceAccountRoleArn": self.cloudwatch_role.role_arn,
1978 },
1979 ),
1980 role=self.aws_custom_resource_role,
1981 )
1983 # Ensure the update happens after the add-on is created. Depend on
1984 # the shared execution role so CFN has fully attached + replicated
1985 # its inline policy before the Lambda fires. No CR→CR dependency
1986 # chain needed anymore — the race it was serializing against is
1987 # eliminated by pre-creating the role.
1988 update_cw_addon.node.add_dependency(cw_addon)
1989 update_cw_addon.node.add_dependency(self.cloudwatch_role)
1990 update_cw_addon.node.add_dependency(self.aws_custom_resource_role)
1992 # Expose the update-addon resource so _apply_kubernetes_manifests can
1993 # make the kubectl Lambda wait for the IRSA annotation patch to land
1994 # before it rollout-restarts the cloudwatch-agent DaemonSet. See the
1995 # EFS CSI equivalent for the full rationale — same race, same fix.
1996 self._cloudwatch_addon_role_update = update_cw_addon
1998 def _create_service_account_role(self) -> None:
1999 """Create IAM role for Kubernetes service account using EKS Pod Identity.
2001 Pod Identity is the recommended mechanism for EKS Auto Mode. It's simpler
2002 and more reliable than IRSA — no OIDC provider, no webhook injection, no
2003 projected tokens. EKS manages the credential injection automatically.
2005 The general workload role is deliberately separate from the manifest
2006 processor role. Job and inference workload service accounts must never
2007 receive queue-table mutation privileges; only the platform API/worker
2008 identity can claim, fence, or transition centralized queue records.
2009 """
2010 self.service_account_role = GCORegionalStack._create_irsa_role(
2011 self,
2012 "ServiceAccountRole",
2013 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
2014 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
2015 service_account_names=[
2016 "gco-service-account",
2017 "gco-inference-monitor-sa",
2018 ],
2019 namespaces=["gco-system", "gco-jobs", "gco-inference"],
2020 )
2022 self.manifest_processor_role = GCORegionalStack._create_irsa_role(
2023 self,
2024 "ManifestProcessorRole",
2025 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
2026 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
2027 service_account_names=["gco-manifest-processor-sa"],
2028 namespaces=["gco-system"],
2029 )
2031 self.inference_proxy_role = GCORegionalStack._create_irsa_role(
2032 self,
2033 "InferenceProxyRole",
2034 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
2035 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
2036 service_account_names=["gco-inference-proxy-sa"],
2037 namespaces=["gco-system"],
2038 )
2040 self.health_monitor_role = GCORegionalStack._create_irsa_role(
2041 self,
2042 "HealthMonitorRole",
2043 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
2044 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
2045 service_account_names=["gco-health-monitor-sa"],
2046 namespaces=["gco-system"],
2047 )
2049 if self._cost_monitoring_active():
2050 self.cost_monitor_role = GCORegionalStack._create_irsa_role(
2051 self,
2052 "CostMonitorRole",
2053 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
2054 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
2055 service_account_names=["gco-cost-monitor-sa"],
2056 namespaces=["gco-system"],
2057 )
2058 self._grant_cost_report_bucket_discovery_to_cost_monitor()
2060 self._create_aws_load_balancer_controller_role()
2062 # Grant permission to read the auth secret.
2063 #
2064 # The resource is built as a *deterministic* ARN from the known secret
2065 # name, the API Gateway region (where the secret lives), and this
2066 # stack's account — rather than from ``self.auth_secret_arn``, which is
2067 # ``api_gateway_stack.secret.secret_arn`` (a cross-stack reference
2068 # token). The token was the source of issue #125: it renders
2069 # differently depending on topology —
2070 # * cross-region -> a literal ARN, and
2071 # * same-region -> a native cross-stack export
2072 # (``gco-api-gateway:ExportsOutputRefGCOAuthSecret<hash>``)
2073 # so the trailing-``*`` IAM resource only matched the stack-level
2074 # AwsSolutions-IAM5 suppression in the cross-region (default) topology.
2075 # Collapsing every stack into one region left the export-token form
2076 # unsuppressed and failed ``cdk synth``. Building the ARN ourselves
2077 # makes the ``Resource`` render identically in both topologies so a
2078 # single deterministic suppression (see ``add_iam_suppressions``)
2079 # always matches.
2080 #
2081 # The trailing ``*`` matches the random 6-character suffix Secrets
2082 # Manager appends to secret ARNs (Secrets Manager accepts either the
2083 # full ARN with suffix or the partial ARN without it).
2084 auth_secret_resource = (
2085 f"arn:{self.partition}:secretsmanager:{self.config.get_api_gateway_region()}"
2086 f":{self.account}:secret:{api_gateway_auth_secret_name(self.config.get_project_name())}*"
2087 )
2088 self.manifest_processor_role.add_to_policy(
2089 iam.PolicyStatement(
2090 effect=iam.Effect.ALLOW,
2091 actions=[
2092 "secretsmanager:GetSecretValue",
2093 "secretsmanager:DescribeSecret",
2094 ],
2095 resources=[auth_secret_resource],
2096 )
2097 )
2098 self.inference_proxy_role.add_to_policy(
2099 iam.PolicyStatement(
2100 effect=iam.Effect.ALLOW,
2101 actions=[
2102 "secretsmanager:GetSecretValue",
2103 "secretsmanager:DescribeSecret",
2104 ],
2105 resources=[auth_secret_resource],
2106 )
2107 )
2108 self.health_monitor_role.add_to_policy(
2109 iam.PolicyStatement(
2110 effect=iam.Effect.ALLOW,
2111 actions=[
2112 "secretsmanager:GetSecretValue",
2113 "secretsmanager:DescribeSecret",
2114 ],
2115 resources=[auth_secret_resource],
2116 )
2117 )
2119 from gco.stacks.nag_suppressions import acknowledge_nag_findings
2121 # The SQS queue processor runs as gco-manifest-processor-sa. Keep
2122 # queue consumption on that dedicated platform identity; KEDA has its
2123 # own read-only queue role and general workload identities must not be
2124 # able to receive or delete submitted jobs.
2125 self.manifest_processor_role.add_to_policy(
2126 iam.PolicyStatement(
2127 effect=iam.Effect.ALLOW,
2128 actions=[
2129 "sqs:ReceiveMessage",
2130 "sqs:DeleteMessage",
2131 ],
2132 resources=[self.job_queue.queue_arn],
2133 )
2134 )
2136 # Manifest API/worker metrics are emitted only by the dedicated
2137 # platform identity, never by user workload service accounts.
2138 self.manifest_processor_role.add_to_policy(
2139 iam.PolicyStatement(
2140 effect=iam.Effect.ALLOW,
2141 actions=["cloudwatch:PutMetricData"],
2142 resources=["*"],
2143 conditions={"StringEquals": {"cloudwatch:namespace": "GCO/ManifestProcessor"}},
2144 )
2145 )
2147 # The central queue worker's spot price gate reads current spot
2148 # pricing before dispatching price-capped jobs.
2149 # ec2:DescribeSpotPriceHistory is a read-only Describe* action that
2150 # does not support resource-level scoping (Resource must be *).
2151 self.manifest_processor_role.add_to_policy(
2152 iam.PolicyStatement(
2153 effect=iam.Effect.ALLOW,
2154 actions=["ec2:DescribeSpotPriceHistory"],
2155 resources=["*"],
2156 )
2157 )
2159 # Add DynamoDB permissions for templates, webhooks, and job queue
2160 # Tables are created in the global stack and accessed from all regions
2161 project_name = self.config.get_project_name()
2162 global_region = self.config.get_global_region()
2164 # Health-monitor runtime grants are isolated from the shared workload
2165 # role. It can read/repair one endpoint-registry parameter, read webhook
2166 # subscriptions, publish only its metric namespace, and read the auth
2167 # secret granted above.
2168 self.health_monitor_role.add_to_policy(
2169 iam.PolicyStatement(
2170 effect=iam.Effect.ALLOW,
2171 actions=["ssm:GetParameter", "ssm:PutParameter"],
2172 resources=[
2173 f"arn:{self.partition}:ssm:{global_region}:{self.account}:"
2174 f"parameter/{project_name}/alb-hostname-{self.deployment_region}"
2175 ],
2176 )
2177 )
2178 self.health_monitor_role.add_to_policy(
2179 iam.PolicyStatement(
2180 effect=iam.Effect.ALLOW,
2181 actions=["dynamodb:Query", "dynamodb:Scan"],
2182 resources=[
2183 f"arn:{self.partition}:dynamodb:{global_region}:{self.account}:"
2184 f"table/{project_name}-webhooks",
2185 f"arn:{self.partition}:dynamodb:{global_region}:{self.account}:"
2186 f"table/{project_name}-webhooks/index/namespace-index",
2187 ],
2188 )
2189 )
2190 self.health_monitor_role.add_to_policy(
2191 iam.PolicyStatement(
2192 effect=iam.Effect.ALLOW,
2193 actions=["cloudwatch:PutMetricData"],
2194 resources=["*"],
2195 conditions={"StringEquals": {"cloudwatch:namespace": "GCO/HealthMonitor"}},
2196 )
2197 )
2198 acknowledge_nag_findings(
2199 self.health_monitor_role,
2200 [
2201 {
2202 "id": "AwsSolutions-IAM5",
2203 "reason": (
2204 "HealthMonitorRole has two unavoidable wildcard shapes: the "
2205 "Secrets Manager random ARN suffix and cloudwatch:PutMetricData's "
2206 "required Resource:*. PutMetricData is constrained to the exact "
2207 "GCO/HealthMonitor namespace; all SSM and DynamoDB resources are exact."
2208 ),
2209 "appliesTo": ["Resource::*"],
2210 }
2211 ],
2212 )
2214 # The manifest processor is the only identity that can mutate the
2215 # centralized queue. Workload identities receive no access to the jobs
2216 # table, preventing submitted pods from forging queue state.
2217 manifest_table_prefix = (
2218 f"arn:{self.partition}:dynamodb:{global_region}:{self.account}:table/{project_name}"
2219 )
2220 self.manifest_processor_role.add_to_policy(
2221 iam.PolicyStatement(
2222 effect=iam.Effect.ALLOW,
2223 actions=[
2224 "dynamodb:GetItem",
2225 "dynamodb:PutItem",
2226 "dynamodb:UpdateItem",
2227 "dynamodb:DeleteItem",
2228 "dynamodb:Query",
2229 "dynamodb:Scan",
2230 ],
2231 resources=[
2232 f"{manifest_table_prefix}-job-templates",
2233 f"{manifest_table_prefix}-job-templates/index/*",
2234 f"{manifest_table_prefix}-webhooks",
2235 f"{manifest_table_prefix}-webhooks/index/*",
2236 ],
2237 )
2238 )
2239 self.manifest_processor_role.add_to_policy(
2240 iam.PolicyStatement(
2241 effect=iam.Effect.ALLOW,
2242 actions=[
2243 "dynamodb:GetItem",
2244 "dynamodb:PutItem",
2245 "dynamodb:UpdateItem",
2246 "dynamodb:Query",
2247 "dynamodb:Scan",
2248 ],
2249 resources=[
2250 f"{manifest_table_prefix}-jobs",
2251 f"{manifest_table_prefix}-jobs/index/*",
2252 ],
2253 )
2254 )
2256 # The inference proxy needs only point reads of endpoint state. It has
2257 # no write, scan, index, S3, Kubernetes, or queue permissions.
2258 self.inference_proxy_role.add_to_policy(
2259 iam.PolicyStatement(
2260 effect=iam.Effect.ALLOW,
2261 actions=["dynamodb:GetItem"],
2262 resources=[f"{manifest_table_prefix}-inference-endpoints"],
2263 )
2264 )
2265 acknowledge_nag_findings(
2266 self.inference_proxy_role,
2267 [
2268 {
2269 "id": "AwsSolutions-IAM5",
2270 "reason": (
2271 "InferenceProxyRole uses one wildcard only for the random "
2272 "Secrets Manager ARN suffix. DynamoDB access is an exact-table "
2273 "GetItem grant, and the role has no Kubernetes, queue, or write access."
2274 ),
2275 "appliesTo": ["Resource::*"],
2276 }
2277 ],
2278 )
2280 # The inference monitor still owns desired-state reconciliation, but
2281 # this shared role intentionally has no jobs-table ARN.
2282 self.service_account_role.add_to_policy(
2283 iam.PolicyStatement(
2284 effect=iam.Effect.ALLOW,
2285 actions=[
2286 "dynamodb:GetItem",
2287 "dynamodb:PutItem",
2288 "dynamodb:UpdateItem",
2289 "dynamodb:DeleteItem",
2290 "dynamodb:Query",
2291 "dynamodb:Scan",
2292 ],
2293 resources=[
2294 f"{manifest_table_prefix}-inference-endpoints",
2295 f"{manifest_table_prefix}-inference-endpoints/index/*",
2296 ],
2297 )
2298 )
2299 acknowledge_nag_findings(
2300 self.manifest_processor_role,
2301 [
2302 {
2303 "id": "AwsSolutions-IAM5",
2304 "reason": (
2305 "ManifestProcessorRole has only four required wildcard shapes: "
2306 "DynamoDB secondary indexes, the Secrets Manager generated ARN "
2307 "suffix, cloudwatch:PutMetricData Resource:*, and the read-only "
2308 "ec2:DescribeSpotPriceHistory Resource:* (Describe* actions do "
2309 "not support resource-level scoping; the central queue worker's "
2310 "spot price gate needs current pricing). DynamoDB table names "
2311 "and the CloudWatch namespace are otherwise exact."
2312 ),
2313 "appliesTo": ["Resource::*"],
2314 }
2315 ],
2316 )
2318 # Workload pods must run as gco-service-account to write artifacts
2319 # (it is the only identity with RW on the regional shared bucket plus
2320 # KMS encrypt), and without this statement that same role could not
2321 # publish training metrics — CloudWatch denied PutMetricData with a
2322 # warning most trainers swallow. Grant exactly one namespace,
2323 # configurable via cdk.json::workload_metrics.cloudwatch_namespace so
2324 # a deployment can point its own consumers at it; the default is a
2325 # GCO-owned workload namespace. PutMetricData does not support
2326 # resource-level scoping (Resource must be *), so the namespace
2327 # condition carries the whole restriction — the same shape as every
2328 # platform role's metrics grant.
2329 workload_metrics_config = self.node.try_get_context("workload_metrics") or {}
2330 workload_metrics_namespace = str(
2331 workload_metrics_config.get("cloudwatch_namespace") or "GCO/Workloads"
2332 )
2333 self.service_account_role.add_to_policy(
2334 iam.PolicyStatement(
2335 effect=iam.Effect.ALLOW,
2336 actions=["cloudwatch:PutMetricData"],
2337 resources=["*"],
2338 conditions={"StringEquals": {"cloudwatch:namespace": workload_metrics_namespace}},
2339 )
2340 )
2341 acknowledge_nag_findings(
2342 self.service_account_role,
2343 [
2344 {
2345 "id": "AwsSolutions-IAM5",
2346 "reason": (
2347 "cloudwatch:PutMetricData does not support resource-level "
2348 "scoping and requires Resource:*. The statement is "
2349 "constrained by a StringEquals condition to exactly one "
2350 "configured workload metric namespace, matching the "
2351 "namespace-conditioned metrics grants on the platform roles."
2352 ),
2353 "appliesTo": ["Resource::*"],
2354 }
2355 ],
2356 )
2358 # Vector-store read path for workloads (opt-in feature). Deliberately
2359 # LOCAL-region ARNs: the store is a DynamoDB global table with a
2360 # replica in this cluster's region, so pods query locally instead of
2361 # crossing regions to the primary. Every resource is exact — the
2362 # table and index names are deterministic from config — and the
2363 # grant is read-only: writes belong to the ingest Lambda in the
2364 # global stack, so a compromised workload cannot poison the corpus.
2365 # The embedding-model grant lets pods embed their own query text
2366 # with the exact model the corpus was ingested with.
2367 if self.config.get_vector_store_enabled():
2368 vector_store_table_prefix = (
2369 f"arn:{self.partition}:dynamodb:{self.region}:{self.account}:"
2370 f"table/{project_name}-vector-store"
2371 )
2372 self.service_account_role.add_to_policy(
2373 iam.PolicyStatement(
2374 effect=iam.Effect.ALLOW,
2375 actions=[
2376 "dynamodb:SearchVectors",
2377 "dynamodb:GetItem",
2378 "dynamodb:Query",
2379 ],
2380 resources=[
2381 vector_store_table_prefix,
2382 f"{vector_store_table_prefix}/index/corpus-embedding-index",
2383 ],
2384 )
2385 )
2386 self.service_account_role.add_to_policy(
2387 iam.PolicyStatement(
2388 effect=iam.Effect.ALLOW,
2389 actions=["bedrock:InvokeModel"],
2390 resources=[
2391 f"arn:{self.partition}:bedrock:{self.region}::foundation-model/"
2392 f"{self.config.get_vector_store_config()['embedding_model_id']}"
2393 ],
2394 )
2395 )
2397 # Add S3 permissions for model weights bucket (used by inference init containers)
2398 self.service_account_role.add_to_policy(
2399 iam.PolicyStatement(
2400 effect=iam.Effect.ALLOW,
2401 actions=[
2402 "s3:GetObject",
2403 "s3:ListBucket",
2404 ],
2405 resources=[
2406 f"arn:{self.partition}:s3:::{project_name}-*",
2407 f"arn:{self.partition}:s3:::{project_name}-*/*",
2408 ],
2409 )
2410 )
2412 # KMS decrypt for model weights bucket (S3-scoped)
2413 self.service_account_role.add_to_policy(
2414 iam.PolicyStatement(
2415 effect=iam.Effect.ALLOW,
2416 actions=["kms:Decrypt", "kms:GenerateDataKey"],
2417 resources=[f"arn:{self.partition}:kms:*:{self.account}:key/*"],
2418 conditions={
2419 "StringLike": {
2420 "kms:ViaService": f"s3.*.{self.url_suffix}",
2421 }
2422 },
2423 )
2424 )
2426 # Create KEDA operator IAM role for SQS access
2427 self._create_keda_operator_role()
2429 # Create Pod Identity Associations for all service accounts
2430 self._create_pod_identity_associations()
2432 def _create_aws_load_balancer_controller_role(self) -> None:
2433 """Create the controller's exact OIDC-only IRSA role and v3.4.2 policy."""
2434 self.aws_load_balancer_controller_role = GCORegionalStack._create_irsa_role(
2435 self,
2436 "AwsLoadBalancerControllerRole",
2437 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
2438 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
2439 service_account_names=["aws-load-balancer-controller"],
2440 namespaces=["kube-system"],
2441 include_pod_identity=False,
2442 )
2443 self.aws_load_balancer_controller_policy = iam.Policy(
2444 self,
2445 "AwsLoadBalancerControllerPolicy",
2446 document=iam.PolicyDocument.from_json(
2447 aws_load_balancer_controller_policy_document(self.partition)
2448 ),
2449 )
2450 self.aws_load_balancer_controller_role.attach_inline_policy(
2451 self.aws_load_balancer_controller_policy
2452 )
2454 from gco.stacks.nag_suppressions import acknowledge_nag_findings
2456 acknowledge_nag_findings(
2457 self.aws_load_balancer_controller_policy,
2458 [
2459 {
2460 "id": "AwsSolutions-IAM5",
2461 "reason": (
2462 "This is the exact upstream AWS Load Balancer Controller v3.4.2 "
2463 "IAM policy. Its Resource::* entries cover AWS APIs that cannot "
2464 "be resource-scoped, while its wildcard ARN segments are limited "
2465 "to the controller's supported EC2 and ELB resource types and "
2466 "constrained by upstream cluster ownership tag conditions. The "
2467 "role trust is restricted to kube-system/aws-load-balancer-controller."
2468 ),
2469 "appliesTo": [
2470 "Resource::*",
2471 "Resource::arn:<AWS::Partition>:ec2:*:*:security-group/*",
2472 (
2473 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
2474 "loadbalancer/app/*/*"
2475 ),
2476 (
2477 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
2478 "loadbalancer/net/*/*"
2479 ),
2480 ("Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:targetgroup/*/*"),
2481 (
2482 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
2483 "listener-rule/app/*/*/*"
2484 ),
2485 (
2486 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
2487 "listener-rule/net/*/*/*"
2488 ),
2489 (
2490 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
2491 "listener/app/*/*/*"
2492 ),
2493 (
2494 "Resource::arn:<AWS::Partition>:elasticloadbalancing:*:*:"
2495 "listener/net/*/*/*"
2496 ),
2497 ],
2498 }
2499 ],
2500 )
2502 def _create_keda_operator_role(self) -> None:
2503 """Create IAM role for KEDA operator service account using EKS Pod Identity.
2505 This role allows the KEDA operator to access SQS queues for scaling
2506 based on queue depth. The role is assumed by the keda-operator service
2507 account in the keda namespace.
2508 """
2509 # Create IAM role with IRSA (OIDC) trust + Pod Identity trust
2510 self.keda_operator_role = GCORegionalStack._create_irsa_role(
2511 self,
2512 "KedaOperatorRole",
2513 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
2514 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
2515 service_account_names=["keda-operator"],
2516 namespaces=["keda"],
2517 )
2519 # Add SQS permissions for KEDA to read queue metrics
2520 self.keda_operator_role.add_to_policy(
2521 iam.PolicyStatement(
2522 effect=iam.Effect.ALLOW,
2523 actions=[
2524 "sqs:GetQueueAttributes",
2525 "sqs:GetQueueUrl",
2526 ],
2527 resources=[
2528 self.job_queue.queue_arn,
2529 self.job_dlq.queue_arn,
2530 ],
2531 )
2532 )
2534 # CloudWatch read permissions for GPU-based autoscaling. The KEDA
2535 # aws-cloudwatch scaler reads ContainerInsights GPU utilization metrics
2536 # to scale inference roles that request GPUs — GPU is not a native HPA
2537 # resource metric, so this is the only path that can drive GPU scaling.
2538 # The CloudWatch read APIs do not support resource-level IAM scoping, so
2539 # they are granted account-wide (read-only).
2540 self.keda_operator_role.add_to_policy(
2541 iam.PolicyStatement(
2542 effect=iam.Effect.ALLOW,
2543 actions=[
2544 "cloudwatch:GetMetricData",
2545 "cloudwatch:GetMetricStatistics",
2546 "cloudwatch:ListMetrics",
2547 ],
2548 resources=["*"],
2549 )
2550 )
2552 # cdk-nag suppression: the CloudWatch metric-read APIs do not support
2553 # resource-level IAM scoping — Resource: * is the only valid form.
2554 from gco.stacks.nag_suppressions import acknowledge_nag_findings
2556 acknowledge_nag_findings(
2557 self.keda_operator_role,
2558 [
2559 {
2560 "id": "AwsSolutions-IAM5",
2561 "reason": (
2562 "The KEDA operator reads CloudWatch metrics "
2563 "(GetMetricData, GetMetricStatistics, ListMetrics) to "
2564 "drive GPU-based autoscaling. These APIs do not support "
2565 "resource-level IAM scoping — Resource: * is the only "
2566 "valid form. The grant is read-only."
2567 ),
2568 "appliesTo": ["Resource::*"],
2569 },
2570 ],
2571 )
2573 def _create_pod_identity_associations(self) -> None:
2574 """Create EKS Pod Identity Associations for all service accounts.
2576 Pod Identity is the recommended mechanism for EKS Auto Mode. Each
2577 association links an IAM role to a Kubernetes service account in a
2578 specific namespace. EKS manages credential injection automatically.
2580 Stores associations in self._pod_identity_associations so the
2581 kubectl-applier custom resource can declare an explicit dependency,
2582 ensuring credentials are available before workloads start.
2583 """
2584 self._pod_identity_associations: list[Any] = []
2586 # Health monitor — isolated write access for ALB hostname self-healing.
2587 health_assoc = eks_l1.CfnPodIdentityAssociation(
2588 self,
2589 "PodIdentity-health-monitor",
2590 cluster_name=self.cluster.cluster_name,
2591 namespace="gco-system",
2592 service_account="gco-health-monitor-sa",
2593 role_arn=self.health_monitor_role.role_arn,
2594 )
2595 self._pod_identity_associations.append(health_assoc)
2597 # Manifest API and central queue worker — dedicated queue mutation role.
2598 manifest_assoc = eks_l1.CfnPodIdentityAssociation(
2599 self,
2600 "PodIdentity-manifest-processor",
2601 cluster_name=self.cluster.cluster_name,
2602 namespace="gco-system",
2603 service_account="gco-manifest-processor-sa",
2604 role_arn=self.manifest_processor_role.role_arn,
2605 )
2606 self._pod_identity_associations.append(manifest_assoc)
2608 # Inference data plane — exact secret + endpoint-table read role, with
2609 # no Kubernetes RBAC binding.
2610 inference_proxy_assoc = eks_l1.CfnPodIdentityAssociation(
2611 self,
2612 "PodIdentity-inference-proxy",
2613 cluster_name=self.cluster.cluster_name,
2614 namespace="gco-system",
2615 service_account="gco-inference-proxy-sa",
2616 role_arn=self.inference_proxy_role.role_arn,
2617 )
2618 self._pod_identity_associations.append(inference_proxy_assoc)
2620 # Inference monitor — reconciles endpoints with the shared platform
2621 # role (its IRSA subject is already in that role's trust policy).
2622 # Every other platform Deployment gets both credential paths; this one
2623 # had only the IRSA annotation.
2624 inference_monitor_assoc = eks_l1.CfnPodIdentityAssociation(
2625 self,
2626 "PodIdentity-inference-monitor",
2627 cluster_name=self.cluster.cluster_name,
2628 namespace="gco-system",
2629 service_account="gco-inference-monitor-sa",
2630 role_arn=self.service_account_role.role_arn,
2631 )
2632 self._pod_identity_associations.append(inference_monitor_assoc)
2634 # Cost monitor — only when the pipeline deploys here (the role and
2635 # 34-cost-monitor.yaml are gated the same way).
2636 if self._cost_monitoring_active():
2637 cost_monitor_assoc = eks_l1.CfnPodIdentityAssociation(
2638 self,
2639 "PodIdentity-cost-monitor",
2640 cluster_name=self.cluster.cluster_name,
2641 namespace="gco-system",
2642 service_account="gco-cost-monitor-sa",
2643 role_arn=self.cost_monitor_role.role_arn,
2644 )
2645 self._pod_identity_associations.append(cost_monitor_assoc)
2647 # Shared GCO service account for user job and inference workloads —
2648 # the two namespaces 01-serviceaccounts.yaml actually declares it in.
2649 # (A gco-system association used to be created as well; no such
2650 # ServiceAccount exists there, so it never bound anything.)
2651 for namespace in ["gco-jobs", "gco-inference"]:
2652 assoc = eks_l1.CfnPodIdentityAssociation(
2653 self,
2654 f"PodIdentity-gco-sa-{namespace}",
2655 cluster_name=self.cluster.cluster_name,
2656 namespace=namespace,
2657 service_account="gco-service-account",
2658 role_arn=self.service_account_role.role_arn,
2659 )
2660 self._pod_identity_associations.append(assoc)
2662 # KEDA operator — needs SQS access for queue-based scaling
2663 keda_assoc = eks_l1.CfnPodIdentityAssociation(
2664 self,
2665 "PodIdentity-keda-operator",
2666 cluster_name=self.cluster.cluster_name,
2667 namespace="keda",
2668 service_account="keda-operator",
2669 role_arn=self.keda_operator_role.role_arn,
2670 )
2671 self._pod_identity_associations.append(keda_assoc)
2673 # EFS CSI driver — needs EFS access for shared storage
2674 efs_assoc = eks_l1.CfnPodIdentityAssociation(
2675 self,
2676 "PodIdentity-efs-csi",
2677 cluster_name=self.cluster.cluster_name,
2678 namespace="kube-system",
2679 service_account="efs-csi-controller-sa",
2680 role_arn=self.efs_csi_role.role_arn,
2681 )
2682 self._pod_identity_associations.append(efs_assoc)
2684 # CloudWatch agent — needs CloudWatch access for observability
2685 cw_assoc = eks_l1.CfnPodIdentityAssociation(
2686 self,
2687 "PodIdentity-cloudwatch",
2688 cluster_name=self.cluster.cluster_name,
2689 namespace="amazon-cloudwatch",
2690 service_account="cloudwatch-agent",
2691 role_arn=self.cloudwatch_role.role_arn,
2692 )
2693 self._pod_identity_associations.append(cw_assoc)
2695 # FSx CSI driver — only when FSx is enabled (created later in _create_fsx_lustre)
2696 # The FSx Pod Identity association is added in _create_fsx_lustre instead
2698 def _resolve_cluster_shared_bucket_from_ssm(self) -> SharedBucketIdentity:
2699 """Resolve the ``Cluster_Shared_Bucket`` identity from cross-region SSM.
2701 ``GCOGlobalStack`` publishes three ``ssm.StringParameter``s in the
2702 global region at ``/gco/cluster-shared-bucket/{name,arn,region}``.
2703 This method reads them back from the regional stack via
2704 ``cr.AwsCustomResource`` with ``service="SSM"``,
2705 ``action="getParameter"``, and ``region=<global-region>`` — matching
2706 the cross-region read pattern already used in
2707 ``_create_ga_registration_lambda`` for the Global Accelerator
2708 endpoint group ARN.
2710 Runs unconditionally in ``__init__`` — no feature toggle, no
2711 conditional guard. The returned :class:`SharedBucketIdentity` feeds
2712 ``_grant_cluster_shared_bucket_to_job_role`` (IAM) and the
2713 ``image_replacements`` dict (ConfigMap) downstream.
2715 Returns:
2716 :class:`SharedBucketIdentity` with ``name``, ``arn``, and
2717 ``region`` populated as CDK tokens that resolve at deploy time.
2718 """
2719 from gco.stacks.nag_suppressions import acknowledge_nag_findings
2721 global_region = self.config.get_global_region()
2722 cluster_shared_prefix = cluster_shared_ssm_parameter_prefix(self.config.get_project_name())
2723 resolved: dict[str, str] = {}
2725 for suffix in ("name", "arn", "region"):
2726 parameter_name = f"{cluster_shared_prefix}/{suffix}"
2727 read_cr = cr.AwsCustomResource(
2728 self,
2729 f"ReadClusterSharedBucket{suffix.capitalize()}",
2730 on_create=cr.AwsSdkCall(
2731 service="SSM",
2732 action="getParameter",
2733 parameters={"Name": parameter_name},
2734 region=global_region,
2735 physical_resource_id=cr.PhysicalResourceId.of(f"cluster-shared-{suffix}"),
2736 ),
2737 on_update=cr.AwsSdkCall(
2738 service="SSM",
2739 action="getParameter",
2740 parameters={"Name": parameter_name},
2741 region=global_region,
2742 physical_resource_id=cr.PhysicalResourceId.of(f"cluster-shared-{suffix}"),
2743 ),
2744 # Cross-region SSM GetParameter doesn't support resource-level
2745 # scoping cleanly — the principal evaluating the call lives in
2746 # this stack's region but the parameter lives in the global
2747 # region. ANY_RESOURCE is the AWS-documented escape hatch; the
2748 # resulting AwsSolutions-IAM5 nag finding is suppressed in a
2749 # scoped add_resource_suppressions call below.
2750 policy=cr.AwsCustomResourcePolicy.from_sdk_calls(
2751 resources=cr.AwsCustomResourcePolicy.ANY_RESOURCE
2752 ),
2753 )
2755 # Scoped suppression: the CR policy is Resource::* because the
2756 # SSM parameter lives in the global region (cross-region calls
2757 # don't support resource-level scoping cleanly). The action is
2758 # fixed to ssm:GetParameter and the parameter Name is fixed to
2759 # a literal string, so the wildcard can only ever read one
2760 # parameter.
2761 acknowledge_nag_findings(
2762 read_cr,
2763 [
2764 {
2765 "id": "AwsSolutions-IAM5",
2766 "reason": (
2767 "Cross-region ssm:GetParameter for "
2768 f"{parameter_name} in the global region. The "
2769 "AwsCustomResource SDK-call policy is scoped to "
2770 "a single fixed action (ssm:GetParameter) with "
2771 "a fixed parameter Name — the Resource: * is "
2772 "the CDK-documented escape hatch because the "
2773 "parameter ARN is not known to the calling "
2774 "principal's region. Effective blast radius: "
2775 "one parameter."
2776 ),
2777 "appliesTo": ["Resource::*"],
2778 },
2779 ],
2780 )
2782 resolved[suffix] = read_cr.get_response_field("Parameter.Value")
2784 return SharedBucketIdentity(
2785 name=resolved["name"],
2786 arn=resolved["arn"],
2787 region=resolved["region"],
2788 )
2790 def _grant_cluster_shared_bucket_to_job_role(self, shared: SharedBucketIdentity) -> None:
2791 """Attach RW + KMS permissions on ``Cluster_Shared_Bucket`` to the job-pod role.
2793 Two ``iam.PolicyStatement``s are added to ``self.service_account_role``
2794 (the EKS Pod Identity role used by every pod in ``gco-jobs``,
2795 ``gco-system``, and ``gco-inference``):
2797 1. S3 object + bucket-level actions (``GetObject``, ``PutObject``,
2798 ``DeleteObject``, ``ListBucket``, ``GetBucketLocation``) scoped
2799 to ``<shared.arn>`` and ``<shared.arn>/*`` — the literal ARN
2800 resolved from SSM (the bucket's name is CloudFormation-generated,
2801 so no prefix pattern is involved).
2802 2. KMS ``Decrypt`` / ``GenerateDataKey`` scoped by the
2803 ``kms:ViaService=s3.<shared.region>.<AWS::URLSuffix>`` condition —
2804 ``resources=["*"]`` because the KMS key ARN is not known to this
2805 stack (it lives in the global region and is referenced indirectly
2806 through the S3 service). The condition is what actually restricts
2807 the grant to the cluster-shared bucket's key.
2809 Runs unconditionally — the grant is
2810 present on every regional cluster whether or not analytics is
2811 enabled.
2812 """
2813 from gco.stacks.nag_suppressions import acknowledge_nag_findings
2815 self.service_account_role.add_to_policy(
2816 iam.PolicyStatement(
2817 effect=iam.Effect.ALLOW,
2818 actions=[
2819 "s3:GetObject",
2820 "s3:PutObject",
2821 "s3:DeleteObject",
2822 "s3:ListBucket",
2823 "s3:GetBucketLocation",
2824 ],
2825 resources=[shared.arn, f"{shared.arn}/*"],
2826 )
2827 )
2829 self.service_account_role.add_to_policy(
2830 iam.PolicyStatement(
2831 effect=iam.Effect.ALLOW,
2832 actions=["kms:Decrypt", "kms:GenerateDataKey"],
2833 resources=["*"],
2834 conditions={
2835 "StringEquals": {
2836 "kms:ViaService": f"s3.{shared.region}.{self.url_suffix}",
2837 }
2838 },
2839 )
2840 )
2842 # The grants contain two necessary wildcard shapes. The S3 bucket ARN
2843 # uses ``/*`` for object keys within the single resolved shared bucket.
2844 # KMS uses ``Resource::*`` because the global key ARN is not exported to
2845 # this stack; ``kms:ViaService`` confines its use to S3 in the bucket's
2846 # region, while the S3 statements separately scope accessible objects.
2847 acknowledge_nag_findings(
2848 self.service_account_role,
2849 [
2850 {
2851 "id": "AwsSolutions-IAM5",
2852 "reason": (
2853 "The Cluster_Shared_Bucket grants require two wildcard shapes: "
2854 "an <arn>/* object-key suffix on the single shared bucket resolved "
2855 "from SSM, and KMS Resource::* because the global key ARN is not "
2856 "exported. KMS use is constrained by kms:ViaService to S3 in the "
2857 "bucket's region, and S3 access is separately limited to the "
2858 "allowed bucket ARNs."
2859 ),
2860 "appliesTo": [
2861 "Resource::*",
2862 "Resource::<ReadClusterSharedBucketArn4B0BD291.Parameter.Value>/*",
2863 ],
2864 },
2865 ],
2866 )
2868 def _create_mlflow_artifact_role(self, shared: SharedBucketIdentity) -> None:
2869 """Create the OIDC-only IRSA role MLflow uses for S3 artifact storage.
2871 The official mlflow chart creates a ``mlflow`` ServiceAccount in the
2872 ``monitoring`` namespace (``fullnameOverride`` keeps the bare name);
2873 the value overrides annotate it with this role's ARN so the tracking
2874 server exchanges its webhook-injected projected token for
2875 credentials, which is what feeds the server-side S3 artifact proxy
2876 (``mlflow.artifactsDestination``). The chart's default
2877 ``automountServiceAccountToken: false`` does not affect IRSA — the
2878 EKS pod identity webhook mounts its own token volume. Controller-
2879 style posture: OIDC-only (``include_pod_identity=False``), trust
2880 bound to exactly one namespace/service-account pair.
2882 Grants are deliberately narrower than the job-pod role's bucket-wide
2883 grant: object access only under the ``mlflow-artifacts/`` prefix of
2884 the cluster-shared bucket, ``ListBucket`` condition-scoped to the
2885 same prefix, and KMS confined by ``kms:ViaService`` exactly like
2886 ``_grant_cluster_shared_bucket_to_job_role``.
2887 """
2888 from gco.stacks.nag_suppressions import acknowledge_nag_findings
2890 self.mlflow_role = GCORegionalStack._create_irsa_role(
2891 self,
2892 "MlflowArtifactRole",
2893 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
2894 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
2895 service_account_names=["mlflow"],
2896 namespaces=["monitoring"],
2897 include_pod_identity=False,
2898 )
2900 self.mlflow_role.add_to_policy(
2901 iam.PolicyStatement(
2902 effect=iam.Effect.ALLOW,
2903 actions=["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
2904 resources=[f"{shared.arn}/mlflow-artifacts/*"],
2905 )
2906 )
2907 self.mlflow_role.add_to_policy(
2908 iam.PolicyStatement(
2909 effect=iam.Effect.ALLOW,
2910 actions=["s3:ListBucket"],
2911 resources=[shared.arn],
2912 conditions={
2913 "StringLike": {
2914 "s3:prefix": "mlflow-artifacts/*",
2915 }
2916 },
2917 )
2918 )
2919 # GetBucketLocation cannot share the ListBucket statement: requests
2920 # for it never carry the s3:prefix key, so the condition above would
2921 # implicitly deny it.
2922 self.mlflow_role.add_to_policy(
2923 iam.PolicyStatement(
2924 effect=iam.Effect.ALLOW,
2925 actions=["s3:GetBucketLocation"],
2926 resources=[shared.arn],
2927 )
2928 )
2929 self.mlflow_role.add_to_policy(
2930 iam.PolicyStatement(
2931 effect=iam.Effect.ALLOW,
2932 actions=["kms:Decrypt", "kms:GenerateDataKey"],
2933 resources=["*"],
2934 conditions={
2935 "StringEquals": {
2936 "kms:ViaService": f"s3.{shared.region}.{self.url_suffix}",
2937 }
2938 },
2939 )
2940 )
2942 acknowledge_nag_findings(
2943 self.mlflow_role,
2944 [
2945 {
2946 "id": "AwsSolutions-IAM5",
2947 "reason": (
2948 "The MLflow artifact grants require two wildcard shapes: a "
2949 "mlflow-artifacts/* object-key suffix within the single shared "
2950 "bucket resolved from SSM, and KMS Resource::* because the "
2951 "global key ARN is not exported to this stack. KMS use is "
2952 "constrained by kms:ViaService to S3 in the bucket's region, "
2953 "and S3 access is separately limited to the artifact prefix."
2954 ),
2955 "appliesTo": [
2956 "Resource::*",
2957 "Resource::<ReadClusterSharedBucketArn4B0BD291.Parameter.Value>/mlflow-artifacts/*",
2958 ],
2959 },
2960 ],
2961 )
2963 def _create_regional_shared_bucket(self) -> None:
2964 """Create the always-on general-purpose regional bucket for this region.
2966 Provisioned unconditionally — there is no ``cdk.json`` toggle and no
2967 feature flag that can suppress it — in addition to the central buckets
2968 owned by ``GCOGlobalStack`` (the model bucket and the cluster-shared
2969 bucket). The bucket is general purpose: any in-region workload may use
2970 it, and the per-region cold KV tier auto-targets it when cold-tier
2971 storage is requested. Its existence is independent of any endpoint's
2972 cold-tier choice.
2974 Three constructs are created, mirroring the cluster-shared bucket
2975 pattern in ``GCOGlobalStack``:
2977 1. ``regional_shared_kms_key`` — a customer-managed KMS key with annual
2978 rotation and a 7-day pending window on destroy. The key policy grants
2979 the ``s3.amazonaws.com`` and ``logs.<region>.amazonaws.com`` service
2980 principals encrypt/decrypt so S3 server-side encryption and access-log
2981 delivery work without role-side grants.
2982 2. ``regional_shared_access_logs_bucket`` — the dedicated S3 access-logs
2983 destination for the primary bucket.
2984 3. ``regional_shared_bucket`` — the primary bucket. Its physical name
2985 is CloudFormation-generated (``<stack>-regionalsharedbucket…``):
2986 S3 bucket names are a global namespace and a deleted name is not
2987 reliably reusable, so a fixed project/account/region name would
2988 make every destroy-and-redeploy (and the ``retain`` policy below)
2989 a collision hazard. Consumers never reconstruct it — they read the
2990 SSM parameters published under
2991 ``regional_shared_ssm_parameter_prefix(project_name)`` or the
2992 ``gco-regional-shared-bucket`` ConfigMap. KMS-encrypted with
2993 ``regional_shared_kms_key``, block-public-access on, SSL enforced,
2994 versioned, destroy-on-teardown.
2996 An explicit ``Deny`` for ``aws:SecureTransport=false`` is added to the
2997 bucket policy independent of ``enforce_ssl=True`` so the deny is
2998 verifiable in the synthesized template under a known SID.
2999 """
3000 # Teardown behavior is configurable (cdk.json::regional_shared_bucket.
3001 # removal_policy) because this bucket holds artifacts jobs just
3002 # produced: 'destroy' (the default, preserving historical teardown
3003 # semantics for existing deployments and validation cycles) deletes
3004 # bucket, logs, and key with the region; 'retain' lets all three
3005 # outlive a regional destroy so checkpoints survive. Invalid values
3006 # fail synthesis rather than silently choosing a side. Keep this in
3007 # sync with the tolerant CLI-side read in cli/storage.py
3008 # (_regional_shared_removal_policy) that `gco storage s3-inventory`
3009 # reports through.
3010 retention_context = self.node.try_get_context("regional_shared_bucket") or {}
3011 configured_policy = str(retention_context.get("removal_policy", "destroy")).strip().lower()
3012 if configured_policy not in ("destroy", "retain"):
3013 raise ValueError(
3014 "regional_shared_bucket.removal_policy must be 'destroy' or 'retain', "
3015 f"got {retention_context.get('removal_policy')!r}"
3016 )
3017 retain_regional_shared = configured_policy == "retain"
3018 regional_shared_removal_policy = (
3019 RemovalPolicy.RETAIN if retain_regional_shared else RemovalPolicy.DESTROY
3020 )
3022 # KMS key for the regional bucket. Matches the cluster-shared key
3023 # posture: annual rotation, 7-day pending window, destroy-on-teardown
3024 # by default. Under 'retain' the key survives with the bucket —
3025 # a retained bucket whose key was scheduled for deletion would be
3026 # undecryptable, so the two always share a fate.
3027 self.regional_shared_kms_key = kms.Key(
3028 self,
3029 "RegionalSharedKmsKey",
3030 description=(
3031 "Customer-managed KMS key for the always-on general-purpose "
3032 "regional bucket in this region's GCORegionalStack."
3033 ),
3034 enable_key_rotation=True,
3035 pending_window=Duration.days(7),
3036 removal_policy=regional_shared_removal_policy,
3037 )
3039 # Key-policy grants for the service principals that encrypt/decrypt on
3040 # behalf of the bucket (S3 server-side encryption) and the access-logs
3041 # bucket (CloudWatch/S3 log delivery).
3042 kms_actions = [
3043 "kms:Encrypt",
3044 "kms:Decrypt",
3045 "kms:ReEncrypt*",
3046 "kms:GenerateDataKey*",
3047 "kms:DescribeKey",
3048 ]
3050 self.regional_shared_kms_key.add_to_resource_policy(
3051 iam.PolicyStatement(
3052 sid="AllowS3ServiceEncryptDecrypt",
3053 effect=iam.Effect.ALLOW,
3054 principals=[iam.ServicePrincipal("s3.amazonaws.com")],
3055 actions=kms_actions,
3056 resources=["*"],
3057 )
3058 )
3060 self.regional_shared_kms_key.add_to_resource_policy(
3061 iam.PolicyStatement(
3062 sid="AllowCloudWatchLogsEncryptDecrypt",
3063 effect=iam.Effect.ALLOW,
3064 principals=[iam.ServicePrincipal(f"logs.{self.deployment_region}.amazonaws.com")],
3065 actions=kms_actions,
3066 resources=["*"],
3067 )
3068 )
3070 # Retention for the access-logs bucket honors the same `s3_access_logs`
3071 # context field used by the central buckets (default 90 days).
3072 s3_access_logs_ctx = self.node.try_get_context("s3_access_logs") or {}
3073 access_logs_retention_days = int(s3_access_logs_ctx.get("retention_days", 90))
3075 # Dedicated access-logs bucket for the regional bucket, encrypted with
3076 # the regional KMS key (its key policy grants the logs service principal
3077 # encrypt/decrypt).
3078 self.regional_shared_access_logs_bucket = s3.Bucket(
3079 self,
3080 "RegionalSharedAccessLogsBucket",
3081 encryption=s3.BucketEncryption.KMS,
3082 encryption_key=self.regional_shared_kms_key,
3083 block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
3084 enforce_ssl=True,
3085 versioned=True,
3086 removal_policy=regional_shared_removal_policy,
3087 auto_delete_objects=not retain_regional_shared,
3088 lifecycle_rules=[
3089 s3.LifecycleRule(
3090 id="ExpireAccessLogs",
3091 enabled=True,
3092 expiration=Duration.days(access_logs_retention_days),
3093 )
3094 ],
3095 )
3097 # Primary general-purpose regional bucket. No ``bucket_name``: the
3098 # physical name is CloudFormation-generated so a destroy-and-redeploy
3099 # (or a retained bucket from an earlier deployment) can never collide
3100 # in S3's global namespace; the IAM grants below reference the
3101 # construct's ARN token and consumers resolve the name from SSM.
3102 # `bucket_key_enabled=True` mirrors the central-bucket pattern to
3103 # reduce per-object KMS request costs.
3104 project_name = self.config.get_project_name()
3105 regional_shared_prefix = regional_shared_ssm_parameter_prefix(project_name)
3106 self.regional_shared_bucket = s3.Bucket(
3107 self,
3108 "RegionalSharedBucket",
3109 encryption=s3.BucketEncryption.KMS,
3110 encryption_key=self.regional_shared_kms_key,
3111 bucket_key_enabled=True,
3112 block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
3113 enforce_ssl=True,
3114 versioned=True,
3115 removal_policy=regional_shared_removal_policy,
3116 auto_delete_objects=not retain_regional_shared,
3117 server_access_logs_bucket=self.regional_shared_access_logs_bucket,
3118 server_access_logs_prefix="regional-shared/",
3119 )
3121 # Explicit Deny for insecure transport. `enforce_ssl=True` already adds
3122 # an equivalent statement, but duplicating it here makes the deny
3123 # verifiable in the synthesized template under a known SID.
3124 self.regional_shared_bucket.add_to_resource_policy(
3125 iam.PolicyStatement(
3126 sid="DenyInsecureTransport",
3127 effect=iam.Effect.DENY,
3128 principals=[iam.AnyPrincipal()],
3129 actions=["s3:*"],
3130 resources=[
3131 self.regional_shared_bucket.bucket_arn,
3132 f"{self.regional_shared_bucket.bucket_arn}/*",
3133 ],
3134 conditions={"Bool": {"aws:SecureTransport": "false"}},
3135 )
3136 )
3138 # Publish the bucket's identity as three SSM parameters in this
3139 # region's own parameter store, mirroring how the model bucket and
3140 # cluster-shared bucket publish theirs. In-region workloads and the
3141 # regional upload surface resolve the always-on regional bucket by
3142 # reading these back rather than reconstructing the name. Because the
3143 # bucket is unconditional, these parameters are always present once the
3144 # region's stack is deployed. The prefix from
3145 # ``regional_shared_ssm_parameter_prefix(project_name)`` is the single
3146 # source of truth for the namespace.
3147 ssm.StringParameter(
3148 self,
3149 "RegionalSharedBucketNameParam",
3150 parameter_name=f"{regional_shared_prefix}/name",
3151 string_value=self.regional_shared_bucket.bucket_name,
3152 description="Name of the always-on general-purpose regional bucket for this region.",
3153 )
3155 ssm.StringParameter(
3156 self,
3157 "RegionalSharedBucketArnParam",
3158 parameter_name=f"{regional_shared_prefix}/arn",
3159 string_value=self.regional_shared_bucket.bucket_arn,
3160 description="ARN of the always-on general-purpose regional bucket for this region.",
3161 )
3163 ssm.StringParameter(
3164 self,
3165 "RegionalSharedBucketRegionParam",
3166 parameter_name=f"{regional_shared_prefix}/region",
3167 string_value=self.deployment_region,
3168 description="Home region of the always-on general-purpose regional bucket.",
3169 )
3171 # CDK-nag suppressions scoped per-resource at the construct site,
3172 # mirroring the central bucket pattern. Every suppression carries an
3173 # explicit reason; no blanket bypasses.
3174 from gco.stacks.nag_suppressions import acknowledge_nag_findings
3176 regional_replication_reason = (
3177 "The general-purpose regional bucket is a region-local store; "
3178 "in-region workloads publish to their own region's bucket and there "
3179 "is no durability requirement that warrants cross-region "
3180 "replication. Access logs do not require replication for the same "
3181 "reason."
3182 )
3184 acknowledge_nag_findings(
3185 self.regional_shared_bucket,
3186 [
3187 {
3188 "id": "HIPAA.Security-S3BucketReplicationEnabled",
3189 "reason": regional_replication_reason,
3190 },
3191 {
3192 "id": "NIST.800.53.R5-S3BucketReplicationEnabled",
3193 "reason": regional_replication_reason,
3194 },
3195 {
3196 "id": "PCI.DSS.321-S3BucketReplicationEnabled",
3197 "reason": regional_replication_reason,
3198 },
3199 ],
3200 )
3202 access_logs_is_self_target_reason = (
3203 "This is the server access logs destination bucket for the "
3204 "general-purpose regional bucket."
3205 )
3206 acknowledge_nag_findings(
3207 self.regional_shared_access_logs_bucket,
3208 [
3209 {
3210 "id": "AwsSolutions-S1",
3211 "reason": access_logs_is_self_target_reason,
3212 },
3213 {
3214 "id": "HIPAA.Security-S3BucketLoggingEnabled",
3215 "reason": access_logs_is_self_target_reason,
3216 },
3217 {
3218 "id": "NIST.800.53.R5-S3BucketLoggingEnabled",
3219 "reason": access_logs_is_self_target_reason,
3220 },
3221 {
3222 "id": "PCI.DSS.321-S3BucketLoggingEnabled",
3223 "reason": access_logs_is_self_target_reason,
3224 },
3225 {
3226 "id": "HIPAA.Security-S3BucketReplicationEnabled",
3227 "reason": regional_replication_reason,
3228 },
3229 {
3230 "id": "NIST.800.53.R5-S3BucketReplicationEnabled",
3231 "reason": regional_replication_reason,
3232 },
3233 {
3234 "id": "PCI.DSS.321-S3BucketReplicationEnabled",
3235 "reason": regional_replication_reason,
3236 },
3237 ],
3238 )
3240 # Grant the in-region pod role read/write on this bucket and use of its
3241 # KMS key — and nothing else. The grant lives next to the bucket it
3242 # scopes to, so the role's regional-bucket access stays exactly as wide
3243 # as this one bucket and its key.
3244 self._grant_regional_shared_bucket_to_service_account()
3246 def _grant_regional_shared_bucket_to_service_account(self) -> None:
3247 """Attach RW + KMS permissions on the regional bucket to the pod role.
3249 Two ``iam.PolicyStatement``s are added to ``self.service_account_role``
3250 (the EKS Pod Identity role used by every pod in ``gco-jobs``,
3251 ``gco-system``, and ``gco-inference``):
3253 1. S3 object + bucket-level actions (``GetObject``, ``PutObject``,
3254 ``DeleteObject``, ``ListBucket``, ``GetBucketLocation``) scoped to
3255 the literal ``regional_shared_bucket`` ARN and its ``<arn>/*``
3256 object-key space — and to no other bucket.
3257 2. KMS ``Decrypt`` / ``Encrypt`` / ``GenerateDataKey`` /
3258 ``DescribeKey`` scoped to the literal ``regional_shared_kms_key``
3259 ARN — and to no other key.
3261 Because both resources are local constructs in this stack, each ARN is
3262 a concrete reference rather than a wildcard, so the role gains access to
3263 precisely this bucket and this key. The grant runs unconditionally as
3264 part of provisioning the always-on regional bucket.
3265 """
3266 from gco.stacks.nag_suppressions import acknowledge_nag_findings
3268 self.service_account_role.add_to_policy(
3269 iam.PolicyStatement(
3270 effect=iam.Effect.ALLOW,
3271 actions=[
3272 "s3:GetObject",
3273 "s3:PutObject",
3274 "s3:DeleteObject",
3275 "s3:ListBucket",
3276 "s3:GetBucketLocation",
3277 ],
3278 resources=[
3279 self.regional_shared_bucket.bucket_arn,
3280 f"{self.regional_shared_bucket.bucket_arn}/*",
3281 ],
3282 )
3283 )
3285 self.service_account_role.add_to_policy(
3286 iam.PolicyStatement(
3287 effect=iam.Effect.ALLOW,
3288 actions=[
3289 "kms:Decrypt",
3290 "kms:Encrypt",
3291 "kms:GenerateDataKey",
3292 "kms:DescribeKey",
3293 ],
3294 resources=[self.regional_shared_kms_key.key_arn],
3295 )
3296 )
3298 # The S3 bucket-ARN resource uses a ``<arn>/*`` object-key wildcard
3299 # which cdk-nag flags as a wildcard resource. The ARN itself is the
3300 # literal regional bucket ARN created in this stack — the ``/*`` covers
3301 # all object keys inside that single bucket, which is the intended
3302 # semantic for the RW grant. The KMS statement carries no wildcard.
3303 acknowledge_nag_findings(
3304 self.service_account_role,
3305 [
3306 {
3307 "id": "AwsSolutions-IAM5",
3308 "reason": (
3309 "The regional bucket RW grant uses an <arn>/* "
3310 "object-key wildcard on the literal ARN of the "
3311 "regional-shared bucket created in this stack "
3312 "(CloudFormation-generated name). The wildcard covers object "
3313 "keys within a single bucket — this is the standard "
3314 "shape for a bucket-scoped RW grant and is what the "
3315 "allow-list assertion is written against."
3316 ),
3317 "appliesTo": [
3318 "Resource::<RegionalSharedBucket3FF19783.Arn>/*",
3319 ],
3320 },
3321 ],
3322 )
3324 def _cost_report_bucket_parameter_name(self) -> str:
3325 """SSM parameter (in the monitoring region) publishing the cost bucket name."""
3326 return f"{cost_report_ssm_parameter_prefix(self.config.get_project_name())}/name"
3328 def _grant_cost_report_bucket_discovery_to_cost_monitor(self) -> None:
3329 """Let the cost-monitor role resolve the cost report bucket it writes to.
3331 The bucket lives in ``GCOMonitoringStack`` in the monitoring region,
3332 which deploys *after* every regional stack, and it carries a
3333 CloudFormation-generated physical name — S3 bucket names are a global
3334 namespace and a deleted name is not reliably reusable, so nothing
3335 reconstructs it from project/account/region any more. That inverts
3336 the old grant direction:
3338 1. The monitoring stack publishes the bucket's identity at
3339 ``<cost_report_ssm_parameter_prefix>/{name,arn,region}`` and grants
3340 every regional cost-monitor role S3 object/bucket actions through
3341 the bucket policy and KMS use through the key policy — principal
3342 based, so this stack never needs the bucket or key ARN.
3343 2. This stack grants the role exactly one permission:
3344 ``ssm:GetParameter`` on the ``/name`` parameter, by literal ARN. The
3345 service reads it at runtime (``COST_REPORT_BUCKET_PARAMETER`` /
3346 ``COST_REPORT_BUCKET_PARAMETER_REGION`` in ``34-cost-monitor.yaml``).
3348 On a fresh ``deploy-all`` the parameter appears only once monitoring is
3349 deployed; the service re-resolves on every scheduled pass, so the
3350 pipeline self-heals without ordering hacks. No wildcard remains on the
3351 role, so no cdk-nag acknowledgement is needed here.
3352 """
3353 monitoring_region = self.config.get_monitoring_region()
3354 parameter_name = self._cost_report_bucket_parameter_name()
3355 self.cost_monitor_role.add_to_policy(
3356 iam.PolicyStatement(
3357 effect=iam.Effect.ALLOW,
3358 actions=["ssm:GetParameter"],
3359 resources=[
3360 f"arn:{self.partition}:ssm:{monitoring_region}:{self.account}:"
3361 f"parameter{parameter_name}"
3362 ],
3363 )
3364 )
3366 def _create_kubectl_lambda(self) -> None:
3367 """Create Lambda function to apply Kubernetes manifests using Python client.
3369 Note: This creates the Lambda and provider but does NOT create the custom resource.
3370 The custom resource is created in _apply_kubernetes_manifests() after ALB is created,
3371 so that target group ARNs can be passed to the manifests.
3372 """
3373 project_name = self.config.get_project_name()
3375 # Create IAM role for kubectl Lambda
3376 kubectl_lambda_role = iam.Role(
3377 self,
3378 "KubectlLambdaRole",
3379 assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
3380 managed_policies=[
3381 iam.ManagedPolicy.from_aws_managed_policy_name(
3382 "service-role/AWSLambdaVPCAccessExecutionRole"
3383 ),
3384 iam.ManagedPolicy.from_aws_managed_policy_name(
3385 "service-role/AWSLambdaBasicExecutionRole"
3386 ),
3387 ],
3388 )
3390 # Add EKS permissions
3391 kubectl_lambda_role.add_to_policy(
3392 iam.PolicyStatement(
3393 actions=[
3394 "eks:DescribeCluster",
3395 "eks:ListClusters",
3396 ],
3397 resources=[self.cluster.cluster_arn],
3398 )
3399 )
3401 # Add permissions to assume cluster admin role
3402 kubectl_lambda_role.add_to_policy(
3403 iam.PolicyStatement(actions=["sts:AssumeRole"], resources=["*"])
3404 )
3406 # Allow the convergence apply tasks to record per-phase status to SSM
3407 # (base-manifests / post-helm-manifests), mirroring the helm worker, so
3408 # `gco stacks addons status` surfaces the apply passes alongside charts.
3409 kubectl_lambda_role.add_to_policy(
3410 iam.PolicyStatement(
3411 actions=["ssm:PutParameter"],
3412 resources=[
3413 f"arn:{self.partition}:ssm:{self.deployment_region}:{self.account}:"
3414 f"parameter/{project_name}/addons/*"
3415 ],
3416 )
3417 )
3419 # Create security group for kubectl Lambda
3420 kubectl_lambda_sg = ec2.SecurityGroup(
3421 self,
3422 "KubectlLambdaSG",
3423 vpc=self.vpc,
3424 description="Security group for kubectl Lambda to access EKS cluster",
3425 security_group_name=f"{self.config.get_project_name()}-kubectl-lambda-sg-{self.deployment_region}",
3426 allow_all_outbound=True, # Lambda needs outbound access to EKS API
3427 )
3429 # Allow Lambda security group to access EKS cluster security group on port 443
3430 # The EKS cluster security group is automatically created by EKS
3431 self.cluster.cluster_security_group.add_ingress_rule(
3432 peer=kubectl_lambda_sg,
3433 connection=ec2.Port.tcp(443),
3434 description="Allow kubectl Lambda to access EKS API",
3435 )
3437 # Create Lambda function (Python-only, no Docker!)
3438 # Store function name as string attribute for cross-stack references
3439 # This avoids CDK cross-environment resolution issues when account is unresolved
3440 self.kubectl_lambda_function_name = f"{project_name}-kubectl-{self.deployment_region}"
3441 self.kubectl_lambda = lambda_.Function(
3442 self,
3443 "KubectlApplierFunction",
3444 function_name=self.kubectl_lambda_function_name,
3445 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
3446 handler="handler.lambda_handler",
3447 code=lambda_.Code.from_asset("lambda/kubectl-applier-simple-build"),
3448 timeout=Duration.minutes(15), # Max Lambda timeout
3449 memory_size=512,
3450 role=kubectl_lambda_role,
3451 vpc=self.vpc,
3452 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
3453 security_groups=[kubectl_lambda_sg], # Use the security group we created
3454 environment={
3455 "CLUSTER_NAME": self.cluster.cluster_name,
3456 "REGION": self.deployment_region,
3457 # Lets the convergence apply tasks record per-phase status to
3458 # SSM (/<project>/addons/<region>/{base,post-helm}-manifests).
3459 "PROJECT_NAME": project_name,
3460 },
3461 tracing=lambda_.Tracing.ACTIVE,
3462 )
3464 # Add EKS access entry for the Lambda role to authenticate with the cluster
3465 # This grants the Lambda role cluster admin permissions
3466 self.kubectl_lambda_access_entry = eks.AccessEntry(
3467 self,
3468 "KubectlLambdaAccessEntry",
3469 cluster=self.cluster, # type: ignore[arg-type]
3470 principal=kubectl_lambda_role.role_arn,
3471 access_policies=[
3472 eks.AccessPolicy.from_access_policy_name(
3473 "AmazonEKSClusterAdminPolicy", access_scope_type=eks.AccessScopeType.CLUSTER
3474 )
3475 ],
3476 )
3478 # No custom-resource provider needed: the kubectl-applier Lambda is now
3479 # invoked directly by the convergence state machine (the base and
3480 # post-Helm apply tasks), not through a CloudFormation custom resource.
3482 # cdk-nag suppression: the kubectl-applier Lambda requires broad
3483 # EKS and Kubernetes API access to apply arbitrary manifests.
3484 from gco.stacks.nag_suppressions import acknowledge_nag_findings
3486 acknowledge_nag_findings(
3487 kubectl_lambda_role,
3488 [
3489 {
3490 "id": "AwsSolutions-IAM5",
3491 "reason": (
3492 "The kubectl-applier Lambda requires broad EKS and Kubernetes API "
3493 "access to apply arbitrary manifests (RBAC, ServiceAccounts, "
3494 "Deployments, Jobs, NetworkPolicies) across multiple namespaces. "
3495 "Resource: * is required because the set of Kubernetes resources "
3496 "is dynamic and not known at synth time."
3497 ),
3498 "appliesTo": ["Resource::*"],
3499 },
3500 ],
3501 )
3503 def _apply_kubernetes_manifests(self) -> None:
3504 """Build the complete base/Helm/post-Helm convergence pipeline.
3506 This is called after the Gateway certificate and shared storage exist.
3507 The post-Helm pass creates the internal ALB through Gateway API only
3508 after the mandatory AWS Load Balancer Controller is installed.
3509 """
3511 # Build image replacements dict
3512 # Include one deployment token to force pod rollouts and bind live
3513 # validation to this exact asynchronous convergence execution.
3514 deployment_timestamp = _deployment_timestamp()
3515 self.addon_deployment_token = deployment_timestamp
3517 # Get resource thresholds from config
3518 thresholds = self.config.get_resource_thresholds()
3520 # Get manifest processor resource quotas.
3521 # Resource quotas and the security/image policy now live under the
3522 # shared job_validation_policy section because both the REST
3523 # manifest_processor and the SQS queue_processor read them. Service-
3524 # specific knobs (replicas, validation_enabled, max_request_body_bytes,
3525 # etc.) stay under manifest_processor. Inference TLS proxy CPU and HPA
3526 # settings live in their own optional block.
3527 mp_config = self.config.get_manifest_processor_config()
3528 inference_proxy_config = self.config.get_inference_proxy_config()
3529 job_policy = self.node.try_get_context("job_validation_policy") or {}
3530 job_quotas = _validated_manifest_caps(
3531 job_policy.get("resource_quotas", {}),
3532 _validated_resource_quota(self.node.try_get_context("resource_quota") or {}),
3533 )
3534 allowed_kinds = job_policy.get(
3535 "allowed_kinds",
3536 # Fallback mirrors manifest_processor.DEFAULT_ALLOWED_KINDS (kept
3537 # inline so CDK synth never imports service modules; lockstep is
3538 # pinned by tests/test_manifest_processor_extended.py::TestAllowedKindsLockstep).
3539 [
3540 "Job",
3541 "CronJob",
3542 "Deployment",
3543 "StatefulSet",
3544 "DaemonSet",
3545 "Service",
3546 "ConfigMap",
3547 "Pod",
3548 "TrainJob",
3549 ],
3550 )
3552 security_policy = validate_manifest_security_policy(
3553 job_policy.get("manifest_security_policy", {})
3554 )
3556 def _policy_str(value: object) -> str:
3557 if type(value) is not bool:
3558 raise ValueError("job validation policy values must be booleans")
3559 return "true" if value else "false"
3561 require_accelerator_toleration = _policy_str(
3562 job_policy.get("require_accelerator_toleration", True)
3563 )
3564 validation_enabled = _policy_str(mp_config.get("validation_enabled", True))
3566 image_replacements = {
3567 "{{BACKEND_TLS_CERTIFICATE_ARN}}": self.backend_tls_certificate_arn,
3568 "{{HEALTH_MONITOR_IMAGE}}": self.health_monitor_image.image_uri,
3569 "{{MANIFEST_PROCESSOR_IMAGE}}": self.manifest_processor_image.image_uri,
3570 "{{INFERENCE_PROXY_IMAGE}}": self.inference_proxy_image.image_uri,
3571 "{{INFERENCE_MONITOR_IMAGE}}": self.inference_monitor_image.image_uri,
3572 # External, pinned upstream image for the shared Mooncake master
3573 # (bundles the mooncake_master binary). Same default as disaggregated
3574 # role pods; per-endpoint spec.mooncake.store.master_image overrides.
3575 "{{MOONCAKE_MASTER_IMAGE}}": MOONCAKE_MASTER_DEFAULT_IMAGE,
3576 "{{CLUSTER_NAME}}": self.cluster.cluster_name,
3577 "{{REGION}}": self.deployment_region,
3578 "{{AUTH_SECRET_ARN}}": self.auth_secret_arn,
3579 "{{SERVICE_ACCOUNT_ROLE_ARN}}": self.service_account_role.role_arn,
3580 "{{MANIFEST_PROCESSOR_ROLE_ARN}}": self.manifest_processor_role.role_arn,
3581 "{{INFERENCE_PROXY_ROLE_ARN}}": self.inference_proxy_role.role_arn,
3582 "{{HEALTH_MONITOR_ROLE_ARN}}": self.health_monitor_role.role_arn,
3583 "{{EFS_FILE_SYSTEM_ID}}": self.efs_file_system.file_system_id,
3584 "{{EFS_ACCESS_POINT_ID}}": self.efs_access_point.access_point_id,
3585 "{{JOB_QUEUE_URL}}": self.job_queue.queue_url,
3586 "{{JOB_QUEUE_ARN}}": self.job_queue.queue_arn,
3587 "{{DEPLOYMENT_TIMESTAMP}}": deployment_timestamp,
3588 # Resource thresholds
3589 "{{CPU_THRESHOLD}}": str(thresholds.cpu_threshold),
3590 "{{MEMORY_THRESHOLD}}": str(thresholds.memory_threshold),
3591 "{{GPU_THRESHOLD}}": str(thresholds.gpu_threshold),
3592 "{{PENDING_PODS_THRESHOLD}}": str(thresholds.pending_pods_threshold),
3593 "{{PENDING_REQUESTED_CPU_VCPUS}}": str(thresholds.pending_requested_cpu_vcpus),
3594 "{{PENDING_REQUESTED_MEMORY_GB}}": str(thresholds.pending_requested_memory_gb),
3595 "{{PENDING_REQUESTED_GPUS}}": str(thresholds.pending_requested_gpus),
3596 # Deployment prefix (#139). Injected so in-cluster services (the
3597 # inference monitor) resolve project-scoped SSM paths
3598 # (/<project>/regional-shared-bucket/*) instead of a hardcoded
3599 # /gco/ namespace, letting two deployments share an account+region.
3600 "{{PROJECT_NAME}}": self.config.get_project_name(),
3601 # DynamoDB table names (from global stack)
3602 "{{TEMPLATES_TABLE_NAME}}": f"{self.config.get_project_name()}-job-templates",
3603 "{{WEBHOOKS_TABLE_NAME}}": f"{self.config.get_project_name()}-webhooks",
3604 "{{JOBS_TABLE_NAME}}": f"{self.config.get_project_name()}-jobs",
3605 "{{INFERENCE_ENDPOINTS_TABLE_NAME}}": (
3606 f"{self.config.get_project_name()}-inference-endpoints"
3607 ),
3608 # DynamoDB region (global stack region, may differ from cluster region)
3609 "{{DYNAMODB_REGION}}": self.config.get_global_region(),
3610 # Global region for cross-region SSM reads/writes (e.g. the health
3611 # monitor's /<project>/alb-hostname-<region> sync in the global region).
3612 "{{GLOBAL_REGION}}": self.config.get_global_region(),
3613 # Manifest processor resource quotas (sourced from shared policy).
3614 "{{MP_MAX_CPU_PER_MANIFEST}}": job_quotas["max_cpu_per_manifest"],
3615 "{{MP_MAX_MEMORY_PER_MANIFEST}}": job_quotas["max_memory_per_manifest"],
3616 "{{MP_MAX_GPU_PER_MANIFEST}}": job_quotas["max_gpu_per_manifest"],
3617 # Require accelerator (GPU/Neuron/EFA) jobs to carry a matching
3618 # toleration (shared policy). Mirrored on the SQS path via
3619 # {{QP_REQUIRE_ACCELERATOR_TOLERATION}} so neither path is a bypass.
3620 "{{MP_REQUIRE_ACCELERATOR_TOLERATION}}": require_accelerator_toleration,
3621 # Manifest processor namespace allowlist (sourced from shared policy).
3622 # Both the REST manifest processor and the SQS queue processor
3623 # read from job_validation_policy.allowed_namespaces so a single
3624 # edit takes effect on both submission paths at the next deploy.
3625 "{{MP_ALLOWED_NAMESPACES}}": ",".join(
3626 job_policy.get("allowed_namespaces", ["gco-jobs"])
3627 ),
3628 # Manifest processor Kubernetes resource kind allowlist (shared policy).
3629 "{{MP_ALLOWED_KINDS}}": ",".join(allowed_kinds),
3630 # Manifest processor image registry allowlist (sourced from shared
3631 # policy). Augmented with the project's own ECR registry hostnames
3632 # so jobs built via ``gco images build`` aren't rejected by the
3633 # REST submission path. Identical augmentation runs on the SQS
3634 # path below — see ``{{QP_TRUSTED_REGISTRIES}}``.
3635 "{{MP_TRUSTED_REGISTRIES}}": ",".join(
3636 _augment_trusted_registries_with_project_ecr(
3637 job_policy.get("trusted_registries", []),
3638 account=self.account,
3639 regions=self.config.get_regions(),
3640 global_region=self.config.get_global_region(),
3641 url_suffix=self.url_suffix,
3642 )
3643 ),
3644 "{{MP_TRUSTED_DOCKERHUB_ORGS}}": ",".join(job_policy.get("trusted_dockerhub_orgs", [])),
3645 # Manifest parsing and pod-security policy use the same values as
3646 # the queue processor, preventing REST/SQS validation drift.
3647 "{{MP_VALIDATION_ENABLED}}": validation_enabled,
3648 "{{MP_YAML_MAX_DEPTH}}": str(mp_config.get("yaml_max_depth", 50)),
3649 "{{MP_BLOCK_PRIVILEGED}}": _policy_str(security_policy["block_privileged"]),
3650 "{{MP_BLOCK_PRIVILEGE_ESCALATION}}": _policy_str(
3651 security_policy["block_privilege_escalation"]
3652 ),
3653 "{{MP_BLOCK_HOST_NETWORK}}": _policy_str(security_policy["block_host_network"]),
3654 "{{MP_BLOCK_HOST_PID}}": _policy_str(security_policy["block_host_pid"]),
3655 "{{MP_BLOCK_HOST_IPC}}": _policy_str(security_policy["block_host_ipc"]),
3656 "{{MP_BLOCK_HOST_PATH}}": _policy_str(security_policy["block_host_path"]),
3657 "{{MP_BLOCK_ADDED_CAPABILITIES}}": _policy_str(
3658 security_policy["block_added_capabilities"]
3659 ),
3660 "{{MP_BLOCK_RUN_AS_ROOT}}": _policy_str(security_policy["block_run_as_root"]),
3661 # Manifest processor request body size cap (HTTP 413 middleware).
3662 # Lives at cdk.json::manifest_processor.max_request_body_bytes.
3663 "{{MP_MAX_REQUEST_BODY_BYTES}}": str(
3664 mp_config.get("max_request_body_bytes", 1_048_576)
3665 ),
3666 # Inference request bodies use the same operator-configured cap,
3667 # but retain a service-specific placeholder for future tuning.
3668 "{{INFERENCE_PROXY_MAX_REQUEST_BODY_BYTES}}": str(
3669 mp_config.get("max_request_body_bytes", 1_048_576)
3670 ),
3671 # Shared pure renderer keeps production and Kind typed values in
3672 # lockstep (Quantity string for request, YAML integers for the HPA
3673 # target and the min/max replica bounds).
3674 **_compute_inference_proxy_tls_replacements(inference_proxy_config),
3675 # Manifest-processor sizing is an operator decision made in
3676 # cdk.json (validated in ConfigLoader): the fixed replica count and
3677 # the application container's limits render verbatim. The optional
3678 # CPU HPA (35-manifest-processor-hpa.yaml) is gated by
3679 # {{MP_HPA_ENABLED}} below; when it is on, the Deployment carries
3680 # gco.aws/hpa-controls-replicas="true" so the applier stops
3681 # re-asserting replicas and the HPA's scale value survives.
3682 "{{MP_REPLICAS}}": str(mp_config["replicas"]),
3683 "{{MP_CPU_LIMIT}}": str(mp_config["resource_limits"]["cpu"]),
3684 "{{MP_MEMORY_LIMIT}}": str(mp_config["resource_limits"]["memory"]),
3685 "{{MP_HPA_CONTROLS_REPLICAS}}": (
3686 "true" if mp_config["autoscaling"]["enabled"] else "false"
3687 ),
3688 # Regional worker for the DynamoDB-backed global queue. Multiple API
3689 # replicas are safe because JobStore claims are conditional and
3690 # lease-backed; each replica also reconciles K8s status transitions.
3691 "{{CENTRAL_QUEUE_WORKER_ENABLED}}": (
3692 "true" if mp_config.get("central_queue_worker_enabled", True) else "false"
3693 ),
3694 "{{CENTRAL_QUEUE_POLL_INTERVAL_SECONDS}}": str(
3695 mp_config.get("central_queue_poll_interval_seconds", 10)
3696 ),
3697 "{{CENTRAL_QUEUE_BATCH_SIZE}}": str(mp_config.get("central_queue_batch_size", 5)),
3698 "{{CENTRAL_QUEUE_RECONCILE_LIMIT}}": str(
3699 mp_config.get("central_queue_reconcile_limit", 100)
3700 ),
3701 "{{CENTRAL_QUEUE_LEASE_SECONDS}}": str(
3702 mp_config.get("central_queue_lease_seconds", 300)
3703 ),
3704 "{{CENTRAL_QUEUE_LEASE_RENEWAL_SECONDS}}": str(
3705 mp_config.get("central_queue_lease_renewal_seconds", 60)
3706 ),
3707 "{{QUEUE_TARGET_REGIONS}}": ",".join(self.config.get_regions()),
3708 }
3710 # Always-on Cluster_Shared_Bucket replacements. Populated from the
3711 # SharedBucketIdentity resolved in __init__ via cross-region SSM
3712 # read from GCOGlobalStack. Never gated on the analytics toggle —
3713 # the gco-cluster-shared-bucket ConfigMap is applied to every
3714 # regional cluster.
3715 image_replacements.update(
3716 _compute_kubectl_cluster_shared_replacements(self.cluster_shared_identity)
3717 )
3719 # Always-on Regional_Shared_Bucket replacements. Read straight off the
3720 # local constructs created by _create_regional_shared_bucket (which
3721 # runs earlier in __init__, before _apply_kubernetes_manifests), so no
3722 # SSM round-trip is needed — unlike the cluster-shared bucket, this one
3723 # is owned by this stack. Never gated on a toggle: the bucket is
3724 # unconditional, so the gco-regional-shared-bucket ConfigMap is applied
3725 # to every regional cluster.
3726 image_replacements.update(
3727 _compute_kubectl_regional_shared_replacements(
3728 name=self.regional_shared_bucket.bucket_name,
3729 arn=self.regional_shared_bucket.bucket_arn,
3730 region=self.deployment_region,
3731 )
3732 )
3734 # Cluster observability (on by default): gate the gp3 StorageClass and
3735 # the ServiceMonitors/dashboards on the toggle. When enabled the gating
3736 # placeholders resolve so those manifests apply; when disabled the keys
3737 # are absent, so the manifests keep an unreplaced placeholder and the
3738 # applier skips them (same mechanism FSx/Valkey use).
3739 _obs_config = self.config.get_cluster_observability_config()
3740 image_replacements.update(
3741 _compute_kubectl_observability_replacements(
3742 bool(_obs_config["enabled"]),
3743 grafana_admin_password_rotation_schedule=str(
3744 _obs_config["grafana"]["admin_password_rotation_schedule"]
3745 ),
3746 )
3747 )
3748 # Scheduler gates: resolve through the same enablement helper that
3749 # selects the Helm charts, so the default Kueue queue topology and the
3750 # Slinky Slurm NetworkPolicies apply exactly when their scheduler does.
3751 _helm_config = self.node.try_get_context("helm") or {}
3752 _helm_overrides = _parse_helm_enabled_overrides(
3753 self.node.try_get_context(_HELM_OVERRIDE_CONTEXT_KEY)
3754 )
3755 image_replacements.update(
3756 _compute_kubectl_scheduler_replacements(
3757 kueue_enabled=_helm_chart_enabled(_helm_config, _helm_overrides, "kueue"),
3758 slurm_enabled=_helm_chart_enabled(_helm_config, _helm_overrides, "slurm"),
3759 kubeflow_trainer_enabled=_helm_chart_enabled(
3760 _helm_config, _helm_overrides, "kubeflow_trainer"
3761 ),
3762 )
3763 )
3765 # Cost monitoring (on by default): gate the cost-monitor Deployment
3766 # and the Grafana cost dashboard on the toggle via the same
3767 # unreplaced-placeholder mechanism. The image/role placeholders exist
3768 # only when the pipeline is active, so a disabled deployment leaves
3769 # 34-cost-monitor.yaml and the cost dashboard unapplied.
3770 if self._cost_monitoring_active():
3771 _cost_config = self.config.get_cost_monitoring_config()
3772 image_replacements.update(
3773 {
3774 "{{COST_MONITORING_ENABLED}}": "true",
3775 "{{COST_MONITOR_IMAGE}}": self.cost_monitor_image.image_uri,
3776 "{{COST_MONITOR_ROLE_ARN}}": self.cost_monitor_role.role_arn,
3777 # The bucket's CloudFormation-generated name is published by
3778 # the monitoring stack (deployed after this one); the service
3779 # resolves it from SSM at runtime instead of receiving a
3780 # reconstructed name here.
3781 "{{COST_REPORT_BUCKET_PARAMETER}}": self._cost_report_bucket_parameter_name(),
3782 "{{COST_REPORT_BUCKET_PARAMETER_REGION}}": self.config.get_monitoring_region(),
3783 "{{COST_REPORT_INTERVAL_MINUTES}}": str(
3784 _cost_config["reports"]["interval_minutes"]
3785 ),
3786 }
3787 )
3789 # Optional manifest-processor CPU autoscaler. When disabled the key is
3790 # absent, 35-manifest-processor-hpa.yaml keeps an unreplaced placeholder,
3791 # the applier skips it and prunes any HPA a previous deploy created.
3792 if mp_config["autoscaling"]["enabled"]:
3793 image_replacements.update(
3794 {
3795 "{{MP_HPA_ENABLED}}": "true",
3796 "{{MP_HPA_MAX_REPLICAS}}": str(mp_config["autoscaling"]["max_replicas"]),
3797 "{{MP_HPA_CPU_TARGET_UTILIZATION}}": str(
3798 mp_config["autoscaling"]["cpu_target_utilization_percentage"]
3799 ),
3800 }
3801 )
3803 # MLflow (on by default, requires observability): gate the client
3804 # egress NetworkPolicy (post-helm-mlflow-network.yaml) on the toggle
3805 # via the same unreplaced-placeholder mechanism; a disabled
3806 # deployment leaves the file unapplied and the applier prunes both
3807 # the policy and the chart-managed metadata claim helm uninstall
3808 # leaves behind (metadata is discarded, artifacts stay in S3).
3809 if self._mlflow_active():
3810 image_replacements.update({"{{MLFLOW_ENABLED}}": "true"})
3812 # Add queue processor replacements if enabled
3813 qp_config = self.node.try_get_context("queue_processor") or {}
3815 # In-VPC ranges job pods may reach on any port (03-network-policies.yaml
3816 # allow-vpc-egress) and the MLflow server admits probes from
3817 # (post-helm-mlflow-network.yaml). Generates a YAML block of ipBlock
3818 # entries from the vpc_endpoint_cidrs array. The placeholder
3819 # {{VPC_ENDPOINT_CIDR_BLOCKS}} sits at 8-space indentation in the
3820 # manifest, so the first entry needs no leading indent (the manifest
3821 # provides it) and subsequent entries are indented to align.
3822 vpc_endpoint_cidrs = self.node.try_get_context("vpc_endpoint_cidrs") or ["10.0.0.0/16"]
3823 cidr_lines = []
3824 for i, cidr in enumerate(vpc_endpoint_cidrs):
3825 prefix = "" if i == 0 else " "
3826 cidr_lines.append(f'{prefix}- ipBlock:\n cidr: "{cidr}"')
3827 image_replacements["{{VPC_ENDPOINT_CIDR_BLOCKS}}"] = "\n".join(cidr_lines)
3829 # NetworkPolicy enforcement on EKS Auto Mode is a ConfigMap-driven
3830 # switch (06-network-policy-controller.yaml); render the operator's
3831 # choice as the literal the controller reads.
3832 image_replacements["{{NETWORK_POLICY_ENFORCEMENT}}"] = (
3833 "true"
3834 if self.config.get_eks_cluster_config().get("network_policy_enforcement", True)
3835 else "false"
3836 )
3838 # Resource governance for gco-jobs namespace: ResourceQuota caps aggregate
3839 # resource consumption across the namespace, LimitRange caps per-container
3840 # maxima. Values come from cdk.json `resource_quota` context merged
3841 # over gco.stacks.constants.DEFAULT_RESOURCE_QUOTA (per-container
3842 # maxima sized to one full accelerator-node slice) and validated at
3843 # synth: every value must parse as a Kubernetes quantity and the
3844 # container maxima must fit inside the namespace ceilings.
3845 resource_quota = _validated_resource_quota(
3846 self.node.try_get_context("resource_quota") or {}
3847 )
3848 image_replacements["{{QUOTA_MAX_CPU}}"] = resource_quota["max_cpu"]
3849 image_replacements["{{QUOTA_MAX_MEMORY}}"] = resource_quota["max_memory"]
3850 image_replacements["{{QUOTA_MAX_GPU}}"] = resource_quota["max_gpu"]
3851 image_replacements["{{QUOTA_MAX_PODS}}"] = resource_quota["max_pods"]
3852 image_replacements["{{LIMIT_MAX_CPU}}"] = resource_quota["container_max_cpu"]
3853 image_replacements["{{LIMIT_MAX_MEMORY}}"] = resource_quota["container_max_memory"]
3854 image_replacements["{{LIMIT_MAX_GPU}}"] = resource_quota["container_max_gpu"]
3856 if self.queue_processor_enabled:
3857 image_replacements["{{QUEUE_PROCESSOR_IMAGE}}"] = self.queue_processor_image.image_uri
3858 image_replacements["{{QP_POLLING_INTERVAL}}"] = str(
3859 qp_config.get("polling_interval", 10)
3860 )
3861 image_replacements["{{QP_MAX_CONCURRENT_JOBS}}"] = str(
3862 qp_config.get("max_concurrent_jobs", 10)
3863 )
3864 image_replacements["{{QP_MESSAGES_PER_JOB}}"] = str(
3865 qp_config.get("messages_per_job", 1)
3866 )
3867 image_replacements["{{QP_SUCCESSFUL_JOBS_HISTORY}}"] = str(
3868 qp_config.get("successful_jobs_history", 20)
3869 )
3870 image_replacements["{{QP_FAILED_JOBS_HISTORY}}"] = str(
3871 qp_config.get("failed_jobs_history", 10)
3872 )
3873 image_replacements["{{QP_ALLOWED_NAMESPACES}}"] = ",".join(
3874 job_policy.get("allowed_namespaces", ["gco-jobs"])
3875 )
3876 image_replacements["{{QP_ALLOWED_KINDS}}"] = ",".join(allowed_kinds)
3877 # Resource caps, image allowlist, and security policy are shared
3878 # with the REST manifest processor. Source them from the
3879 # job_validation_policy section so a single change in cdk.json
3880 # takes effect on both submission paths at the next deploy.
3881 image_replacements["{{QP_MAX_GPU_PER_MANIFEST}}"] = job_quotas["max_gpu_per_manifest"]
3882 image_replacements["{{QP_MAX_CPU_PER_MANIFEST}}"] = job_quotas["max_cpu_per_manifest"]
3883 image_replacements["{{QP_MAX_MEMORY_PER_MANIFEST}}"] = job_quotas[
3884 "max_memory_per_manifest"
3885 ]
3886 image_replacements["{{QP_TRUSTED_REGISTRIES}}"] = ",".join(
3887 _augment_trusted_registries_with_project_ecr(
3888 job_policy.get("trusted_registries", []),
3889 account=self.account,
3890 regions=self.config.get_regions(),
3891 global_region=self.config.get_global_region(),
3892 url_suffix=self.url_suffix,
3893 )
3894 )
3895 image_replacements["{{QP_TRUSTED_DOCKERHUB_ORGS}}"] = ",".join(
3896 job_policy.get("trusted_dockerhub_orgs", [])
3897 )
3899 # Security policy toggles — shared with the REST manifest_processor.
3900 # Both services read the same cdk.json section so a single policy
3901 # flip (e.g. block_run_as_root: true) takes effect on both paths.
3902 image_replacements["{{QP_BLOCK_PRIVILEGED}}"] = _policy_str(
3903 security_policy["block_privileged"]
3904 )
3905 image_replacements["{{QP_BLOCK_PRIVILEGE_ESCALATION}}"] = _policy_str(
3906 security_policy["block_privilege_escalation"]
3907 )
3908 image_replacements["{{QP_BLOCK_HOST_NETWORK}}"] = _policy_str(
3909 security_policy["block_host_network"]
3910 )
3911 image_replacements["{{QP_BLOCK_HOST_PID}}"] = _policy_str(
3912 security_policy["block_host_pid"]
3913 )
3914 image_replacements["{{QP_BLOCK_HOST_IPC}}"] = _policy_str(
3915 security_policy["block_host_ipc"]
3916 )
3917 image_replacements["{{QP_BLOCK_HOST_PATH}}"] = _policy_str(
3918 security_policy["block_host_path"]
3919 )
3920 image_replacements["{{QP_BLOCK_ADDED_CAPABILITIES}}"] = _policy_str(
3921 security_policy["block_added_capabilities"]
3922 )
3923 image_replacements["{{QP_BLOCK_RUN_AS_ROOT}}"] = _policy_str(
3924 security_policy["block_run_as_root"]
3925 )
3926 # Require accelerator (GPU/Neuron/EFA) jobs to carry a matching
3927 # toleration — shared with the REST manifest_processor via
3928 # {{MP_REQUIRE_ACCELERATOR_TOLERATION}}.
3929 image_replacements["{{QP_REQUIRE_ACCELERATOR_TOLERATION}}"] = (
3930 require_accelerator_toleration
3931 )
3933 # Add Valkey endpoint if enabled
3934 if hasattr(self, "valkey_cache") and self.valkey_cache:
3935 image_replacements["{{VALKEY_ENDPOINT}}"] = self.valkey_cache.attr_endpoint_address
3936 image_replacements["{{VALKEY_PORT}}"] = self.valkey_cache.attr_endpoint_port
3938 # Add Aurora pgvector endpoint if enabled
3939 if hasattr(self, "aurora_cluster") and self.aurora_cluster:
3940 image_replacements["{{AURORA_PGVECTOR_ENDPOINT}}"] = (
3941 self.aurora_cluster.cluster_endpoint.hostname
3942 )
3943 image_replacements["{{AURORA_PGVECTOR_READER_ENDPOINT}}"] = (
3944 self.aurora_cluster.cluster_read_endpoint.hostname
3945 )
3946 image_replacements["{{AURORA_PGVECTOR_PORT}}"] = str(
3947 self.aurora_cluster.cluster_endpoint.port
3948 )
3949 image_replacements["{{AURORA_PGVECTOR_SECRET_ARN}}"] = self.aurora_secret.secret_arn
3951 # Add vector-store discovery if enabled. Every value is deterministic
3952 # at synth time (the global stack names the table and index from the
3953 # same config), so no cross-stack reference is needed; the manifest's
3954 # {{REGION}} key points pods at their cluster's LOCAL global-table
3955 # replica. When disabled, the placeholders stay unreplaced and the
3956 # applier skips 26-storage-vector-store.yaml entirely.
3957 if self.config.get_vector_store_enabled():
3958 vector_store_config = self.config.get_vector_store_config()
3959 image_replacements["{{VECTOR_STORE_TABLE_NAME}}"] = (
3960 f"{self.config.get_project_name()}-vector-store"
3961 )
3962 image_replacements["{{VECTOR_STORE_INDEX_NAME}}"] = "corpus-embedding-index"
3963 image_replacements["{{VECTOR_STORE_EMBEDDING_MODEL_ID}}"] = str(
3964 vector_store_config["embedding_model_id"]
3965 )
3966 image_replacements["{{VECTOR_STORE_DIMENSIONS}}"] = str(
3967 vector_store_config["dimensions"]
3968 )
3970 # Add FSx replacements if enabled
3971 if self.fsx_file_system:
3972 image_replacements["{{FSX_FILE_SYSTEM_ID}}"] = self.fsx_file_system.ref
3973 image_replacements["{{FSX_DNS_NAME}}"] = self.fsx_file_system.attr_dns_name
3974 image_replacements["{{FSX_MOUNT_NAME}}"] = self.fsx_file_system.attr_lustre_mount_name
3975 image_replacements["{{PRIVATE_SUBNET_ID}}"] = self.vpc.private_subnets[0].subnet_id
3976 image_replacements["{{FSX_SECURITY_GROUP_ID}}"] = (
3977 self.fsx_security_group.security_group_id
3978 )
3980 # ── Trigger the convergence pipeline (fire-and-forget) ───────────────
3981 # A single custom resource starts the HelmInstallStateMachine, which now
3982 # owns the WHOLE cluster convergence: apply base manifests -> install
3983 # Helm charts -> apply post-Helm (CRD-dependent) manifests -> publish
3984 # the Gateway-created ALB and optionally register it with Global
3985 # Accelerator. The resource returns
3986 # as soon as the execution is *started* (no isComplete waiter), so the
3987 # cluster's CloudFormation lifecycle is never bound to the multi-minute
3988 # add-on convergence — a slow chart can't blow CloudFormation's ~1h
3989 # custom-resource ceiling and roll back (destroy) the freshly-created
3990 # cluster. Status lives in SSM and is surfaced via `gco stacks addons
3991 # status`; re-converge out-of-band with `gco stacks addons install`.
3992 #
3993 # The execution input carries everything the state-machine tasks need:
3994 # chart selection/overrides, the manifest ImageReplacements (for the base
3995 # and post-Helm kubectl passes), the endpoint-registry identity, and the
3996 # optional Global Accelerator EndpointGroupArn.
3997 convergence_properties: dict[str, Any] = {
3998 "ClusterName": self.cluster.cluster_name,
3999 "Region": self.deployment_region,
4000 # Helm chart selection + per-chart value overrides (e.g. Volcano
4001 # image_registry redirected to the ECR mirror when enabled).
4002 "EnabledCharts": self._get_enabled_helm_charts(),
4003 "Charts": self._helm_chart_value_overrides(),
4004 "KedaOperatorRoleArn": self.keda_operator_role.role_arn,
4005 # Template substitutions for the base + post-Helm kubectl passes.
4006 "ImageReplacements": image_replacements,
4007 # Project name lets the orchestrator persist the execution input
4008 # to SSM so `gco stacks addons install` can replay the whole
4009 # pipeline without reconstructing chart/manifest config.
4010 "ProjectName": self.config.get_project_name(),
4011 "RegistryRegion": self.config.get_global_region(),
4012 # Force re-invocation on every deployment (new charts.yaml,
4013 # manifest, or image) so convergence re-runs end to end.
4014 "DeploymentTimestamp": deployment_timestamp,
4015 }
4016 if self.global_accelerator_enabled:
4017 convergence_properties["EndpointGroupArn"] = self.endpoint_group_arn
4019 converge_trigger = CustomResource(
4020 self,
4021 "HelmInstallCharts",
4022 service_token=self.helm_installer_provider.service_token,
4023 properties=convergence_properties,
4024 )
4025 converge_trigger.node.add_dependency(self.helm_installer_provider_log_group)
4026 converge_trigger.node.add_dependency(self.aws_load_balancer_controller_policy)
4028 # The trigger (and therefore the whole convergence pipeline) must run
4029 # after the cluster, shared storage, managed-addon IRSA patches, and Pod
4030 # Identity associations exist: the base manifests reference their tokens,
4031 # and the rollout-restarts at the end of the base pass need the patched
4032 # service accounts (otherwise the mutating webhook can't inject
4033 # AWS_ROLE_ARN and the controllers fail with "no EC2 IMDS role found" —
4034 # PVCs stuck Pending, missing Container Insights metrics; see the
4035 # UpdateEfsCsiAddonRole resource in _create_efs_csi_driver_addon). These
4036 # gates previously sat on the synchronous KubectlApplyManifests custom
4037 # resource; the base apply now lives in the state machine, so the gate
4038 # moves to the trigger.
4039 converge_trigger.node.add_dependency(self.cluster)
4040 converge_trigger.node.add_dependency(self.efs_file_system)
4041 if self.fsx_file_system:
4042 converge_trigger.node.add_dependency(self.fsx_file_system)
4043 for attr in (
4044 "_efs_csi_addon_role_update",
4045 "_fsx_csi_addon_role_update",
4046 "_cloudwatch_addon_role_update",
4047 ):
4048 update_cr = getattr(self, attr, None)
4049 if update_cr is not None:
4050 converge_trigger.node.add_dependency(update_cr)
4051 # The trigger also needs both EKS access entries before it starts the
4052 # asynchronous pipeline. Keeping these explicit is essential on delete:
4053 # the ordered Helm teardown runs while its Kubernetes authentication is
4054 # still valid, then the trigger/access entries/cluster can disappear.
4055 for attr in (
4056 "kubectl_lambda_access_entry",
4057 "helm_installer_access_entry",
4058 "ga_registration_access_entry",
4059 ):
4060 access_entry = getattr(self, attr, None)
4061 if access_entry is not None:
4062 converge_trigger.node.add_dependency(access_entry)
4063 for assoc in self._pod_identity_associations:
4064 converge_trigger.node.add_dependency(assoc)
4066 # Deletion must run in the opposite safety order: synchronous Helm
4067 # teardown first (quiescing endpoint writers and removing Gateway
4068 # resources), then the unconditional endpoint deregistration guard,
4069 # then the convergence trigger and its EKS access entries. Build the
4070 # create-time chain as trigger -> endpoint guard -> Helm teardown so
4071 # CloudFormation reverses it during stack deletion.
4072 helm_teardown = getattr(self, "helm_teardown_resource", None)
4073 ga_deregistration = getattr(self, "ga_deregistration_resource", None)
4074 if helm_teardown is not None:
4075 if ga_deregistration is not None:
4076 helm_teardown.node.add_dependency(ga_deregistration)
4077 ga_deregistration.node.add_dependency(converge_trigger)
4078 else:
4079 helm_teardown.node.add_dependency(converge_trigger)
4080 elif ga_deregistration is not None:
4081 ga_deregistration.node.add_dependency(converge_trigger)
4083 def _create_ga_registration_lambda(self) -> None:
4084 """Create the exact Gateway ALB discovery and endpoint-publication Lambda.
4086 Every partition uses this function to discover ``gco-system/gco-gateway``
4087 and publish its internal ALB hostname to the regional SSM registry.
4088 Commercial partitions additionally pass an endpoint-group ARN so the
4089 same exact ALB is registered with Global Accelerator.
4090 """
4091 project_name = self.config.get_project_name()
4093 # Create Lambda function for GA registration using external handler
4094 ga_registration_lambda = lambda_.Function(
4095 self,
4096 "GaRegistrationFunction",
4097 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
4098 handler="handler.lambda_handler",
4099 code=lambda_.Code.from_asset("lambda/ga-registration"),
4100 timeout=Duration.minutes(15), # Max Lambda timeout; handler uses 14 min budget
4101 memory_size=256,
4102 vpc=self.vpc,
4103 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
4104 environment={
4105 "CLUSTER_NAME": self.cluster.cluster_name,
4106 "REGION": self.deployment_region,
4107 },
4108 tracing=lambda_.Tracing.ACTIVE,
4109 )
4111 # Grant permissions
4112 ga_registration_lambda.add_to_role_policy(
4113 iam.PolicyStatement(
4114 effect=iam.Effect.ALLOW,
4115 actions=["eks:DescribeCluster"],
4116 resources=[self.cluster.cluster_arn],
4117 )
4118 )
4119 ga_registration_lambda.add_to_role_policy(
4120 iam.PolicyStatement(
4121 effect=iam.Effect.ALLOW,
4122 actions=[
4123 "elasticloadbalancing:DescribeLoadBalancers",
4124 "elasticloadbalancing:DescribeTags", # Required for tag-based ALB detection
4125 ],
4126 resources=["*"],
4127 )
4128 )
4129 if self.global_accelerator_enabled:
4130 ga_registration_lambda.add_to_role_policy(
4131 iam.PolicyStatement(
4132 effect=iam.Effect.ALLOW,
4133 actions=[
4134 "globalaccelerator:AddEndpoints",
4135 "globalaccelerator:RemoveEndpoints",
4136 "globalaccelerator:UpdateEndpointGroup",
4137 "globalaccelerator:DescribeEndpointGroup",
4138 # The teardown-time cleanup_gateway_endpoint task runs
4139 # on this Lambda and strictly waits for the accelerator
4140 # to reach DEPLOYED after endpoint removal.
4141 "globalaccelerator:DescribeAccelerator",
4142 ],
4143 resources=["*"],
4144 )
4145 )
4146 ga_registration_lambda.add_to_role_policy(
4147 iam.PolicyStatement(
4148 effect=iam.Effect.ALLOW,
4149 actions=["ssm:GetParameter", "ssm:PutParameter", "ssm:DeleteParameter"],
4150 resources=[
4151 f"arn:{self.partition}:ssm:{self.config.get_global_region()}:"
4152 f"{self.account}:parameter/{project_name}/*"
4153 ],
4154 )
4155 )
4157 # Retain the access entry so asynchronous convergence cannot start
4158 # until the endpoint publisher can read the exact Gateway object.
4159 ga_registration_role = cast(iam.IRole, ga_registration_lambda.role)
4160 self.ga_registration_access_entry = eks.AccessEntry(
4161 self,
4162 "GaRegistrationLambdaAccessEntry",
4163 cluster=self.cluster, # type: ignore[arg-type]
4164 principal=ga_registration_role.role_arn,
4165 access_policies=[
4166 eks.AccessPolicy.from_access_policy_name(
4167 "AmazonEKSClusterAdminPolicy", access_scope_type=eks.AccessScopeType.CLUSTER
4168 )
4169 ],
4170 )
4172 # Allow Lambda to access EKS API
4173 self.cluster.cluster_security_group.add_ingress_rule(
4174 peer=ec2.Peer.ipv4(self.vpc.vpc_cidr_block),
4175 connection=ec2.Port.tcp(443),
4176 description="Allow GA registration Lambda to access EKS API",
4177 )
4179 # Global Accelerator exists only in supported partitions. Resolve its
4180 # endpoint group lazily there; regional endpoint publication itself is
4181 # unconditional and needs no GA lookup.
4182 self.endpoint_group_arn: str | None = None
4183 if self.global_accelerator_enabled:
4184 global_region = self.config.get_global_region()
4185 get_endpoint_group_arn = cr.AwsCustomResource(
4186 self,
4187 "GetEndpointGroupArn",
4188 on_create=cr.AwsSdkCall(
4189 service="SSM",
4190 action="getParameter",
4191 parameters={
4192 "Name": f"/{project_name}/endpoint-group-{self.deployment_region}-arn"
4193 },
4194 region=global_region,
4195 physical_resource_id=cr.PhysicalResourceId.of(
4196 f"{project_name}-get-endpoint-group-arn-{self.deployment_region}"
4197 ),
4198 ),
4199 on_update=cr.AwsSdkCall(
4200 service="SSM",
4201 action="getParameter",
4202 parameters={
4203 "Name": f"/{project_name}/endpoint-group-{self.deployment_region}-arn"
4204 },
4205 region=global_region,
4206 ),
4207 role=self.aws_custom_resource_role,
4208 )
4209 get_endpoint_group_arn.node.add_dependency(self.aws_custom_resource_role)
4210 self.endpoint_group_arn = get_endpoint_group_arn.get_response_field("Parameter.Value")
4212 # Invoked directly by the convergence state machine's final task.
4213 self.ga_registration_lambda = ga_registration_lambda
4215 # cdk-nag suppression: the GA registration Lambda needs broad
4216 # Global Accelerator and ELB Describe access with Resource: *.
4217 from gco.stacks.nag_suppressions import acknowledge_nag_findings
4219 acknowledge_nag_findings(
4220 ga_registration_lambda,
4221 [
4222 {
4223 "id": "AwsSolutions-IAM5",
4224 "reason": (
4225 "The endpoint-publication Lambda needs ELB Describe access to "
4226 "resolve the exact Gateway-owned ALB. In partitions with Global "
4227 "Accelerator it also needs the service's endpoint-group mutation "
4228 "APIs. These APIs do not support resource-level scoping."
4229 ),
4230 "appliesTo": ["Resource::*"],
4231 },
4232 ],
4233 )
4235 # Wire the delete-time teardown guard that deregisters this region's ALB
4236 # from Global Accelerator before its VPC subnets are deleted.
4237 self._create_ga_deregistration_resource()
4239 def _create_ga_deregistration_resource(self) -> None:
4240 """Create the unconditional endpoint-registry delete guard.
4242 On stack deletion the guard always removes this region's SSM hostname.
4243 When an endpoint group exists it first deregisters the ALB and waits for
4244 Global Accelerator to release its managed ENIs. The resource therefore
4245 exists in every partition even though the GA portion is optional.
4246 """
4247 project_name = self.config.get_project_name()
4249 # Dedicated Lambda built from the SAME asset as the registration Lambda
4250 # (it reuses the shared remove/wait helpers, entry point
4251 # handler.on_delete_event). Deliberately NOT in the VPC: it only calls
4252 # the public Global Accelerator API and must not create its own ENIs in
4253 # the VPC it is helping to tear down.
4254 ga_deregistration_lambda = lambda_.Function(
4255 self,
4256 "GaDeregistrationFunction",
4257 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
4258 handler="handler.on_delete_event",
4259 code=lambda_.Code.from_asset("lambda/ga-registration"),
4260 timeout=Duration.minutes(15), # covers the GA redeploy wait budget
4261 memory_size=256,
4262 tracing=lambda_.Tracing.ACTIVE,
4263 )
4264 if self.global_accelerator_enabled:
4265 ga_deregistration_lambda.add_to_role_policy(
4266 iam.PolicyStatement(
4267 effect=iam.Effect.ALLOW,
4268 actions=[
4269 "globalaccelerator:DescribeAccelerator",
4270 "globalaccelerator:DescribeEndpointGroup",
4271 "globalaccelerator:RemoveEndpoints",
4272 "globalaccelerator:UpdateEndpointGroup",
4273 ],
4274 resources=["*"],
4275 )
4276 )
4278 ga_deregistration_lambda.add_to_role_policy(
4279 iam.PolicyStatement(
4280 effect=iam.Effect.ALLOW,
4281 actions=["ssm:DeleteParameter"],
4282 resources=[
4283 f"arn:{self.partition}:ssm:{self.config.get_global_region()}:{self.account}:"
4284 f"parameter/{project_name}/alb-hostname-{self.deployment_region}"
4285 ],
4286 )
4287 )
4289 # Strict live validation retains the exact generation through the
4290 # provider's final delete invocation; its identity-fenced post-stack
4291 # cleanup removes it. Ordinary deployments retain DESTROY semantics.
4292 ga_deregistration_log_group = logs.LogGroup(
4293 self,
4294 "GaDeregistrationProviderLogGroup",
4295 retention=logs.RetentionDays.ONE_WEEK,
4296 removal_policy=self.provider_log_group_removal_policy,
4297 )
4298 ga_deregistration_provider = cr.Provider(
4299 self,
4300 "GaDeregistrationProvider",
4301 on_event_handler=ga_deregistration_lambda,
4302 log_group=ga_deregistration_log_group,
4303 )
4305 deregistration_properties: dict[str, Any] = {
4306 "Region": self.deployment_region,
4307 "RegistryRegion": self.config.get_global_region(),
4308 "ProjectName": project_name,
4309 }
4310 if self.endpoint_group_arn is not None:
4311 deregistration_properties["EndpointGroupArn"] = self.endpoint_group_arn
4313 ga_deregistration = CustomResource(
4314 self,
4315 "GaDeregistration",
4316 service_token=ga_deregistration_provider.service_token,
4317 properties=deregistration_properties,
4318 )
4320 # Teardown ordering: this deregistration must run BEFORE the VPC (and its
4321 # public subnets, where Global Accelerator pins its managed ENIs) is
4322 # deleted. Depending on the VPC means CloudFormation creates the VPC
4323 # first and — critically — deletes this custom resource first on
4324 # teardown, releasing the GA ENIs so the subnets can be removed cleanly.
4325 self.ga_deregistration_resource = ga_deregistration
4326 ga_deregistration.node.add_dependency(self.vpc)
4327 ga_deregistration.node.add_dependency(ga_deregistration_log_group)
4329 # cdk-nag: the deregistration Lambda needs globalaccelerator Describe*/
4330 # RemoveEndpoints with Resource: * (these Global Accelerator APIs do not
4331 # support resource-level IAM scoping), mirroring the registration Lambda.
4332 from gco.stacks.nag_suppressions import acknowledge_nag_findings
4334 acknowledge_nag_findings(
4335 ga_deregistration_lambda,
4336 [
4337 {
4338 "id": "AwsSolutions-IAM5",
4339 "reason": (
4340 "Where Global Accelerator is enabled, the delete guard needs its "
4341 "Describe and endpoint-group mutation APIs to release managed ENIs. "
4342 "Those APIs do not support resource-level IAM scoping; non-GA "
4343 "partitions receive no Global Accelerator actions."
4344 ),
4345 "appliesTo": ["Resource::*"],
4346 },
4347 ],
4348 )
4350 def _get_volcano_image_mirror_config(self) -> dict[str, Any]:
4351 """Parse the ``volcano_image_mirror`` block from cdk.json.
4353 Returns a normalized dict ``{enabled, ecr_namespace}``. Validation is
4354 strict so a misconfiguration fails at synth rather than silently leaving
4355 Volcano pointed at docker.io.
4357 - ``enabled`` (default False) — master toggle.
4358 - ``ecr_namespace`` (default ``"<project_name>/dockerhub"``, i.e.
4359 ``gco/dockerhub`` for the stock project) — the ECR repository
4360 namespace the mirrored Volcano images live under. Must start with
4361 ``<project_name>/`` so it inherits the project's existing
4362 ``<project_name>/*`` machinery (node pull access, replication rule,
4363 trusted-registry allow-list) with no extra IAM, and must be a valid
4364 (possibly nested) ECR repository path.
4365 """
4366 raw = self.node.try_get_context("volcano_image_mirror") or {}
4367 if not isinstance(raw, dict):
4368 raise ValueError(f"volcano_image_mirror must be a mapping, got {type(raw).__name__}")
4370 # The mirror namespace lives under this deployment's project prefix
4371 # (``<project_name>/``) so it inherits the project's ECR access,
4372 # replication rule, and trusted-registry allow-list (#139). Defaults to
4373 # ``<project_name>/dockerhub`` — ``gco/dockerhub`` for the stock project.
4374 project_prefix = f"{self.config.get_project_name()}/"
4375 enabled = bool(raw.get("enabled", False))
4376 ecr_namespace = (
4377 str(raw.get("ecr_namespace", f"{project_prefix}dockerhub")).strip().strip("/")
4378 )
4380 if not enabled:
4381 return {"enabled": False, "ecr_namespace": ecr_namespace}
4383 # Must live under the project prefix and be a valid nested ECR repo
4384 # path (lowercase alphanumerics + . _ - per segment, slash-separated).
4385 if not ecr_namespace.startswith(project_prefix):
4386 raise ValueError(
4387 f"volcano_image_mirror.ecr_namespace must start with {project_prefix!r} so it "
4388 f"inherits the project's {project_prefix}* ECR access/replication, got "
4389 f"{ecr_namespace!r}"
4390 )
4391 segment = r"[a-z0-9]+(?:[._-][a-z0-9]+)*"
4392 if not re.fullmatch(rf"{segment}(?:/{segment})+", ecr_namespace):
4393 raise ValueError(
4394 "volcano_image_mirror.ecr_namespace must be a valid ECR repository "
4395 f"path (lowercase alphanumerics + . _ - per slash-separated segment), "
4396 f"got {ecr_namespace!r}"
4397 )
4399 return {"enabled": True, "ecr_namespace": ecr_namespace}
4401 def _configure_volcano_image_mirror(self) -> None:
4402 """Resolve the optional Volcano image-mirror registry (no infra).
4404 Volcano is the only default chart whose images live exclusively on
4405 docker.io (``volcanosh/vc-*``). On a cold EKS Auto Mode cluster those
4406 anonymous pulls are slow / rate-limited, so Volcano's blocking
4407 ``helm --wait`` could never finish inside the installer Lambda's
4408 wall-clock guard and the whole add-on batch looped on it.
4410 The fix is to mirror Volcano's pinned images into the project's own ECR
4411 under ``gco/*`` and point Volcano's ``basic.image_registry`` there, so
4412 the cluster makes fast, same-account ECR pulls with the pull-only node
4413 role it already has — no Docker Hub credential, no pull-through cache
4414 rule, and no registry permissions policy. The mirror itself is populated
4415 out-of-band (``gco images mirror``) before the add-ons
4416 converge; this method only computes the registry override and creates no
4417 CloudFormation resources.
4419 Sets ``self.volcano_mirror_registry`` to
4420 ``<account>.dkr.ecr.<region>.<url-suffix>/<ecr_namespace>`` that
4421 ``_helm_chart_value_overrides`` feeds into Volcano's
4422 ``basic.image_registry``; left ``None`` when disabled.
4423 """
4424 # Always define the attribute so downstream code can branch on it.
4425 self.volcano_mirror_registry: str | None = None
4427 cfg = self._get_volcano_image_mirror_config()
4428 if not cfg["enabled"]:
4429 return
4431 ecr_namespace = cfg["ecr_namespace"]
4432 self.volcano_mirror_registry = (
4433 f"{self.account}.dkr.ecr.{self.deployment_region}.{self.url_suffix}/{ecr_namespace}"
4434 )
4436 def _helm_chart_value_overrides(self) -> dict[str, Any]:
4437 """Per-chart helm value overrides injected into the install payload.
4439 Returned dict is forwarded verbatim as the ``Charts`` property of the
4440 ``HelmInstallCharts`` custom resource; the installer deep-merges each
4441 chart's ``values`` over ``charts.yaml``. The mandatory
4442 ``aws-load-balancer-controller`` chart always receives the cluster,
4443 region, VPC, and dedicated IRSA role values. Optional overrides are:
4445 - ``volcano``: point ``basic.image_registry`` at the project's ECR
4446 image mirror when enabled, so every Volcano image (controller,
4447 scheduler, admission webhook, and the pre-install admission-init
4448 hook) resolves from ECR instead of docker.io. The upstream names
4449 (``volcanosh/vc-*``) are preserved, so each resolves to
4450 ``<mirror_registry>/volcanosh/vc-*``.
4451 - ``kube-prometheus-stack``: inject the ``cdk.json``-derived dynamic
4452 values (Grafana/Prometheus/Alertmanager persistence sizes, Prometheus
4453 retention, the gp3 ``storageClassName``, and the GPU/Neuron/EFA
4454 node-exporter tolerations) over the static hardening values in
4455 ``charts.yaml`` when ``cluster_observability.enabled`` is true.
4457 The result is never empty because Gateway API requires the controller.
4458 """
4459 overrides: dict[str, Any] = {
4460 "aws-load-balancer-controller": {
4461 "values": {
4462 "clusterName": self.cluster.cluster_name,
4463 "region": self.deployment_region,
4464 "vpcId": self.vpc.vpc_id,
4465 "serviceAccount": {
4466 "annotations": {
4467 "eks.amazonaws.com/role-arn": (
4468 self.aws_load_balancer_controller_role.role_arn
4469 )
4470 }
4471 },
4472 }
4473 }
4474 }
4476 if getattr(self, "volcano_mirror_registry", None):
4477 overrides["volcano"] = {
4478 "values": {
4479 "basic": {
4480 "image_registry": self.volcano_mirror_registry,
4481 }
4482 }
4483 }
4485 if self.config.get_cluster_observability_enabled():
4486 overrides["kube-prometheus-stack"] = self._observability_chart_values()
4488 if self._cost_monitoring_active():
4489 overrides["opencost"] = self._opencost_chart_values()
4491 if self._mlflow_active():
4492 overrides["mlflow"] = self._mlflow_chart_values()
4494 return overrides
4496 def _cost_monitoring_active(self) -> bool:
4497 """Return whether the per-region cost monitoring pipeline deploys here.
4499 Delegates to ``ConfigLoader.get_cost_monitoring_enabled``, which is
4500 already the conjunction of the ``cost_monitoring`` toggle and its
4501 ``cluster_observability`` data-source dependency — disabling either
4502 switches OpenCost, the cost-monitor service, and the cost dashboard
4503 off together.
4504 """
4505 return self.config.get_cost_monitoring_enabled()
4507 def _mlflow_active(self) -> bool:
4508 """Return whether the MLflow tracking server deploys on this cluster.
4510 Delegates to ``ConfigLoader.get_mlflow_enabled`` — the conjunction of
4511 ``cluster_observability.mlflow.enabled`` and observability itself,
4512 so the chart, its IRSA role, the gated backend PVC, and the value
4513 overrides all switch together.
4514 """
4515 return self.config.get_mlflow_enabled()
4517 def _mlflow_chart_values(self) -> dict[str, Any]:
4518 """Build the MLflow value overrides that carry deployment tokens.
4520 Only four things are dynamic — everything static (image pin, PVC
4521 wiring, resources, posture toggles) lives in ``charts.yaml``:
4523 - ``mlflow.artifactsDestination``: run artifacts go to the
4524 cluster-shared bucket under ``mlflow-artifacts/<region>/`` —
4525 region-suffixed because each regional tracking server numbers
4526 experiments independently, so a shared root would interleave
4527 unrelated runs' artifacts. The server proxies artifact traffic
4528 (``--serve-artifacts`` is the server default), so client pods
4529 never need S3 credentials of their own.
4530 - ``serviceAccount.annotations``: the IRSA role ARN, which is how
4531 the server-side artifact proxy gets its S3 credentials.
4532 - ``storage.size``: the metadata claim size from
4533 ``cluster_observability.mlflow.persistence_size``.
4534 - ``server.value_options.allowed_hosts``: the complete
4535 host-validation allow-list — service DNS plus wildcard patterns
4536 derived from ``vpc_endpoint_cidrs`` (see
4537 ``_mlflow_allowed_hosts``); the deep merge keeps the static
4538 ``workers`` value while replacing the charts.yaml DNS-only
4539 fallback with this full list.
4540 """
4541 s3_destination = (
4542 f"s3://{self.cluster_shared_identity.name}/mlflow-artifacts/{self.deployment_region}"
4543 )
4544 vpc_endpoint_cidrs = self.node.try_get_context("vpc_endpoint_cidrs") or ["10.0.0.0/16"]
4545 return {
4546 "values": {
4547 "mlflow": {
4548 "artifactsDestination": s3_destination,
4549 },
4550 "serviceAccount": {
4551 "annotations": {
4552 "eks.amazonaws.com/role-arn": self.mlflow_role.role_arn,
4553 },
4554 },
4555 "storage": {
4556 "size": str(
4557 self.config.get_cluster_observability_config()["mlflow"]["persistence_size"]
4558 ),
4559 },
4560 "server": {
4561 "value_options": {
4562 "allowed_hosts": _mlflow_allowed_hosts(vpc_endpoint_cidrs),
4563 },
4564 },
4565 }
4566 }
4568 def _opencost_chart_values(self) -> dict[str, Any]:
4569 """Build the OpenCost value overrides that carry deployment tokens.
4571 Only the cluster identity is dynamic — every static hardening value
4572 (Prometheus wiring, ServiceMonitor, resource limits, security
4573 contexts) lives in ``charts.yaml``. The identity is
4574 ``opencost.exporter.defaultClusterId`` — the value OpenCost stamps on
4575 every allocation row, which is what lets the multi-region Athena data
4576 distinguish clusters. The chart's root-level ``clusterName`` is NOT
4577 set here: that value is the Kubernetes DNS zone (``cluster.local``)
4578 used to build the Prometheus URL, and overriding it with the EKS
4579 cluster name breaks in-cluster DNS resolution.
4580 """
4581 return {
4582 "values": {
4583 "opencost": {
4584 "exporter": {
4585 "defaultClusterId": self.cluster.cluster_name,
4586 },
4587 },
4588 }
4589 }
4591 def _observability_chart_values(self) -> dict[str, Any]:
4592 """Build the kube-prometheus-stack value overrides from cdk.json.
4594 Sizes/retention come from ``cluster_observability`` in ``cdk.json``; the
4595 gp3 ``storageClassName`` is the shared ``_OBSERVABILITY_STORAGE_CLASS``
4596 (also the name of the gated StorageClass manifest), and the
4597 node-exporter tolerations reuse the shared accelerator-node tolerations
4598 so the DaemonSet schedules on tainted GPU/Neuron/EFA nodes. Deep-merged
4599 by the installer over the static hardening values in ``charts.yaml``.
4600 """
4601 obs = self.config.get_cluster_observability_config()
4602 storage_class = _OBSERVABILITY_STORAGE_CLASS
4603 return {
4604 "values": {
4605 "grafana": {
4606 "persistence": {
4607 "storageClassName": storage_class,
4608 "size": obs["grafana"]["persistence_size"],
4609 },
4610 },
4611 "prometheus": {
4612 "prometheusSpec": {
4613 "retention": obs["prometheus"]["retention"],
4614 "storageSpec": {
4615 "volumeClaimTemplate": {
4616 "spec": {
4617 "storageClassName": storage_class,
4618 "resources": {
4619 "requests": {
4620 "storage": obs["prometheus"]["persistence_size"],
4621 },
4622 },
4623 },
4624 },
4625 },
4626 },
4627 },
4628 "alertmanager": {
4629 "enabled": obs["alertmanager"]["enabled"],
4630 "alertmanagerSpec": {
4631 "storage": {
4632 "volumeClaimTemplate": {
4633 "spec": {
4634 "storageClassName": storage_class,
4635 "resources": {
4636 "requests": {
4637 "storage": obs["alertmanager"]["persistence_size"],
4638 },
4639 },
4640 },
4641 },
4642 },
4643 },
4644 },
4645 "prometheus-node-exporter": {
4646 "tolerations": GCORegionalStack._ADDON_NODE_TOLERATIONS,
4647 },
4648 }
4649 }
4651 def _get_enabled_helm_charts(self) -> list[str]:
4652 """Return the list of Helm charts to install based on cdk.json helm config.
4654 Reads the 'helm' section from cdk.json context. Each key maps to one or
4655 more Helm chart names. Charts are returned in dependency order with Kueue
4656 last (its webhook intercepts all Job/Deployment mutations).
4657 """
4658 helm_config = self.node.try_get_context("helm") or {}
4660 # Mapping from cdk.json helm key → Helm chart name(s) in charts.yaml
4661 # Order matters: dependencies first, Kueue last
4662 chart_map: list[tuple[str, list[str]]] = [
4663 ("aws_load_balancer_controller", ["aws-load-balancer-controller"]),
4664 ("keda", ["keda"]),
4665 ("aws_efa_device_plugin", ["aws-efa-device-plugin"]),
4666 ("aws_neuron_device_plugin", ["aws-neuron-device-plugin"]),
4667 ("volcano", ["volcano"]),
4668 ("kuberay", ["kuberay-operator"]),
4669 ("cert_manager", ["cert-manager"]),
4670 ("slurm", ["slinky-slurm-operator", "slinky-slurm"]),
4671 ("yunikorn", ["yunikorn"]),
4672 ("kubeflow_trainer", ["kubeflow-trainer"]),
4673 ("kueue", ["kueue"]), # Must be last
4674 ]
4676 # Charts that are mandatory platform components and cannot be disabled
4677 # via cdk.json (see _MANDATORY_CHART_KEYS). KEDA is always installed:
4678 # it backs the built-in SQS queue processor (a ScaledJob) and is the
4679 # only metrics bridge that lets autoscalers consume GPU/CloudWatch
4680 # metrics (the keda-metrics-apiserver serves external.metrics.k8s.io).
4681 # Disabling it would silently break both, so the cdk.json toggle is
4682 # ignored for KEDA. A `helm_enabled_overrides` context value (see
4683 # _parse_helm_enabled_overrides) can force optional charts on for one
4684 # deploy without editing cdk.json.
4685 if {key for key, _names in chart_map} != _HELM_CHART_CONFIG_KEYS:
4686 raise RuntimeError(
4687 "chart_map keys drifted from _HELM_CHART_CONFIG_KEYS; update both together"
4688 )
4689 overrides = _parse_helm_enabled_overrides(
4690 self.node.try_get_context(_HELM_OVERRIDE_CONTEXT_KEY)
4691 )
4693 enabled_charts = []
4694 for config_key, chart_names in chart_map:
4695 if _helm_chart_enabled(helm_config, overrides, config_key):
4696 enabled_charts.extend(chart_names)
4698 # kube-prometheus-stack is driven by the separate on-by-default
4699 # cluster_observability toggle (not the helm block), so include it here
4700 # when enabled. Its install order comes from its file position in
4701 # charts.yaml (before kueue), not from where it sits in this list — the
4702 # installer runs one task per chart in charts.yaml order and skips any
4703 # task whose chart is absent from this enabled set.
4704 if self.config.get_cluster_observability_enabled():
4705 enabled_charts.append("kube-prometheus-stack")
4707 # OpenCost is driven by the on-by-default cost_monitoring toggle and
4708 # additionally requires observability (its Prometheus data source).
4709 # charts.yaml places it after kube-prometheus-stack so the Prometheus
4710 # Operator CRDs exist before its ServiceMonitor renders.
4711 if self._cost_monitoring_active():
4712 enabled_charts.append("opencost")
4714 # MLflow is driven by the on-by-default cluster_observability.mlflow
4715 # sub-toggle and requires observability itself (monitoring namespace,
4716 # gp3 StorageClass, ServiceMonitor discovery, tunnel access path).
4717 if self._mlflow_active():
4718 enabled_charts.append("mlflow")
4720 return enabled_charts
4722 def _create_helm_installer_lambda(self) -> None:
4723 """Create Lambda function to install Helm charts (KEDA, NVIDIA DRA, etc.).
4725 This Lambda uses Helm to install charts that require complex setup
4726 (TLS certificates, CRDs, etc.) that are difficult to manage via raw manifests.
4728 Charts installed:
4729 - KEDA: Kubernetes Event-Driven Autoscaling (mandatory, always installed)
4730 - Volcano, KubeRay, Kueue, cert-manager, and other schedulers (toggle via cdk.json)
4731 """
4732 project_name = self.config.get_project_name()
4734 # Create IAM role for Helm installer Lambda
4735 helm_lambda_role = iam.Role(
4736 self,
4737 "HelmInstallerLambdaRole",
4738 assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
4739 managed_policies=[
4740 iam.ManagedPolicy.from_aws_managed_policy_name(
4741 "service-role/AWSLambdaVPCAccessExecutionRole"
4742 ),
4743 iam.ManagedPolicy.from_aws_managed_policy_name(
4744 "service-role/AWSLambdaBasicExecutionRole"
4745 ),
4746 ],
4747 )
4749 # Add EKS permissions
4750 helm_lambda_role.add_to_policy(
4751 iam.PolicyStatement(
4752 actions=["eks:DescribeCluster", "eks:ListClusters"],
4753 resources=[self.cluster.cluster_arn],
4754 )
4755 )
4757 # Create security group for Helm installer Lambda
4758 helm_lambda_sg = ec2.SecurityGroup(
4759 self,
4760 "HelmInstallerLambdaSG",
4761 vpc=self.vpc,
4762 description="Security group for Helm installer Lambda to access EKS cluster",
4763 security_group_name=f"{project_name}-helm-lambda-sg-{self.deployment_region}",
4764 allow_all_outbound=True,
4765 )
4767 # Allow Lambda to access EKS cluster API
4768 self.cluster.cluster_security_group.add_ingress_rule(
4769 peer=helm_lambda_sg,
4770 connection=ec2.Port.tcp(443),
4771 description="Allow Helm installer Lambda to access EKS API",
4772 )
4774 # Build Docker image for Helm installer Lambda
4775 # Points at helm-installer-build/ which is rebuilt fresh every deploy
4776 # by _build_helm_installer_lambda() in cli/stacks.py
4777 ecr_assets.DockerImageAsset(
4778 self,
4779 "HelmInstallerImage",
4780 directory="lambda/helm-installer-build",
4781 platform=ecr_assets.Platform.LINUX_AMD64,
4782 )
4784 # Create Lambda function using Docker image
4785 # Store function name as string attribute for cross-stack references
4786 # This avoids CDK cross-environment resolution issues when account is unresolved
4787 self.helm_installer_lambda_function_name = f"{project_name}-helm-{self.deployment_region}"
4788 self.helm_installer_lambda = lambda_.DockerImageFunction(
4789 self,
4790 "HelmInstallerFunction",
4791 function_name=self.helm_installer_lambda_function_name,
4792 code=lambda_.DockerImageCode.from_image_asset(
4793 directory="lambda/helm-installer-build",
4794 platform=ecr_assets.Platform.LINUX_AMD64,
4795 ),
4796 timeout=Duration.minutes(15),
4797 memory_size=1024,
4798 architecture=lambda_.Architecture.X86_64,
4799 role=helm_lambda_role,
4800 vpc=self.vpc,
4801 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
4802 security_groups=[helm_lambda_sg],
4803 environment={
4804 "CLUSTER_NAME": self.cluster.cluster_name,
4805 "REGION": self.deployment_region,
4806 "PROJECT_NAME": project_name,
4807 },
4808 tracing=lambda_.Tracing.ACTIVE,
4809 )
4811 # Allow the installer to record per-chart add-on status to SSM so the
4812 # add-on layer's health is observable out-of-band (decoupled from the
4813 # CloudFormation rollback path). Read back via `gco stacks addons-status`.
4814 helm_lambda_role.add_to_policy(
4815 iam.PolicyStatement(
4816 actions=["ssm:PutParameter"],
4817 resources=[
4818 f"arn:{self.partition}:ssm:{self.deployment_region}:{self.account}:"
4819 f"parameter/{project_name}/addons/*"
4820 ],
4821 )
4822 )
4824 # Add EKS access entry for the Lambda role
4825 self.helm_installer_access_entry = eks.AccessEntry(
4826 self,
4827 "HelmInstallerLambdaAccessEntry",
4828 cluster=self.cluster, # type: ignore[arg-type]
4829 principal=helm_lambda_role.role_arn,
4830 access_policies=[
4831 eks.AccessPolicy.from_access_policy_name(
4832 "AmazonEKSClusterAdminPolicy", access_scope_type=eks.AccessScopeType.CLUSTER
4833 )
4834 ],
4835 )
4837 # ------------------------------------------------------------------
4838 # Step Functions state machine: one task per chart, in charts.yaml
4839 # order. Each chart gets its own retry + Step Functions console
4840 # visibility, and — critically — no single Lambda invocation is bound
4841 # by the 15-minute Lambda limit, so a slow operator (e.g. a cold NVIDIA
4842 # image pull) just costs extra retries instead of failing the deploy.
4843 # ------------------------------------------------------------------
4844 chart_order = _load_helm_chart_order()
4846 def _chart_task(chart_name: str) -> sfn_tasks.LambdaInvoke:
4847 task = sfn_tasks.LambdaInvoke(
4848 self,
4849 f"HelmChart-{chart_name}",
4850 lambda_function=self.helm_installer_lambda,
4851 payload=sfn.TaskInput.from_object(
4852 {
4853 "Action": "install_chart",
4854 "Chart": chart_name,
4855 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
4856 "Region": sfn.JsonPath.string_at("$.Region"),
4857 "EnabledCharts": sfn.JsonPath.list_at("$.EnabledCharts"),
4858 "Charts": sfn.JsonPath.object_at("$.Charts"),
4859 "KedaOperatorRoleArn": sfn.JsonPath.string_at("$.KedaOperatorRoleArn"),
4860 }
4861 ),
4862 payload_response_only=True,
4863 # Keep the execution input intact so the next chart task can
4864 # still read $.ClusterName, $.EnabledCharts, etc.
4865 result_path="$.lastChart",
4866 task_timeout=sfn.Timeout.duration(Duration.minutes(16)),
4867 )
4868 # Per-chart retry with backoff. A cold image pull or a webhook race
4869 # clears on a later attempt; only after exhausting these does the
4870 # chart (and the deploy) fail.
4871 task.add_retry(
4872 errors=["States.ALL"],
4873 max_attempts=4,
4874 interval=Duration.seconds(30),
4875 backoff_rate=2.0,
4876 max_delay=Duration.minutes(5),
4877 )
4878 return task
4880 def _kubectl_task(task_id: str, *, post_helm: bool) -> sfn_tasks.LambdaInvoke:
4881 """One kubectl-apply pass (base or post-Helm) as a state-machine task.
4883 Reads ClusterName / Region / ImageReplacements from the execution
4884 input; the handler raises on any manifest failure so the task's
4885 Retry/Catch can react.
4886 """
4887 task = sfn_tasks.LambdaInvoke(
4888 self,
4889 task_id,
4890 lambda_function=self.kubectl_lambda,
4891 payload=sfn.TaskInput.from_object(
4892 {
4893 "Action": "apply_manifests",
4894 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
4895 "Region": sfn.JsonPath.string_at("$.Region"),
4896 "ImageReplacements": sfn.JsonPath.object_at("$.ImageReplacements"),
4897 "PostHelm": "true" if post_helm else "false",
4898 }
4899 ),
4900 payload_response_only=True,
4901 # Keep the execution input intact so later tasks still read
4902 # $.ClusterName, $.ImageReplacements, $.EndpointGroupArn, etc.
4903 result_path="$.lastApply",
4904 task_timeout=sfn.Timeout.duration(Duration.minutes(15)),
4905 )
4906 task.add_retry(
4907 errors=["States.ALL"],
4908 max_attempts=3,
4909 interval=Duration.seconds(30),
4910 backoff_rate=2.0,
4911 max_delay=Duration.minutes(3),
4912 )
4913 return task
4915 def _manifest_validation_task() -> sfn_tasks.LambdaInvoke:
4916 """Require every effective raw manifest object to exist and be ready."""
4917 task = sfn_tasks.LambdaInvoke(
4918 self,
4919 "ValidateKubernetesManifests",
4920 lambda_function=self.kubectl_lambda,
4921 payload=sfn.TaskInput.from_object(
4922 {
4923 "Action": "validate_manifests",
4924 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
4925 "Region": sfn.JsonPath.string_at("$.Region"),
4926 "ImageReplacements": sfn.JsonPath.object_at("$.ImageReplacements"),
4927 "DeploymentToken": sfn.JsonPath.string_at("$.DeploymentToken"),
4928 }
4929 ),
4930 payload_response_only=True,
4931 result_path="$.manifestValidation",
4932 task_timeout=sfn.Timeout.duration(Duration.minutes(15)),
4933 )
4934 # Cold EKS Auto Mode clusters can need more than the original
4935 # ~9-minute retry schedule for node capacity, cert-manager Secrets,
4936 # PDBs, and EndpointSlices to converge. Validation is read-only;
4937 # eight retries sample through ~21 minutes without replaying apply.
4938 task.add_retry(
4939 errors=["States.ALL"],
4940 max_attempts=8,
4941 interval=Duration.minutes(1),
4942 backoff_rate=2.0,
4943 max_delay=Duration.minutes(3),
4944 )
4945 return task
4947 def _helm_validation_task() -> sfn_tasks.LambdaInvoke:
4948 """Require every configured Helm release and rendered object to be ready."""
4949 task = sfn_tasks.LambdaInvoke(
4950 self,
4951 "ValidateHelmReleases",
4952 lambda_function=self.helm_installer_lambda,
4953 payload=sfn.TaskInput.from_object(
4954 {
4955 "Action": "validate_releases",
4956 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
4957 "Region": sfn.JsonPath.string_at("$.Region"),
4958 "EnabledCharts": sfn.JsonPath.list_at("$.EnabledCharts"),
4959 "Charts": sfn.JsonPath.object_at("$.Charts"),
4960 "DeploymentToken": sfn.JsonPath.string_at("$.DeploymentToken"),
4961 }
4962 ),
4963 payload_response_only=True,
4964 result_path="$.helmValidation",
4965 task_timeout=sfn.Timeout.duration(Duration.minutes(16)),
4966 )
4967 task.add_retry(
4968 errors=["States.ALL"],
4969 max_attempts=4,
4970 interval=Duration.minutes(1),
4971 backoff_rate=2.0,
4972 max_delay=Duration.minutes(3),
4973 )
4974 return task
4976 def _endpoint_publication_task() -> sfn_tasks.LambdaInvoke:
4977 """Publish the exact Gateway ALB and optionally register it with GA."""
4978 payload: dict[str, Any] = {
4979 "Action": "publish_gateway_endpoint",
4980 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
4981 "Region": sfn.JsonPath.string_at("$.Region"),
4982 "RegistryRegion": sfn.JsonPath.string_at("$.RegistryRegion"),
4983 "ProjectName": sfn.JsonPath.string_at("$.ProjectName"),
4984 }
4985 if self.global_accelerator_enabled:
4986 payload["EndpointGroupArn"] = sfn.JsonPath.string_at("$.EndpointGroupArn")
4987 # The configured health-check contract is a synth-time
4988 # constant, so it is baked into the task payload as literals
4989 # rather than threaded through the execution input: replayed
4990 # pre-upgrade inputs (`gco stacks addons install`) carry no
4991 # such keys, and a JsonPath reference to an absent input key
4992 # would fail the whole convergence pipeline. The handler's
4993 # defaults match the values it historically hardcoded.
4994 ga_config = self.config.get_global_accelerator_config()
4995 payload["GaHealthCheckPath"] = ga_config.get("health_check_path", "/api/v1/health")
4996 payload["GaHealthCheckInterval"] = ga_config.get("health_check_interval", 30)
4997 payload["GaHealthCheckThreshold"] = ga_config.get("health_check_threshold", 3)
4999 task = sfn_tasks.LambdaInvoke(
5000 self,
5001 "PublishGatewayEndpoint",
5002 lambda_function=self.ga_registration_lambda,
5003 payload=sfn.TaskInput.from_object(payload),
5004 payload_response_only=True,
5005 result_path="$.endpointPublication",
5006 task_timeout=sfn.Timeout.duration(Duration.minutes(16)),
5007 )
5008 task.add_retry(
5009 errors=["States.ALL"],
5010 max_attempts=3,
5011 interval=Duration.seconds(30),
5012 backoff_rate=2.0,
5013 max_delay=Duration.minutes(3),
5014 )
5015 return task
5017 chart_tasks = [_chart_task(name) for name in chart_order]
5019 # The state machine owns the full convergence pipeline:
5020 # base apply -> Helm charts -> post-Helm apply -> exhaustive raw
5021 # manifest validation -> exhaustive Helm/rendered-object validation
5022 # -> unconditional Gateway endpoint publication, with optional Global
5023 # Accelerator registration.
5024 #
5025 # Individual chart failures still continue so every release gets an
5026 # install attempt and diagnostic. The terminal validators then make the
5027 # overall execution fail unless every expected object and release is
5028 # present and ready. Post-Helm apply, both validators, and endpoint
5029 # publication deliberately have no catch-to-success path: topology may
5030 # trust only an exact SUCCEEDED execution for the current deployment
5031 # token.
5032 done = sfn.Succeed(self, "HelmInstallComplete")
5033 base_apply = _kubectl_task("ApplyBaseManifests", post_helm=False)
5034 post_apply = _kubectl_task("ApplyPostHelmManifests", post_helm=True)
5035 manifest_validation = _manifest_validation_task()
5036 helm_validation = _helm_validation_task()
5038 endpoint_publication = _endpoint_publication_task()
5039 post_apply.next(manifest_validation)
5040 manifest_validation.next(helm_validation)
5041 helm_validation.next(endpoint_publication)
5042 endpoint_publication.next(done)
5044 if chart_tasks:
5045 for i, task in enumerate(chart_tasks):
5046 next_state: sfn.IChainable = (
5047 chart_tasks[i + 1] if i + 1 < len(chart_tasks) else post_apply
5048 )
5049 task.add_catch(
5050 next_state,
5051 errors=["States.ALL"],
5052 result_path="$.lastChartError",
5053 )
5054 task.next(next_state)
5055 base_apply.next(chart_tasks[0])
5056 else: # pragma: no cover - charts.yaml is always present in the repo
5057 base_apply.next(post_apply)
5059 # base apply has NO catch on purpose: a persistent base-manifest failure
5060 # fails the execution rather than converging onto an incomplete base.
5061 start_state: sfn.IChainable = base_apply
5063 helm_sm_log_group = logs.LogGroup(
5064 self,
5065 "HelmInstallStateMachineLogGroup",
5066 retention=logs.RetentionDays.ONE_WEEK,
5067 removal_policy=RemovalPolicy.DESTROY,
5068 )
5070 self.helm_install_state_machine = sfn.StateMachine(
5071 self,
5072 "HelmInstallStateMachine",
5073 definition_body=sfn.DefinitionBody.from_chainable(start_state),
5074 state_machine_type=sfn.StateMachineType.STANDARD,
5075 timeout=Duration.hours(2),
5076 tracing_enabled=True,
5077 logs=sfn.LogOptions(destination=helm_sm_log_group, level=sfn.LogLevel.ALL),
5078 )
5080 # Thin fire-and-forget provider: onEvent starts the execution and
5081 # returns immediately. It does no Helm/Kubernetes work, so it never
5082 # approaches the Lambda timeout — all the heavy lifting lives in the
5083 # state machine, which converges charts in the background.
5084 helm_orchestrator_on_event = lambda_.Function(
5085 self,
5086 "HelmOrchestratorOnEvent",
5087 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
5088 handler="handler.on_event",
5089 code=lambda_.Code.from_asset("lambda/helm-orchestrator"),
5090 timeout=Duration.minutes(1),
5091 memory_size=256,
5092 environment={
5093 "STATE_MACHINE_ARN": self.helm_install_state_machine.state_machine_arn,
5094 },
5095 tracing=lambda_.Tracing.ACTIVE,
5096 )
5097 self.helm_install_state_machine.grant_start_execution(helm_orchestrator_on_event)
5098 self.helm_install_state_machine.grant_execution(
5099 helm_orchestrator_on_event,
5100 "states:StopExecution",
5101 "states:DescribeExecution",
5102 )
5104 # Let on_event persist the execution input to SSM so the add-on install
5105 # can be replayed out-of-band (gco stacks addons install) without the
5106 # CLI reconstructing chart config or the KEDA role ARN.
5107 helm_orchestrator_on_event.add_to_role_policy(
5108 iam.PolicyStatement(
5109 actions=[
5110 "ssm:PutParameter",
5111 "ssm:GetParameter",
5112 "ssm:DeleteParameter",
5113 ],
5114 resources=[
5115 f"arn:{self.partition}:ssm:{self.deployment_region}:{self.account}:"
5116 f"parameter/{project_name}/addons/*"
5117 ],
5118 )
5119 )
5121 # HelmInstallCharts depends on this explicit bounded-retention group,
5122 # forcing the trigger's final provider invocation to finish before
5123 # CloudFormation removes the group.
5124 self.helm_installer_provider = cr.Provider(
5125 self,
5126 "HelmInstallerProvider",
5127 on_event_handler=helm_orchestrator_on_event,
5128 log_group=self.helm_installer_provider_log_group,
5129 )
5131 # Unlike create/update convergence, stack deletion must be synchronous:
5132 # Helm releases can own admission webhooks, load balancers, and CRs that
5133 # have to disappear while the Kubernetes API and installer AccessEntry
5134 # still exist. A delete-only provider waits on a reverse-order state
5135 # machine and fails CloudFormation if any real uninstall fails.
5136 self._create_helm_teardown(chart_order)
5138 # cdk-nag suppressions for the install path.
5139 from gco.stacks.nag_suppressions import acknowledge_nag_findings
5141 acknowledge_nag_findings(
5142 helm_lambda_role,
5143 [
5144 {
5145 "id": "AwsSolutions-IAM5",
5146 "reason": (
5147 "The Helm installer Lambda requires broad EKS and Kubernetes API "
5148 "access to install Helm charts (KEDA, NVIDIA DRA, etc.) that create "
5149 "CRDs, RBAC rules, and workloads across multiple namespaces. "
5150 "Resource: * is required because the set of Kubernetes resources "
5151 "is dynamic and not known at synth time."
5152 ),
5153 "appliesTo": ["Resource::*"],
5154 },
5155 ],
5156 )
5157 # The state machine role (auto-generated) invokes the worker Lambda
5158 # across versions using the AWS-standard ``:*`` qualifier that cannot be
5159 # enumerated at synth time.
5160 acknowledge_nag_findings(
5161 self.helm_install_state_machine,
5162 [
5163 {
5164 "id": "AwsSolutions-IAM5",
5165 "reason": (
5166 "The state machine invokes the helm worker Lambda; CDK grants "
5167 "lambda:InvokeFunction with the :* version qualifier, which is the "
5168 "standard form and cannot be narrowed at synth time."
5169 ),
5170 "appliesTo": ["Resource::*"],
5171 },
5172 ],
5173 )
5174 acknowledge_nag_findings(
5175 helm_orchestrator_on_event,
5176 [
5177 {
5178 "id": "AwsSolutions-IAM5",
5179 "reason": (
5180 "Lambda active tracing requires X-Ray write APIs against "
5181 "Resource::*, as X-Ray does not expose resource-level "
5182 "permissions for these telemetry calls."
5183 ),
5184 "appliesTo": ["Resource::*"],
5185 },
5186 {
5187 "id": "AwsSolutions-IAM5",
5188 "reason": (
5189 "If execution metadata persistence fails, the orchestrator must "
5190 "stop the just-started convergence execution before CloudFormation "
5191 "can roll back. This grant is limited to executions of the single "
5192 "regional Helm install state machine."
5193 ),
5194 "appliesTo": [
5195 "Resource::arn:<AWS::Partition>:states:<AWS::Region>:"
5196 "<AWS::AccountId>:execution:"
5197 '{"Fn::Select":[6,{"Fn::Split":[":",'
5198 '{"Ref":"HelmInstallStateMachine7DB71CDC"}]}]}:*'
5199 ],
5200 },
5201 ],
5202 )
5204 # The cr.Provider framework auto-generates a framework-onEvent Lambda and
5205 # its role (and, were an is_complete_handler set, a waiter state machine —
5206 # which this fire-and-forget provider does NOT create). None of these are
5207 # configurable by us: the framework role invokes our handler Lambda via
5208 # the standard ``<lambda-arn>:*`` version qualifier that cannot be narrowed
5209 # at synth time. Suppress the relevant rules across the whole provider
5210 # subtree; appliesTo is omitted because the findings are granted on
5211 # CDK-managed resources we do not author. The SF1/SF2/X-Ray entries are
5212 # retained defensively to cover any helper state machine the framework may
5213 # emit across CDK versions; they are harmless no-ops when none exists.
5214 acknowledge_nag_findings(
5215 self.helm_installer_provider,
5216 [
5217 {
5218 "id": "AwsSolutions-IAM5",
5219 "reason": (
5220 "CDK custom-resource provider framework roles invoke the "
5221 "orchestrator Lambdas via the standard '<lambda-arn>:*' version "
5222 "qualifier, which cannot be enumerated at synth time."
5223 ),
5224 "appliesTo": [
5225 "Resource::<HelmOrchestratorOnEventD0D51D9B.Arn>:*",
5226 ],
5227 },
5228 {
5229 "id": "AwsSolutions-SF1",
5230 "reason": (
5231 "The waiter state machine is auto-generated by the CDK "
5232 "cr.Provider framework and does not expose log configuration; "
5233 "ALL-event logging cannot be enabled on it."
5234 ),
5235 },
5236 {
5237 "id": "AwsSolutions-SF2",
5238 "reason": (
5239 "The waiter state machine is auto-generated by the CDK "
5240 "cr.Provider framework and does not expose tracing "
5241 "configuration; X-Ray cannot be enabled on it."
5242 ),
5243 },
5244 {
5245 "id": "Serverless-StepFunctionStateMachineXray",
5246 "reason": (
5247 "The waiter state machine is auto-generated by the CDK "
5248 "cr.Provider framework and does not expose tracing "
5249 "configuration; X-Ray cannot be enabled on it."
5250 ),
5251 },
5252 ],
5253 )
5255 def _create_helm_teardown(self, chart_order: list[str]) -> None:
5256 """Create the synchronous, reverse-order Helm stack-delete path.
5258 Create/update remain fire-and-forget through ``HelmInstallCharts``. This
5259 separate custom resource is a no-op for those events, but on Delete it
5260 starts a state machine whose per-chart tasks call ``uninstall_chart`` in
5261 reverse install order. The provider polls to terminal state, so a failed
5262 release blocks deletion before EKS authentication or the API disappears.
5263 """
5265 lbc_chart = "aws-load-balancer-controller"
5266 if not chart_order or chart_order[0] != lbc_chart:
5267 raise RuntimeError(
5268 "aws-load-balancer-controller must be the first Helm chart for safe teardown"
5269 )
5271 provider_code = lambda_.Code.from_asset("lambda/helm-installer")
5272 drain_checker = lambda_.Function(
5273 self,
5274 "HelmTeardownDrainChecker",
5275 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
5276 handler="teardown_provider.drain_install_executions",
5277 code=provider_code,
5278 timeout=Duration.minutes(1),
5279 memory_size=256,
5280 environment={
5281 "INSTALL_STATE_MACHINE_ARN": self.helm_install_state_machine.state_machine_arn,
5282 },
5283 tracing=lambda_.Tracing.ACTIVE,
5284 )
5286 def _uninstall_task(chart_name: str) -> sfn_tasks.LambdaInvoke:
5287 timeout_minutes = 5 if chart_name == lbc_chart else 4 if chart_name == "keda" else 2
5288 task = sfn_tasks.LambdaInvoke(
5289 self,
5290 f"HelmUninstallChart-{chart_name}",
5291 lambda_function=self.helm_installer_lambda,
5292 payload=sfn.TaskInput.from_object(
5293 {
5294 "Action": "uninstall_chart",
5295 "Chart": chart_name,
5296 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
5297 "Region": sfn.JsonPath.string_at("$.Region"),
5298 "EnabledCharts": sfn.JsonPath.list_at("$.EnabledCharts"),
5299 "Charts": sfn.JsonPath.object_at("$.Charts"),
5300 "KedaOperatorRoleArn": sfn.JsonPath.string_at("$.KedaOperatorRoleArn"),
5301 }
5302 ),
5303 payload_response_only=True,
5304 result_path="$.lastChart",
5305 task_timeout=sfn.Timeout.duration(Duration.minutes(timeout_minutes)),
5306 )
5307 return task
5309 def _drain_check_task() -> sfn_tasks.LambdaInvoke:
5310 return sfn_tasks.LambdaInvoke(
5311 self,
5312 "CheckRunningConvergence",
5313 lambda_function=drain_checker,
5314 payload=sfn.TaskInput.from_object({}),
5315 payload_response_only=True,
5316 result_path="$.drainCheck",
5317 task_timeout=sfn.Timeout.duration(Duration.minutes(1)),
5318 )
5320 def _quiesce_task() -> sfn_tasks.LambdaInvoke:
5321 task = sfn_tasks.LambdaInvoke(
5322 self,
5323 "QuiesceHealthMonitor",
5324 lambda_function=self.helm_installer_lambda,
5325 payload=sfn.TaskInput.from_object(
5326 {
5327 "Action": "quiesce_health_monitor",
5328 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
5329 "Region": sfn.JsonPath.string_at("$.Region"),
5330 }
5331 ),
5332 payload_response_only=True,
5333 result_path="$.healthMonitorQuiesce",
5334 task_timeout=sfn.Timeout.duration(Duration.minutes(3)),
5335 )
5336 return task
5338 def _endpoint_cleanup_task() -> sfn_tasks.LambdaInvoke:
5339 """Fence SSM/GA publication after all endpoint writers are quiesced."""
5340 payload: dict[str, Any] = {
5341 "Action": "cleanup_gateway_endpoint",
5342 "Region": sfn.JsonPath.string_at("$.Region"),
5343 "RegistryRegion": sfn.JsonPath.string_at("$.RegistryRegion"),
5344 "ProjectName": sfn.JsonPath.string_at("$.ProjectName"),
5345 }
5346 if self.global_accelerator_enabled:
5347 payload["EndpointGroupArn"] = sfn.JsonPath.string_at("$.EndpointGroupArn")
5348 return sfn_tasks.LambdaInvoke(
5349 self,
5350 "CleanupGatewayEndpoint",
5351 lambda_function=self.ga_registration_lambda,
5352 payload=sfn.TaskInput.from_object(payload),
5353 payload_response_only=True,
5354 result_path="$.endpointCleanup",
5355 task_timeout=sfn.Timeout.duration(Duration.minutes(15)),
5356 )
5358 def _gateway_cleanup_task() -> sfn_tasks.LambdaInvoke:
5359 return sfn_tasks.LambdaInvoke(
5360 self,
5361 "DeleteGatewayResources",
5362 lambda_function=self.kubectl_lambda,
5363 payload=sfn.TaskInput.from_object(
5364 {
5365 "Action": "delete_gateway_resources",
5366 "ClusterName": sfn.JsonPath.string_at("$.ClusterName"),
5367 "Region": sfn.JsonPath.string_at("$.Region"),
5368 }
5369 ),
5370 payload_response_only=True,
5371 result_path="$.gatewayCleanup",
5372 task_timeout=sfn.Timeout.duration(Duration.minutes(5)),
5373 )
5375 non_lbc_tasks = [
5376 _uninstall_task(name) for name in reversed(chart_order) if name != lbc_chart
5377 ]
5378 endpoint_cleanup = _endpoint_cleanup_task()
5379 pre_gateway_cleanup = sfn.Parallel(
5380 self,
5381 "CleanupEndpointAndCharts",
5382 result_path="$.preGatewayCleanup",
5383 )
5384 pre_gateway_cleanup.branch(endpoint_cleanup)
5385 if non_lbc_tasks:
5386 for index, task in enumerate(non_lbc_tasks[:-1]):
5387 task.next(non_lbc_tasks[index + 1])
5388 pre_gateway_cleanup.branch(non_lbc_tasks[0])
5390 lbc_uninstall = _uninstall_task(lbc_chart)
5391 gateway_cleanup = _gateway_cleanup_task()
5392 done = sfn.Succeed(self, "HelmTeardownComplete")
5393 quiesce = _quiesce_task()
5395 quiesce.next(pre_gateway_cleanup)
5396 pre_gateway_cleanup.next(gateway_cleanup)
5397 gateway_cleanup.next(lbc_uninstall)
5398 lbc_uninstall.next(done)
5400 # StopExecution cannot cancel a Lambda invocation already in flight,
5401 # and ListExecutions is eventually consistent. The provider stops the
5402 # initially visible executions before this unconditional 16-minute
5403 # drain. The checker then stops any late-visible work and loops through
5404 # another complete drain interval before quiescence. The SSM teardown
5405 # fence blocks supported convergence entrypoints from creating new work.
5406 drain_in_flight = sfn.Wait(
5407 self,
5408 "DrainInFlightConvergence",
5409 time=sfn.WaitTime.seconds_path("$.WaitForInFlightSeconds"),
5410 )
5411 drain_check = _drain_check_task()
5412 late_work = sfn.Choice(self, "LateConvergenceFound")
5413 drain_in_flight.next(drain_check)
5414 drain_check.next(late_work)
5415 late_work.when(
5416 sfn.Condition.number_greater_than("$.drainCheck.StoppedExecutions", 0),
5417 drain_in_flight,
5418 ).otherwise(quiesce)
5419 start_state: sfn.IChainable = drain_in_flight
5421 teardown_log_group = logs.LogGroup(
5422 self,
5423 "HelmTeardownStateMachineLogGroup",
5424 retention=logs.RetentionDays.ONE_WEEK,
5425 removal_policy=RemovalPolicy.DESTROY,
5426 )
5427 self.helm_teardown_state_machine = sfn.StateMachine(
5428 self,
5429 "HelmTeardownStateMachine",
5430 definition_body=sfn.DefinitionBody.from_chainable(start_state),
5431 state_machine_type=sfn.StateMachineType.STANDARD,
5432 # 16m drain + 3m quiesce + max(15m endpoint cleanup, 24m ordinary
5433 # chart cleanup) + 5m Gateway deletion + 5m LBC uninstall = 53m.
5434 # Three minutes of workflow margin leave another three minutes for
5435 # the provider's final poll inside CloudFormation's one-hour ceiling.
5436 timeout=Duration.minutes(56),
5437 tracing_enabled=True,
5438 logs=sfn.LogOptions(destination=teardown_log_group, level=sfn.LogLevel.ALL),
5439 )
5441 teardown_on_event = lambda_.Function(
5442 self,
5443 "HelmTeardownOnEvent",
5444 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
5445 handler="teardown_provider.on_event",
5446 code=provider_code,
5447 timeout=Duration.minutes(1),
5448 memory_size=256,
5449 environment={
5450 "TEARDOWN_STATE_MACHINE_ARN": self.helm_teardown_state_machine.state_machine_arn,
5451 "INSTALL_STATE_MACHINE_ARN": self.helm_install_state_machine.state_machine_arn,
5452 },
5453 tracing=lambda_.Tracing.ACTIVE,
5454 )
5455 teardown_is_complete = lambda_.Function(
5456 self,
5457 "HelmTeardownIsComplete",
5458 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
5459 handler="teardown_provider.is_complete",
5460 code=provider_code,
5461 timeout=Duration.minutes(1),
5462 memory_size=256,
5463 environment={
5464 "TEARDOWN_STATE_MACHINE_ARN": self.helm_teardown_state_machine.state_machine_arn,
5465 "INSTALL_STATE_MACHINE_ARN": self.helm_install_state_machine.state_machine_arn,
5466 },
5467 tracing=lambda_.Tracing.ACTIVE,
5468 )
5469 self.helm_teardown_state_machine.grant_start_execution(teardown_on_event)
5470 self.helm_teardown_state_machine.grant_read(teardown_is_complete)
5471 install_execution_detail = (
5472 "Resource::arn:<AWS::Partition>:states:<AWS::Region>:<AWS::AccountId>:execution:"
5473 '{"Fn::Select":[6,{"Fn::Split":[":",'
5474 '{"Ref":"HelmInstallStateMachine7DB71CDC"}]}]}:*'
5475 )
5476 teardown_execution_detail = (
5477 "Resource::arn:<AWS::Partition>:states:<AWS::Region>:<AWS::AccountId>:execution:"
5478 '{"Fn::Select":[6,{"Fn::Split":[":",'
5479 '{"Ref":"HelmTeardownStateMachine1C15895F"}]}]}:*'
5480 )
5481 for handler in (teardown_on_event, drain_checker):
5482 self.helm_install_state_machine.grant(
5483 handler,
5484 "states:ListExecutions",
5485 )
5486 self.helm_install_state_machine.grant_execution(
5487 handler,
5488 "states:StopExecution",
5489 "states:DescribeExecution",
5490 )
5491 teardown_on_event.add_to_role_policy(
5492 iam.PolicyStatement(
5493 actions=["ssm:PutParameter"],
5494 resources=[
5495 f"arn:{self.partition}:ssm:{self.deployment_region}:{self.account}:"
5496 f"parameter/{self.config.get_project_name()}/addons/"
5497 f"{self.deployment_region}/_teardown"
5498 ],
5499 )
5500 )
5502 # Strict live validation preserves this generation until exact
5503 # post-stack cleanup. Ordinary deployments retain DESTROY semantics.
5504 provider_log_group = logs.LogGroup(
5505 self,
5506 "HelmTeardownProviderLogGroup",
5507 retention=logs.RetentionDays.ONE_WEEK,
5508 removal_policy=self.provider_log_group_removal_policy,
5509 )
5510 self.helm_teardown_provider = cr.Provider(
5511 self,
5512 "HelmTeardownProvider",
5513 on_event_handler=teardown_on_event,
5514 is_complete_handler=teardown_is_complete,
5515 query_interval=Duration.seconds(15),
5516 total_timeout=Duration.minutes(59),
5517 log_group=provider_log_group,
5518 )
5519 teardown_properties: dict[str, Any] = {
5520 "ClusterName": self.cluster.cluster_name,
5521 "Region": self.deployment_region,
5522 "RegistryRegion": self.config.get_global_region(),
5523 "ProjectName": self.config.get_project_name(),
5524 "EnabledCharts": self._get_enabled_helm_charts(),
5525 "Charts": self._helm_chart_value_overrides(),
5526 "KedaOperatorRoleArn": self.keda_operator_role.role_arn,
5527 }
5528 if self.endpoint_group_arn is not None:
5529 teardown_properties["EndpointGroupArn"] = self.endpoint_group_arn
5530 self.helm_teardown_resource = CustomResource(
5531 self,
5532 "HelmTeardown",
5533 service_token=self.helm_teardown_provider.service_token,
5534 properties=teardown_properties,
5535 )
5536 self.helm_teardown_resource.node.add_dependency(self.cluster)
5537 self.helm_teardown_resource.node.add_dependency(self.helm_installer_access_entry)
5538 self.helm_teardown_resource.node.add_dependency(self.kubectl_lambda_access_entry)
5539 self.helm_teardown_resource.node.add_dependency(self.helm_install_state_machine)
5540 self.helm_teardown_resource.node.add_dependency(self.aws_load_balancer_controller_policy)
5541 self.helm_teardown_resource.node.add_dependency(provider_log_group)
5543 from gco.stacks.nag_suppressions import acknowledge_nag_findings
5545 acknowledge_nag_findings(
5546 self.helm_teardown_state_machine,
5547 [
5548 {
5549 "id": "AwsSolutions-IAM5",
5550 "reason": (
5551 "The teardown state machine's X-Ray integration requires "
5552 "Resource::*, and its Lambda task grants use CDK's required :* "
5553 "version qualifier. The drain-checker detail names the single "
5554 "dedicated function created by this stack."
5555 ),
5556 "appliesTo": [
5557 "Resource::*",
5558 "Resource::<HelmTeardownDrainCheckerCCF8D9D1.Arn>:*",
5559 ],
5560 },
5561 ],
5562 )
5563 acknowledge_nag_findings(
5564 drain_checker,
5565 [
5566 {
5567 "id": "AwsSolutions-IAM5",
5568 "reason": (
5569 "The drain checker uses X-Ray Resource::* APIs and stops only "
5570 "runtime-generated executions of the single regional Helm install "
5571 "state machine before Kubernetes teardown."
5572 ),
5573 "appliesTo": ["Resource::*", install_execution_detail],
5574 }
5575 ],
5576 )
5577 for handler in (teardown_on_event, teardown_is_complete):
5578 acknowledge_nag_findings(
5579 handler,
5580 [
5581 {
5582 "id": "AwsSolutions-IAM5",
5583 "reason": (
5584 "X-Ray write APIs require Resource::*. StopExecution and "
5585 "DescribeExecution are otherwise limited to runtime-generated "
5586 "execution ARNs belonging to the two regional Helm state machines."
5587 ),
5588 "appliesTo": [
5589 "Resource::*",
5590 install_execution_detail,
5591 teardown_execution_detail,
5592 ],
5593 }
5594 ],
5595 )
5596 acknowledge_nag_findings(
5597 self.helm_teardown_provider,
5598 [
5599 {
5600 "id": "AwsSolutions-IAM5",
5601 "reason": (
5602 "The CDK provider framework invokes only versioned onEvent/isComplete "
5603 "handlers and its generated waiter invokes only its versioned timeout "
5604 "and completion handlers; every wildcard is a Lambda qualifier."
5605 ),
5606 "appliesTo": [
5607 "Resource::<HelmTeardownIsComplete5ECB4605.Arn>:*",
5608 "Resource::<HelmTeardownOnEvent3DB6F756.Arn>:*",
5609 ("Resource::<HelmTeardownProviderframeworkisComplete3D7339F4.Arn>:*"),
5610 ("Resource::<HelmTeardownProviderframeworkonTimeout3415E5E9.Arn>:*"),
5611 ],
5612 },
5613 {
5614 "id": "AwsSolutions-SF1",
5615 "reason": (
5616 "The provider waiter state machine is generated by CDK and does not "
5617 "expose logging configuration."
5618 ),
5619 },
5620 {
5621 "id": "AwsSolutions-SF2",
5622 "reason": (
5623 "The provider waiter state machine is generated by CDK and does not "
5624 "expose tracing configuration."
5625 ),
5626 },
5627 {
5628 "id": "Serverless-StepFunctionStateMachineXray",
5629 "reason": (
5630 "The provider waiter state machine is generated by CDK and does not "
5631 "expose tracing configuration."
5632 ),
5633 },
5634 ],
5635 )
5637 def _create_efs(self) -> None:
5638 """Create EFS file system for shared storage across jobs.
5640 Creates an EFS file system with mount targets in each private subnet,
5641 allowing pods to share data and persist outputs. The EFS is configured
5642 with:
5643 - Encryption at rest
5644 - Automatic backups (disabled only for disposable live validation)
5645 - General Purpose performance mode (suitable for most workloads)
5646 - Bursting throughput mode
5648 Kubernetes resources (StorageClass, PV, PVC) are created via manifests.
5649 """
5650 project_name = self.config.get_project_name()
5652 # Create security group for EFS
5653 self.efs_security_group = ec2.SecurityGroup(
5654 self,
5655 "EfsSecurityGroup",
5656 vpc=self.vpc,
5657 description=f"Security group for {project_name} EFS in {self.deployment_region}",
5658 security_group_name=f"{project_name}-efs-sg-{self.deployment_region}",
5659 allow_all_outbound=False, # EFS doesn't need outbound
5660 )
5662 # Allow NFS traffic from EKS cluster security group
5663 self.efs_security_group.add_ingress_rule(
5664 peer=self.cluster.cluster_security_group,
5665 connection=ec2.Port.tcp(2049),
5666 description="Allow NFS from EKS cluster",
5667 )
5669 # Create EFS file system
5670 self.efs_file_system = efs.FileSystem(
5671 self,
5672 "GCOEfs",
5673 vpc=self.vpc,
5674 file_system_name=f"{project_name}-efs-{self.deployment_region}",
5675 security_group=self.efs_security_group,
5676 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
5677 encrypted=True,
5678 performance_mode=efs.PerformanceMode.GENERAL_PURPOSE,
5679 throughput_mode=efs.ThroughputMode.BURSTING,
5680 removal_policy=RemovalPolicy.DESTROY, # For dev/test; use RETAIN for production
5681 enable_automatic_backups=not self.disable_efs_automatic_backups,
5682 )
5683 if self.disable_efs_automatic_backups:
5684 cfn_file_system = self.efs_file_system.node.default_child
5685 if not isinstance(cfn_file_system, efs.CfnFileSystem):
5686 raise TypeError("GCOEfs default child must be AWS::EFS::FileSystem")
5687 cfn_file_system.backup_policy = efs.CfnFileSystem.BackupPolicyProperty(
5688 status="DISABLED"
5689 )
5691 # Add file system policy to allow mounting without IAM authorization
5692 # This allows any client that can reach the mount target to mount the file system
5693 self.efs_file_system.add_to_resource_policy(
5694 iam.PolicyStatement(
5695 effect=iam.Effect.ALLOW,
5696 principals=[iam.AnyPrincipal()],
5697 actions=[
5698 "elasticfilesystem:ClientMount",
5699 "elasticfilesystem:ClientWrite",
5700 "elasticfilesystem:ClientRootAccess",
5701 ],
5702 conditions={"Bool": {"elasticfilesystem:AccessedViaMountTarget": "true"}},
5703 )
5704 )
5706 # Create access point for the gco-jobs directory
5707 self.efs_access_point = self.efs_file_system.add_access_point(
5708 "JobsAccessPoint",
5709 path="/gco-jobs",
5710 create_acl=efs.Acl(owner_uid="1000", owner_gid="1000", permissions="755"),
5711 posix_user=efs.PosixUser(uid="1000", gid="1000"),
5712 )
5714 # Output EFS information
5715 CfnOutput(
5716 self,
5717 "EfsFileSystemId",
5718 value=self.efs_file_system.file_system_id,
5719 description="EFS File System ID for shared job storage",
5720 )
5722 CfnOutput(
5723 self,
5724 "EfsAccessPointId",
5725 value=self.efs_access_point.access_point_id,
5726 description="EFS Access Point ID for job outputs",
5727 )
5729 def _create_fsx_lustre(self) -> None:
5730 """Create FSx for Lustre file system for high-performance storage.
5732 FSx for Lustre provides high-performance parallel file system storage
5733 ideal for ML training workloads that require high throughput and low latency.
5735 This is optional and controlled by the fsx_lustre.enabled config setting.
5737 Supported deployment types:
5738 - SCRATCH_1: Temporary storage, no data replication
5739 - SCRATCH_2: Temporary storage with better burst performance
5740 - PERSISTENT_1: Persistent storage with data replication
5741 - PERSISTENT_2: Latest persistent storage with higher throughput
5742 """
5743 fsx_config = self.config.get_fsx_lustre_config(self.deployment_region)
5745 if not fsx_config.get("enabled", False):
5746 self.fsx_file_system = None
5747 return
5749 project_name = self.config.get_project_name()
5751 # Create security group for FSx
5752 self.fsx_security_group = ec2.SecurityGroup(
5753 self,
5754 "FsxSecurityGroup",
5755 vpc=self.vpc,
5756 description=f"Security group for {project_name} FSx Lustre in {self.deployment_region}",
5757 security_group_name=f"{project_name}-fsx-sg-{self.deployment_region}",
5758 allow_all_outbound=False,
5759 )
5761 # Allow Lustre traffic from EKS cluster security group
5762 # Lustre uses ports 988 (control) and 1021-1023 (data)
5763 self.fsx_security_group.add_ingress_rule(
5764 peer=self.cluster.cluster_security_group,
5765 connection=ec2.Port.tcp(988),
5766 description="Allow Lustre control traffic from EKS cluster",
5767 )
5768 self.fsx_security_group.add_ingress_rule(
5769 peer=self.cluster.cluster_security_group,
5770 connection=ec2.Port.tcp_range(1021, 1023),
5771 description="Allow Lustre data traffic from EKS cluster",
5772 )
5774 # Allow self-referencing traffic for FSx Lustre internal communication
5775 # FSx Lustre nodes need to communicate with each other on port 988
5776 self.fsx_security_group.add_ingress_rule(
5777 peer=self.fsx_security_group,
5778 connection=ec2.Port.tcp(988),
5779 description="Allow Lustre internal traffic on port 988",
5780 )
5781 self.fsx_security_group.add_ingress_rule(
5782 peer=self.fsx_security_group,
5783 connection=ec2.Port.tcp_range(1021, 1023),
5784 description="Allow Lustre internal traffic on ports 1021-1023",
5785 )
5787 # Get deployment type
5788 deployment_type = fsx_config.get("deployment_type", "SCRATCH_2")
5789 storage_capacity = fsx_config.get("storage_capacity_gib", 1200)
5791 # Build Lustre configuration based on deployment type
5792 lustre_config = {
5793 "deploymentType": deployment_type,
5794 "dataCompressionType": fsx_config.get("data_compression_type", "LZ4"),
5795 }
5797 # Add throughput for PERSISTENT types
5798 if deployment_type.startswith("PERSISTENT"):
5799 lustre_config["perUnitStorageThroughput"] = fsx_config.get(
5800 "per_unit_storage_throughput", 200
5801 )
5803 # Add S3 import/export if configured
5804 import_path = fsx_config.get("import_path")
5805 export_path = fsx_config.get("export_path")
5807 if import_path:
5808 lustre_config["importPath"] = import_path
5809 lustre_config["autoImportPolicy"] = fsx_config.get(
5810 "auto_import_policy", "NEW_CHANGED_DELETED"
5811 )
5813 if export_path:
5814 lustre_config["exportPath"] = export_path
5816 # Get file system type version (default to 2.15 for kernel 6.x compatibility)
5817 # IMPORTANT: Lustre 2.10 is NOT compatible with kernel 6.x (AL2023, Bottlerocket 1.19+)
5818 # See: https://docs.aws.amazon.com/fsx/latest/LustreGuide/lustre-client-matrix.html
5819 file_system_type_version = fsx_config.get("file_system_type_version", "2.15")
5821 # Create FSx for Lustre file system
5822 self.fsx_file_system = fsx.CfnFileSystem(
5823 self,
5824 "GCOFsxLustre",
5825 file_system_type="LUSTRE",
5826 file_system_type_version=file_system_type_version,
5827 storage_capacity=storage_capacity,
5828 subnet_ids=[self.vpc.private_subnets[0].subnet_id],
5829 security_group_ids=[self.fsx_security_group.security_group_id],
5830 lustre_configuration=lustre_config,
5831 tags=[
5832 {"key": "Name", "value": f"{project_name}-fsx-{self.deployment_region}"},
5833 {"key": "Project", "value": project_name},
5834 ],
5835 )
5837 # Ensure FSx file system waits for security group ingress rules to be created
5838 # This prevents "security group does not permit Lustre LNET traffic" errors
5839 self.fsx_file_system.node.add_dependency(self.fsx_security_group)
5841 # Create FSx CSI Driver add-on for Kubernetes integration
5842 self._create_fsx_csi_driver_addon()
5844 # Output FSx information
5845 CfnOutput(
5846 self,
5847 "FsxFileSystemId",
5848 value=self.fsx_file_system.ref,
5849 description="FSx for Lustre File System ID",
5850 )
5852 CfnOutput(
5853 self,
5854 "FsxDnsName",
5855 value=self.fsx_file_system.attr_dns_name,
5856 description="FSx for Lustre DNS Name",
5857 )
5859 CfnOutput(
5860 self,
5861 "FsxMountName",
5862 value=self.fsx_file_system.attr_lustre_mount_name,
5863 description="FSx for Lustre Mount Name",
5864 )
5866 def _create_valkey_cache(self) -> None:
5867 """Create an ElastiCache Serverless Valkey cache for K/V caching.
5869 Provides a low-latency key-value store that inference endpoints and
5870 jobs can use for prompt caching, session state, feature stores, or
5871 any shared state across pods. Valkey Serverless auto-scales and
5872 requires no node management.
5874 The cache is placed in the VPC private subnets and accessible from
5875 any pod via the cluster security group.
5876 """
5877 valkey_config = self.config.get_valkey_config()
5878 if not valkey_config.get("enabled", False):
5879 return
5881 from aws_cdk import aws_elasticache as elasticache
5883 # Security group for Valkey (allow access from EKS cluster)
5884 valkey_sg = ec2.SecurityGroup(
5885 self,
5886 "ValkeySG",
5887 vpc=self.vpc,
5888 description="Security group for Valkey Serverless cache",
5889 allow_all_outbound=False,
5890 )
5891 valkey_sg.add_ingress_rule(
5892 ec2.Peer.ipv4(self.vpc.vpc_cidr_block),
5893 ec2.Port.tcp(6379),
5894 "Allow Valkey access from VPC",
5895 )
5897 # The Valkey SG ingress allows 6379 from the VPC CIDR (an ``Fn::GetAtt``
5898 # token cdk-nag can't resolve), so the SG-ingress rules throw. Scope the
5899 # acknowledgment to the Valkey SG construct itself.
5900 from gco.stacks.nag_suppressions import acknowledge_security_group_cidr_findings
5902 acknowledge_security_group_cidr_findings(
5903 valkey_sg,
5904 reason=(
5905 "The Valkey Serverless cache security group allows the Valkey "
5906 "port (6379) from the VPC CIDR only, referenced via an "
5907 "``Fn::GetAtt`` token that cdk-nag cannot resolve at synth "
5908 "time. Ingress is restricted to intra-VPC traffic from the "
5909 "job pods that use the cache."
5910 ),
5911 )
5913 # ElastiCache Serverless accepts only 2-3 subnets ("Serverless Cache
5914 # should have total subnetIds between 2 and 3" — caught live by the
5915 # example-job validation run ex241-2913b044 in us-east-1, where the
5916 # VPC's span-every-AZ layout yields six private subnets). CDK orders
5917 # ``vpc.private_subnets`` deterministically by AZ, so taking the
5918 # first three keeps the selection stable across deploys; the cache
5919 # is reachable from every subnet regardless (routing, not placement).
5920 private_subnet_ids = [s.subnet_id for s in self.vpc.private_subnets[:3]]
5922 self.valkey_cache = elasticache.CfnServerlessCache(
5923 self,
5924 "ValkeyCache",
5925 engine="valkey",
5926 serverless_cache_name=f"{self.config.get_project_name()}-{self.deployment_region}",
5927 description=f"GCO K/V cache for {self.deployment_region}",
5928 major_engine_version="8",
5929 security_group_ids=[valkey_sg.security_group_id],
5930 subnet_ids=private_subnet_ids,
5931 cache_usage_limits=elasticache.CfnServerlessCache.CacheUsageLimitsProperty(
5932 data_storage=elasticache.CfnServerlessCache.DataStorageProperty(
5933 maximum=valkey_config.get("max_data_storage_gb", 5),
5934 minimum=1,
5935 unit="GB",
5936 ),
5937 ecpu_per_second=elasticache.CfnServerlessCache.ECPUPerSecondProperty(
5938 maximum=valkey_config.get("max_ecpu_per_second", 5000),
5939 minimum=1000,
5940 ),
5941 ),
5942 snapshot_retention_limit=valkey_config.get("snapshot_retention_limit", 1),
5943 tags=[
5944 CfnTag(key="Project", value=self.config.get_project_name()),
5945 CfnTag(key="gco:project", value=self.config.get_project_name()),
5946 CfnTag(key="Region", value=self.deployment_region),
5947 ],
5948 )
5950 CfnOutput(
5951 self,
5952 "ValkeyEndpoint",
5953 value=self.valkey_cache.attr_endpoint_address,
5954 description="Valkey Serverless cache endpoint",
5955 )
5956 CfnOutput(
5957 self,
5958 "ValkeyPort",
5959 value=self.valkey_cache.attr_endpoint_port,
5960 description="Valkey Serverless cache port",
5961 )
5963 # Store endpoint in SSM for discovery by pods
5964 ssm.StringParameter(
5965 self,
5966 "ValkeyEndpointParam",
5967 parameter_name=f"/{self.config.get_project_name()}/valkey-endpoint-{self.deployment_region}",
5968 string_value=self.valkey_cache.attr_endpoint_address,
5969 description=f"Valkey endpoint for {self.deployment_region}",
5970 )
5972 def _create_aurora_pgvector(self) -> None:
5973 """Create an Aurora Serverless v2 PostgreSQL cluster with pgvector.
5975 Provides a fully managed vector database that inference endpoints and
5976 jobs can use for RAG (retrieval-augmented generation), semantic search,
5977 embedding storage, and similarity queries. Aurora Serverless v2
5978 auto-scales capacity and requires no instance management.
5980 The cluster is placed in the VPC private subnets and accessible from
5981 any pod via the cluster security group. Credentials are stored in
5982 Secrets Manager and the endpoint is published to SSM + a K8s ConfigMap
5983 for automatic discovery.
5985 See: https://aws.amazon.com/blogs/database/accelerate-generative-ai-workloads-on-amazon-aurora-with-optimized-reads-and-pgvector/
5986 """
5987 aurora_config = self.config.get_aurora_pgvector_config()
5988 if not aurora_config.get("enabled", False):
5989 return
5991 from aws_cdk import aws_rds as rds
5993 project_name = self.config.get_project_name()
5995 # Security group for Aurora (allow PostgreSQL access from EKS cluster only)
5996 aurora_sg = ec2.SecurityGroup(
5997 self,
5998 "AuroraPgvectorSG",
5999 vpc=self.vpc,
6000 description="Security group for Aurora Serverless v2 pgvector",
6001 allow_all_outbound=False,
6002 )
6003 aurora_sg.add_ingress_rule(
6004 self.cluster.cluster_security_group,
6005 ec2.Port.tcp(5432),
6006 "Allow PostgreSQL access from EKS cluster",
6007 )
6009 # Subnet group for Aurora (private subnets only)
6010 subnet_group = rds.SubnetGroup(
6011 self,
6012 "AuroraPgvectorSubnetGroup",
6013 description=f"Subnet group for GCO Aurora pgvector in {self.deployment_region}",
6014 vpc=self.vpc,
6015 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
6016 )
6018 # RDS creates the exported postgresql log group itself, outside
6019 # CloudFormation, and never deletes it — live example-job validation
6020 # run ex241-edf33111-r2 found it as the only post-teardown residue.
6021 # Pre-creating the group under the exact name RDS uses
6022 # (/aws/rds/cluster/<cluster-identifier>/<export>) hands its whole
6023 # lifecycle to this stack; RDS then writes into the existing group.
6024 aurora_log_group = logs.LogGroup(
6025 self,
6026 "AuroraPgvectorPostgresqlLogs",
6027 log_group_name=(
6028 f"/aws/rds/cluster/{project_name}-pgvector-{self.deployment_region}/postgresql"
6029 ),
6030 retention=logs.RetentionDays.ONE_MONTH,
6031 removal_policy=RemovalPolicy.DESTROY,
6032 )
6034 # Aurora Serverless v2 cluster with PostgreSQL 16 + pgvector
6035 self.aurora_cluster = rds.DatabaseCluster(
6036 self,
6037 "AuroraPgvectorCluster",
6038 engine=rds.DatabaseClusterEngine.aurora_postgres(
6039 # ``of()`` rather than a ``VER_X_Y`` enum member: the pin in
6040 # constants.py is a plain version string so an Aurora minor
6041 # bump never has to wait for an aws-cdk-lib enum release.
6042 version=rds.AuroraPostgresEngineVersion.of(
6043 AURORA_POSTGRES_VERSION,
6044 AURORA_POSTGRES_VERSION.split(".", 1)[0],
6045 ),
6046 ),
6047 serverless_v2_min_capacity=aurora_config.get("min_acu", 0),
6048 serverless_v2_max_capacity=aurora_config.get("max_acu", 16),
6049 writer=rds.ClusterInstance.serverless_v2(
6050 "Writer",
6051 auto_minor_version_upgrade=True,
6052 ),
6053 readers=[
6054 rds.ClusterInstance.serverless_v2(
6055 "Reader",
6056 auto_minor_version_upgrade=True,
6057 scale_with_writer=True,
6058 ),
6059 ],
6060 vpc=self.vpc,
6061 vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
6062 subnet_group=subnet_group,
6063 security_groups=[aurora_sg],
6064 default_database_name="gco_vectors",
6065 backup=rds.BackupProps(
6066 retention=Duration.days(aurora_config.get("backup_retention_days", 7)),
6067 ),
6068 deletion_protection=aurora_config.get("deletion_protection", False),
6069 removal_policy=RemovalPolicy.DESTROY,
6070 storage_encrypted=True,
6071 iam_authentication=True,
6072 cloudwatch_logs_exports=["postgresql"],
6073 monitoring_interval=Duration.seconds(60),
6074 cluster_identifier=f"{project_name}-pgvector-{self.deployment_region}",
6075 )
6076 self.aurora_secret = cast(secretsmanager.ISecret, self.aurora_cluster.secret)
6077 # The group must exist before the cluster starts exporting, and must
6078 # outlive it on delete (reverse order) so late writes cannot recreate
6079 # an unowned group.
6080 self.aurora_cluster.node.add_dependency(aurora_log_group)
6082 # aws-cdk-lib >= 2.262 ships a built-in "CloudFormation Validate"
6083 # pack whose W9008 wants StorageEncrypted on every CfnDBInstance. The
6084 # cluster above sets storage_encrypted=True, and Aurora cluster
6085 # members inherit the cluster's storage encryption — the
6086 # instance-level property is not applicable to Aurora members, so the
6087 # finding cannot be satisfied at the instance. ``Validations.acknowledge``
6088 # is the API that feeds the validation report's suppression pass
6089 # (``Annotations.acknowledge_warning`` only silences the console
6090 # annotation). Note: the current CDK implementation collects these
6091 # acknowledgments app-wide per rule ID, so this quiets W9008
6092 # everywhere — attaching it here records this cluster as the
6093 # provenance in the report, and the five cdk-nag packs' own
6094 # RDS storage-encryption rules remain scoped and would still fail
6095 # a genuinely unencrypted instance elsewhere.
6096 Validations.of(self.aurora_cluster).acknowledge(
6097 Acknowledgment(
6098 id="CloudFormation-Validate::W9008",
6099 reason=(
6100 "Aurora cluster members inherit the cluster's "
6101 "storage_encrypted=True; StorageEncrypted is not applicable "
6102 "on Aurora member DBInstances."
6103 ),
6104 )
6105 )
6107 # E9006 checks EngineVersion against the enum embedded in the CDK's
6108 # bundled CloudFormation resource spec, which lags new Aurora minor
6109 # releases (at 17.10's release the spec listed 17.9 and even 18.3,
6110 # but not 17.10). The pin in constants.py is validated against the
6111 # authoritative source instead: the monthly dependency scan compares
6112 # it with live ``rds describe-db-engine-versions`` output, and the
6113 # live release validation deploys it for real. Same app-wide
6114 # collection caveat as W9008 above; the compensating controls are
6115 # those live checks, which a stale spec enum cannot see.
6116 Validations.of(self.aurora_cluster).acknowledge(
6117 Acknowledgment(
6118 id="CloudFormation-Validate::E9006",
6119 reason=(
6120 "EngineVersion is validated against live RDS (monthly "
6121 "dependency scan + live release validation); the CDK's "
6122 "embedded CloudFormation spec enum lags new Aurora minors."
6123 ),
6124 )
6125 )
6127 # Construct-level cdk-nag suppressions for Aurora pgvector
6128 from gco.stacks.nag_suppressions import NagSuppression, acknowledge_nag_findings
6130 acknowledge_nag_findings(
6131 self.aurora_cluster,
6132 [
6133 NagSuppression(
6134 id="AwsSolutions-RDS10",
6135 reason=(
6136 "Deletion protection is intentionally disabled for dev/test deployments. "
6137 "Production deployments should set aurora_pgvector.deletion_protection=true "
6138 "in cdk.json."
6139 ),
6140 ),
6141 NagSuppression(
6142 id="AwsSolutions-SMG4",
6143 reason=(
6144 "Aurora manages credential rotation via the RDS integration with Secrets "
6145 "Manager. Manual Secrets Manager rotation is not required. "
6146 "See: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/rds-secrets-manager.html"
6147 ),
6148 ),
6149 NagSuppression(
6150 id="HIPAA.Security-RDSInstanceDeletionProtectionEnabled",
6151 reason=(
6152 "Deletion protection is intentionally disabled for dev/test deployments. "
6153 "Production deployments should set aurora_pgvector.deletion_protection=true "
6154 "in cdk.json."
6155 ),
6156 ),
6157 NagSuppression(
6158 id="NIST.800.53.R5-RDSInstanceDeletionProtectionEnabled",
6159 reason=(
6160 "Deletion protection is intentionally disabled for dev/test deployments. "
6161 "Production deployments should set aurora_pgvector.deletion_protection=true "
6162 "in cdk.json."
6163 ),
6164 ),
6165 NagSuppression(
6166 id="PCI.DSS.321-SecretsManagerUsingKMSKey",
6167 reason=(
6168 "Aurora Serverless v2 credentials in Secrets Manager are encrypted with "
6169 "AWS-managed keys by default. Customer-managed KMS can be enabled if "
6170 "required for PCI compliance."
6171 ),
6172 ),
6173 ],
6174 )
6176 # Outputs
6177 CfnOutput(
6178 self,
6179 "AuroraPgvectorEndpoint",
6180 value=self.aurora_cluster.cluster_endpoint.hostname,
6181 description="Aurora pgvector cluster writer endpoint",
6182 )
6183 CfnOutput(
6184 self,
6185 "AuroraPgvectorReaderEndpoint",
6186 value=self.aurora_cluster.cluster_read_endpoint.hostname,
6187 description="Aurora pgvector cluster reader endpoint",
6188 )
6189 CfnOutput(
6190 self,
6191 "AuroraPgvectorPort",
6192 value=str(self.aurora_cluster.cluster_endpoint.port),
6193 description="Aurora pgvector cluster port",
6194 )
6195 CfnOutput(
6196 self,
6197 "AuroraPgvectorSecretArn",
6198 value=self.aurora_secret.secret_arn,
6199 description="Aurora pgvector credentials secret ARN",
6200 )
6202 # Store endpoint in SSM for discovery by pods and external tools
6203 ssm.StringParameter(
6204 self,
6205 "AuroraPgvectorEndpointParam",
6206 parameter_name=f"/{project_name}/aurora-pgvector-endpoint-{self.deployment_region}",
6207 string_value=self.aurora_cluster.cluster_endpoint.hostname,
6208 description=f"Aurora pgvector endpoint for {self.deployment_region}",
6209 )
6211 # Grant the ServiceAccountRole read access to the Aurora secret
6212 # so pods can retrieve credentials via the ConfigMap + Secrets Manager.
6213 self.aurora_secret.grant_read(self.service_account_role)
6215 def _create_fsx_csi_driver_addon(self) -> None:
6216 """Create FSx CSI Driver add-on for Kubernetes integration.
6218 The FSx CSI driver enables Kubernetes pods to mount FSx for Lustre
6219 file systems as persistent volumes.
6220 """
6221 # Create IAM role for FSx CSI Driver using IRSA + Pod Identity
6222 self.fsx_csi_role = GCORegionalStack._create_irsa_role(
6223 self,
6224 "FsxCsiDriverRole",
6225 oidc_provider_arn=self.oidc_provider.open_id_connect_provider_arn,
6226 oidc_issuer_url=self.cluster.cluster_open_id_connect_issuer_url,
6227 service_account_names=["fsx-csi-controller-sa"],
6228 namespaces=["kube-system"],
6229 )
6231 # Add FSx CSI driver permissions
6232 self.fsx_csi_role.add_to_policy(
6233 iam.PolicyStatement(
6234 effect=iam.Effect.ALLOW,
6235 actions=[
6236 "fsx:DescribeFileSystems",
6237 "fsx:DescribeVolumes",
6238 "fsx:CreateVolume",
6239 "fsx:DeleteVolume",
6240 "fsx:TagResource",
6241 ],
6242 resources=["*"],
6243 )
6244 )
6246 self.fsx_csi_role.add_to_policy(
6247 iam.PolicyStatement(
6248 effect=iam.Effect.ALLOW,
6249 actions=[
6250 "ec2:DescribeInstances",
6251 "ec2:DescribeVolumes",
6252 "ec2:DescribeVpcs",
6253 "ec2:DescribeSubnets",
6254 "ec2:DescribeSecurityGroups",
6255 ],
6256 resources=["*"],
6257 )
6258 )
6260 # cdk-nag suppression: the FSx CSI driver role grants
6261 # ec2:Describe* APIs that don't support resource-level scoping.
6262 from gco.stacks.nag_suppressions import acknowledge_nag_findings
6264 acknowledge_nag_findings(
6265 self.fsx_csi_role,
6266 [
6267 {
6268 "id": "AwsSolutions-IAM5",
6269 "reason": (
6270 "The FSx CSI driver role grants ec2:Describe* for volume "
6271 "and network discovery. These AWS APIs do not support "
6272 "resource-level IAM scoping — Resource: * is the only "
6273 "valid form."
6274 ),
6275 "appliesTo": ["Resource::*"],
6276 },
6277 ],
6278 )
6280 # Create FSx CSI Driver add-on
6281 fsx_addon = eks.Addon(
6282 self,
6283 "FsxCsiDriverAddon",
6284 cluster=self.cluster, # type: ignore[arg-type]
6285 addon_name="aws-fsx-csi-driver",
6286 addon_version=EKS_ADDON_FSX_CSI_DRIVER,
6287 preserve_on_delete=False,
6288 configuration_values={
6289 # DaemonSet node agent must run on every node type; the
6290 # Deployment-shaped controller deliberately carries no
6291 # accelerator tolerations (see _ADDON_NODE_TOLERATIONS).
6292 "node": {
6293 "tolerations": self._ADDON_NODE_TOLERATIONS,
6294 },
6295 },
6296 )
6298 # Append the PassRole statement for the FSx CSI role to the shared
6299 # AwsCustomResource execution role. See
6300 # _create_aws_custom_resource_role for the full rationale.
6301 self.aws_custom_resource_role.add_to_policy(
6302 iam.PolicyStatement(
6303 effect=iam.Effect.ALLOW,
6304 actions=["iam:PassRole"],
6305 resources=[self.fsx_csi_role.role_arn],
6306 )
6307 )
6309 # Update the add-on to use the IRSA role
6310 update_fsx_addon = cr.AwsCustomResource(
6311 self,
6312 "UpdateFsxCsiAddonRole",
6313 on_create=cr.AwsSdkCall(
6314 service="EKS",
6315 action="updateAddon",
6316 parameters={
6317 "clusterName": self.cluster.cluster_name,
6318 "addonName": "aws-fsx-csi-driver",
6319 "serviceAccountRoleArn": self.fsx_csi_role.role_arn,
6320 },
6321 physical_resource_id=cr.PhysicalResourceId.of(
6322 f"{self.cluster.cluster_name}-fsx-csi-role-update"
6323 ),
6324 ),
6325 on_update=cr.AwsSdkCall(
6326 service="EKS",
6327 action="updateAddon",
6328 parameters={
6329 "clusterName": self.cluster.cluster_name,
6330 "addonName": "aws-fsx-csi-driver",
6331 "serviceAccountRoleArn": self.fsx_csi_role.role_arn,
6332 },
6333 ),
6334 role=self.aws_custom_resource_role,
6335 )
6337 update_fsx_addon.node.add_dependency(fsx_addon)
6338 update_fsx_addon.node.add_dependency(self.fsx_csi_role)
6339 update_fsx_addon.node.add_dependency(self.aws_custom_resource_role)
6341 # Expose the update-addon resource so _apply_kubernetes_manifests can
6342 # make the kubectl Lambda wait for the IRSA annotation patch to land
6343 # before it rollout-restarts the fsx-csi-controller. See the EFS CSI
6344 # equivalent for the full rationale — same race, same fix, same
6345 # symptom (PVCs stuck Pending with "no EC2 IMDS role found").
6346 self._fsx_csi_addon_role_update = update_fsx_addon
6348 # Create Pod Identity Association for FSx CSI driver
6349 eks_l1.CfnPodIdentityAssociation(
6350 self,
6351 "PodIdentity-fsx-csi",
6352 cluster_name=self.cluster.cluster_name,
6353 namespace="kube-system",
6354 service_account="fsx-csi-controller-sa",
6355 role_arn=self.fsx_csi_role.role_arn,
6356 )
6358 def _create_drift_detection(self) -> None:
6359 """Create CloudFormation drift detection on a daily schedule.
6361 Creates:
6362 - SNS topic (KMS-encrypted) for drift alerts
6363 - Lambda function that initiates drift detection on this stack, polls
6364 until detection completes, and publishes to SNS if drift is found
6365 - EventBridge rule on a daily schedule (configurable via cdk.json
6366 ``drift_detection.schedule_hours``) that invokes the Lambda
6368 Operators can disable drift detection entirely by setting
6369 ``drift_detection.enabled`` to ``false`` in cdk.json. When disabled,
6370 no resources are created.
6371 """
6372 drift_config = self.node.try_get_context("drift_detection") or {}
6373 if not drift_config.get("enabled", True):
6374 return
6376 schedule_hours = int(drift_config.get("schedule_hours", 24))
6378 # KMS key for SNS topic encryption. SNS with AWS-managed keys doesn't
6379 # allow CloudFormation/Lambda to publish, so we use a customer-managed
6380 # key we can grant publish access on.
6381 drift_topic_key = kms.Key(
6382 self,
6383 "DriftDetectionTopicKey",
6384 description="KMS key for GCO drift detection SNS topic",
6385 enable_key_rotation=True,
6386 removal_policy=RemovalPolicy.DESTROY,
6387 )
6389 self.drift_detection_topic = sns.Topic(
6390 self,
6391 "DriftDetectionTopic",
6392 display_name="GCO CloudFormation Drift Alerts",
6393 master_key=drift_topic_key,
6394 )
6396 # IAM role for the drift detection Lambda
6397 drift_lambda_role = iam.Role(
6398 self,
6399 "DriftDetectionLambdaRole",
6400 assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
6401 managed_policies=[
6402 iam.ManagedPolicy.from_aws_managed_policy_name(
6403 "service-role/AWSLambdaBasicExecutionRole"
6404 ),
6405 ],
6406 )
6408 # CloudFormation drift APIs operate at the stack level; the API does
6409 # not support resource-level ARN scoping for these actions, so we scope
6410 # to this stack's ARN where supported and accept "*" where not.
6411 drift_lambda_role.add_to_policy(
6412 iam.PolicyStatement(
6413 effect=iam.Effect.ALLOW,
6414 actions=[
6415 "cloudformation:DetectStackDrift",
6416 "cloudformation:DescribeStackDriftDetectionStatus",
6417 "cloudformation:DescribeStackResourceDrifts",
6418 "cloudformation:DescribeStackResource",
6419 "cloudformation:DescribeStackResources",
6420 ],
6421 resources=["*"],
6422 )
6423 )
6425 self.drift_detection_topic.grant_publish(drift_lambda_role)
6427 # Lambda function — one per stack; stack name is baked into env vars
6428 drift_lambda = lambda_.Function(
6429 self,
6430 "DriftDetectionFunction",
6431 runtime=getattr(lambda_.Runtime, LAMBDA_PYTHON_RUNTIME),
6432 handler="handler.lambda_handler",
6433 code=lambda_.Code.from_asset("lambda/drift-detection"),
6434 timeout=Duration.minutes(14), # Leave headroom under Lambda 15-min cap
6435 memory_size=256,
6436 role=drift_lambda_role,
6437 environment={
6438 "STACK_NAME": self.stack_name,
6439 "SNS_TOPIC_ARN": self.drift_detection_topic.topic_arn,
6440 "REGION": self.deployment_region,
6441 },
6442 tracing=lambda_.Tracing.ACTIVE,
6443 )
6445 # Dead-letter queue for EventBridge → Lambda target failures.
6446 # Captures events that fail to reach the Lambda (e.g. due to
6447 # throttling or permission issues) so operators can retry or
6448 # investigate. Required by Serverless-EventBusDLQ cdk-nag rule.
6449 drift_rule_dlq = sqs.Queue(
6450 self,
6451 "DriftDetectionRuleDlq",
6452 retention_period=Duration.days(14),
6453 enforce_ssl=True,
6454 encryption=sqs.QueueEncryption.SQS_MANAGED,
6455 removal_policy=RemovalPolicy.DESTROY,
6456 )
6458 # DLQs themselves are terminal — they don't need their own DLQ.
6459 # Suppress the circular AwsSolutions-SQS3 nag finding.
6460 from gco.stacks.nag_suppressions import acknowledge_nag_findings
6462 acknowledge_nag_findings(
6463 drift_rule_dlq,
6464 [
6465 {
6466 "id": "AwsSolutions-SQS3",
6467 "reason": (
6468 "This queue IS the dead-letter queue for the "
6469 "DriftDetectionSchedule EventBridge rule. A DLQ for a "
6470 "DLQ is circular; if events fail to reach this queue "
6471 "they are captured by EventBridge's own retry metrics "
6472 "(CloudWatch FailedInvocations)."
6473 ),
6474 },
6475 ],
6476 )
6478 # EventBridge rule — daily schedule by default
6479 events.Rule(
6480 self,
6481 "DriftDetectionSchedule",
6482 description=(f"Daily CloudFormation drift detection for {self.stack_name}"),
6483 schedule=events.Schedule.rate(Duration.hours(schedule_hours)),
6484 targets=[
6485 events_targets.LambdaFunction(
6486 drift_lambda,
6487 dead_letter_queue=drift_rule_dlq,
6488 retry_attempts=2,
6489 )
6490 ],
6491 )
6493 # Outputs for operators to subscribe to the topic
6494 CfnOutput(
6495 self,
6496 "DriftDetectionTopicArn",
6497 value=self.drift_detection_topic.topic_arn,
6498 description=(
6499 f"SNS topic ARN for CloudFormation drift alerts in "
6500 f"{self.deployment_region}. Subscribe an endpoint (email, "
6501 f"Slack, PagerDuty) to receive drift notifications."
6502 ),
6503 )
6505 # cdk-nag suppressions for this component
6506 from gco.stacks.nag_suppressions import acknowledge_nag_findings
6508 acknowledge_nag_findings(
6509 drift_lambda_role,
6510 [
6511 {
6512 "id": "AwsSolutions-IAM4",
6513 "reason": (
6514 "AWSLambdaBasicExecutionRole provides standard "
6515 "CloudWatch Logs permissions required for Lambda "
6516 "logging. This is the AWS-recommended managed policy."
6517 ),
6518 },
6519 {
6520 "id": "AwsSolutions-IAM5",
6521 "reason": (
6522 "CloudFormation drift detection APIs (DetectStackDrift, "
6523 "DescribeStackDriftDetectionStatus, "
6524 "DescribeStackResourceDrifts) cannot be scoped to a "
6525 "specific stack resource via IAM; the action-level "
6526 "scoping requires wildcard resources. The Lambda's "
6527 "environment pins it to a single stack name, so the "
6528 "effective blast radius is limited. The "
6529 "``kms:GenerateDataKey*`` action wildcard is the "
6530 "AWS-recommended grant for publishing to the "
6531 "KMS-encrypted drift-detection SNS topic."
6532 ),
6533 "appliesTo": [
6534 "Resource::*",
6535 "Action::kms:GenerateDataKey*",
6536 ],
6537 },
6538 ],
6539 )
6541 def _create_mcp_role(self) -> None:
6542 """Create dedicated IAM role for the MCP server.
6544 The MCP server exposes GCO CLI tools to LLM agents. Without a dedicated
6545 role, the server would inherit the full ambient credentials of the user
6546 who launches it (often an administrator). This method creates a
6547 least-privilege role that the MCP server can assume at startup via
6548 ``GCO_MCP_ROLE_ARN``.
6550 Permissions are scoped to the minimum needed by the tools exposed:
6552 - ``eks:DescribeCluster`` on this regional EKS cluster ARN only.
6553 - ``s3:GetObject`` on model weights buckets. The model bucket lives in
6554 the global stack, so we scope to the same name pattern used by the
6555 service account role (``{project_name}-*``). This is a deliberate
6556 compromise: a precise cross-stack ARN export would force a tight
6557 dependency on the global stack, and cdk-nag will flag it anyway
6558 because the bucket name is auto-generated.
6559 - ``cloudwatch:GetMetricData`` / ``cloudwatch:ListMetrics``. These APIs
6560 do not support resource-level IAM, so wildcard is required. Read-only.
6561 - ``sqs:SendMessage`` scoped to this region's job queue ARN only.
6563 The trust policy uses ``AccountRootPrincipal`` so any IAM user/role in
6564 the account can assume it (gated by an explicit sts:AssumeRole
6565 permission on the caller — standard AWS behavior). Operators who want
6566 to restrict assumption further should add an external-id or principal
6567 condition to the trust policy after deployment.
6569 Operators can disable this component entirely by setting
6570 ``mcp_server.enabled`` to ``false`` in cdk.json.
6571 """
6572 mcp_config = self.node.try_get_context("mcp_server") or {}
6573 if not mcp_config.get("enabled", True):
6574 return
6576 project_name = self.config.get_project_name()
6578 self.mcp_server_role = iam.Role(
6579 self,
6580 "McpServerRole",
6581 assumed_by=iam.AccountRootPrincipal(),
6582 description=(
6583 "Least-privilege role assumed by the GCO MCP server at startup. "
6584 "Grants only the permissions needed by MCP tools: eks:DescribeCluster, "
6585 "s3:GetObject on model buckets, cloudwatch read-only metrics, and "
6586 "sqs:SendMessage to the regional job queue."
6587 ),
6588 max_session_duration=Duration.hours(12),
6589 )
6591 # eks:DescribeCluster on this region's cluster only
6592 self.mcp_server_role.add_to_policy(
6593 iam.PolicyStatement(
6594 effect=iam.Effect.ALLOW,
6595 actions=["eks:DescribeCluster"],
6596 resources=[self.cluster.cluster_arn],
6597 )
6598 )
6600 # s3:GetObject on model weights buckets. Bucket name is auto-generated
6601 # in the global stack, so we match the same prefix pattern used by the
6602 # service account role.
6603 self.mcp_server_role.add_to_policy(
6604 iam.PolicyStatement(
6605 effect=iam.Effect.ALLOW,
6606 actions=["s3:GetObject", "s3:ListBucket"],
6607 resources=[
6608 f"arn:{self.partition}:s3:::{project_name}-*",
6609 f"arn:{self.partition}:s3:::{project_name}-*/*",
6610 ],
6611 )
6612 )
6614 # CloudWatch read-only metrics APIs. These APIs do not support
6615 # resource-level IAM so wildcard is required.
6616 self.mcp_server_role.add_to_policy(
6617 iam.PolicyStatement(
6618 effect=iam.Effect.ALLOW,
6619 actions=[
6620 "cloudwatch:GetMetricData",
6621 "cloudwatch:GetMetricStatistics",
6622 "cloudwatch:ListMetrics",
6623 ],
6624 resources=["*"],
6625 )
6626 )
6628 # sqs:SendMessage scoped to the regional job queue only
6629 self.mcp_server_role.add_to_policy(
6630 iam.PolicyStatement(
6631 effect=iam.Effect.ALLOW,
6632 actions=["sqs:SendMessage", "sqs:GetQueueUrl", "sqs:GetQueueAttributes"],
6633 resources=[self.job_queue.queue_arn],
6634 )
6635 )
6637 # Export the role ARN so operators can set GCO_MCP_ROLE_ARN in their
6638 # MCP server environment.
6639 CfnOutput(
6640 self,
6641 "McpServerRoleArn",
6642 value=self.mcp_server_role.role_arn,
6643 description=(
6644 "IAM role ARN for the GCO MCP server. Set GCO_MCP_ROLE_ARN to "
6645 "this value when launching the MCP server so it assumes a "
6646 "least-privilege role instead of ambient credentials."
6647 ),
6648 export_name=f"{project_name}-mcp-server-role-arn-{self.deployment_region}",
6649 )
6651 # cdk-nag suppressions: CloudWatch metrics APIs cannot be scoped.
6652 from gco.stacks.nag_suppressions import acknowledge_nag_findings
6654 acknowledge_nag_findings(
6655 self.mcp_server_role,
6656 [
6657 {
6658 "id": "AwsSolutions-IAM5",
6659 "reason": (
6660 "The CloudWatch metrics APIs (GetMetricData, "
6661 "GetMetricStatistics, ListMetrics) do not support "
6662 "resource-level IAM; wildcard resource is required. "
6663 "The S3 permissions use the {project_name}-* prefix "
6664 "pattern because the model weights bucket name is "
6665 "auto-generated by CDK in the global stack and a "
6666 "cross-stack ARN export would create tight stack "
6667 "coupling. All actions are read-only or scoped "
6668 "send-only (SQS)."
6669 ),
6670 "appliesTo": [
6671 "Resource::*",
6672 ],
6673 },
6674 ],
6675 )
6677 def _create_outputs(self) -> None:
6678 """Create CloudFormation outputs for cluster information"""
6679 project_name = self.config.get_project_name()
6681 # Export cluster information
6682 CfnOutput(
6683 self,
6684 "ClusterName",
6685 value=self.cluster.cluster_name,
6686 description=f"EKS cluster name for {self.deployment_region}",
6687 export_name=f"{project_name}-cluster-name-{self.deployment_region}",
6688 )
6690 CfnOutput(
6691 self,
6692 "AddonDeploymentToken",
6693 value=self.addon_deployment_token,
6694 description=(
6695 "Exact token for the asynchronous Kubernetes and Helm convergence execution"
6696 ),
6697 )
6699 CfnOutput(
6700 self,
6701 "ClusterArn",
6702 value=self.cluster.cluster_arn,
6703 description=f"EKS cluster ARN for {self.deployment_region}",
6704 export_name=f"{project_name}-cluster-arn-{self.deployment_region}",
6705 )
6707 CfnOutput(
6708 self,
6709 "ClusterEndpoint",
6710 value=self.cluster.cluster_endpoint,
6711 description=f"EKS cluster endpoint for {self.deployment_region}",
6712 export_name=f"{project_name}-cluster-endpoint-{self.deployment_region}",
6713 )
6715 CfnOutput(
6716 self,
6717 "ClusterSecurityGroupId",
6718 value=self.cluster.cluster_security_group_id,
6719 description=f"EKS cluster security group ID for {self.deployment_region}",
6720 export_name=f"{project_name}-cluster-sg-{self.deployment_region}",
6721 )
6723 CfnOutput(
6724 self,
6725 "VpcId",
6726 value=self.vpc.vpc_id,
6727 description=f"VPC ID for {self.deployment_region}",
6728 export_name=f"{project_name}-vpc-id-{self.deployment_region}",
6729 )
6731 # Export public subnet IDs for ALB
6732 public_subnet_ids = [subnet.subnet_id for subnet in self.vpc.public_subnets]
6733 CfnOutput(
6734 self,
6735 "PublicSubnetIds",
6736 value=Fn.join(",", public_subnet_ids),
6737 description=f"Public subnet IDs for ALB in {self.deployment_region}",
6738 export_name=f"{project_name}-public-subnets-{self.deployment_region}",
6739 )
6741 # Note: the ALB is created by the AWS Load Balancer Controller from the
6742 # gco-system/gco-gateway Gateway API resources; the GA registration
6743 # Lambda registers its ARN with Global Accelerator
6745 def get_cluster(self) -> eks.Cluster:
6746 """Get the EKS cluster"""
6747 return self.cluster
6749 def get_vpc(self) -> ec2.Vpc:
6750 """Get the VPC"""
6751 return self.vpc