Coverage for gco/stacks/monitoring

1"""

2Monitoring stack for GCO (Global Capacity Orchestrator on AWS) - Cross-region monitoring and observability.

4This stack creates centralized monitoring resources for all GCO deployments:

5- CloudWatch Dashboard with comprehensive widgets for all regions

6- SNS topic for alerting

7- CloudWatch Alarms for critical metrics

8- Log groups for application logs

9- Anomaly detection for traffic patterns

10- Composite alarms for better signal-to-noise

12Dashboard Sections:

13- Global Accelerator: Flow counts, processed bytes

14- API Gateway: Request counts, latency, error rates

15- Lambda Functions: Invocations, errors, duration, throttles

16- SQS Queues: Message counts, age, dead letter queue depth

17- DynamoDB Tables: Capacity, latency, throttles, errors

18- EKS Clusters: CPU/memory utilization per region

19- FSx for Lustre (when enabled): Throughput, IOPS, free storage

20- Valkey Serverless (when enabled): ECPU, hit rate, latency, bytes used

21- Aurora pgvector (when enabled): ACU utilization, connections, latency, CPU

22- ALBs: Request counts, response times, healthy hosts

23- Applications: Custom metrics from health monitor and manifest processor

25Cross-Region Metrics:

26 CloudWatch metrics are region-specific. This stack handles cross-region

27 monitoring by specifying the `region` parameter on metrics:

28 - Global Accelerator metrics: Always in us-west-2

29 - DynamoDB metrics: In the global region (where tables are deployed)

30 - Regional metrics: In each cluster's region

32Alarms:

33- High CPU/memory utilization on EKS clusters

34- Unhealthy hosts in ALB target groups

35- High response times

36- Manifest processing failures

37- Lambda errors and throttles

38- SQS message age (stuck jobs)

39- DynamoDB throttling and system errors

40- API Gateway 5XX errors

41- Secret rotation failures

42"""

44from typing import TYPE_CHECKING, Any

46from aws_cdk import (

47 CfnOutput,

48 Duration,

49 RemovalPolicy,

50 Stack,

51)

52from aws_cdk import aws_athena as athena

53from aws_cdk import aws_cloudwatch as cloudwatch

54from aws_cdk import aws_cloudwatch_actions as cw_actions

55from aws_cdk import aws_glue as glue

56from aws_cdk import aws_iam as iam

57from aws_cdk import aws_kms as kms

58from aws_cdk import aws_logs as logs

59from aws_cdk import aws_s3 as s3

60from aws_cdk import aws_sns as sns

61from aws_cdk import aws_ssm as ssm

62from constructs import Construct

64from gco.config.config_loader import ConfigLoader

65from gco.stacks.constants import (

66 COST_ATHENA_RESULTS_PREFIX,

67 COST_GLUE_ALLOCATION_TABLE,

68 COST_REPORT_SCHEDULED_PREFIX,

69 cost_athena_workgroup_name,

70 cost_glue_database_name,

71 cost_report_ssm_parameter_prefix,

72)

74# <pyflowchart-code-diagram> BEGIN - auto-inserted, do not edit

75# Generated at (UTC): 2026-09-18T02:11:36Z

76# Generated from Git commit: b8faa9689385cea16155a285a7f70cf6d488e512

77# Flowchart(s) generated from this file:

78# * ``GCOMonitoringStack.__init__`` -> ``diagrams/code_diagrams/gco/stacks/monitoring_stack.GCOMonitoringStack___init__.html``

79# (PNG: ``diagrams/code_diagrams/gco/stacks/monitoring_stack.GCOMonitoringStack___init__.png``)

80# Regenerate with ``SOURCE_DATE_EPOCH=<unix-seconds> GCO_DIAGRAM_SOURCE_COMMIT=<40-char-sha> python diagrams/generate.py --code-only``.

81# <pyflowchart-code-diagram> END

84if TYPE_CHECKING:

85 from gco.stacks.api_gateway_global_stack import GCOApiGatewayGlobalStack

86 from gco.stacks.global_stack import GCOGlobalStack

87 from gco.stacks.regional_stack import GCORegionalStack

90class GCOMonitoringStack(Stack):

91 """

92 Cross-region monitoring and observability stack.

94 Creates a centralized CloudWatch dashboard and alarms that aggregate

95 metrics from all regional deployments.

97 Attributes:

98 alert_topic: SNS topic for alarm notifications

99 dashboard: CloudWatch dashboard with all monitoring widgets

100 """

101

102 def __init__(

103 self,

104 scope: Construct,

105 construct_id: str,

106 config: ConfigLoader,

107 global_stack: GCOGlobalStack,

108 regional_stacks: list[GCORegionalStack],

109 api_gateway_stack: GCOApiGatewayGlobalStack | None = None,

110 **kwargs: Any,

111 ) -> None:

112 # Enable CDK's native cross-region references. The monitoring stack

113 # lives in the monitoring region (by default us-east-2) and needs

114 # resource identifiers from the regional stacks for dashboard

115 # dimensions — specifically the auto-generated FSx file system IDs,

116 # whose values aren't known until deploy time.

117 #

118 # CDK implements this by provisioning a small Lambda-backed custom

119 # resource in each source stack that writes the referenced value to

120 # an SSM parameter in the target region, plus a reader custom

121 # resource in the target stack. Cost is negligible (the Lambdas run

122 # once per deploy) and the pattern is the documented canonical

123 # answer for ``CrossRegionReferencesNotEnabled`` errors.

124 kwargs.setdefault("cross_region_references", True)

125 super().__init__(scope, construct_id, **kwargs)

126

127 self.config = config

128 self.global_stack = global_stack

129 self.regional_stacks = regional_stacks

130 self.api_gateway_stack = api_gateway_stack

131 self.project_name = config.get_project_name()

132 self.regions = config.get_regions()

133

134 # Create SNS topic for alerts

135 self.alert_topic = self._create_alert_topic()

136

137 # Cost monitoring pipeline (on by default): the cost report bucket the

138 # regional cost-monitor services write Parquet allocation reports to,

139 # plus the Glue database/table and Athena workgroup that make the

140 # cross-region data queryable from the CLI.

141 if self.config.get_cost_monitoring_enabled():

142 self._create_cost_report_storage()

143 self._create_cost_analytics()

144

145 # Create CloudWatch dashboard

146 self.dashboard = self._create_dashboard()

147

148 # Create alarms

149 self._create_alarms()

150

151 # Create composite alarms

152 self._create_composite_alarms()

153

154 # Create custom metrics

155 self._create_custom_metrics()

156

157 # Export monitoring resources

158 self._create_outputs()

159

160 # Apply cdk-nag suppressions

161 self._apply_nag_suppressions()

162

163 def _apply_nag_suppressions(self) -> None:

164 """Apply cdk-nag suppressions for this stack."""

165 from gco.stacks.nag_suppressions import apply_all_suppressions

166

167 apply_all_suppressions(

168 self,

169 stack_type="monitoring",

170 regions=self.config.get_regions(),

171 global_region=self.config.get_global_region(),

172 project_name=self.project_name,

173 )

174

175 def _create_alert_topic(self) -> sns.Topic:

176 """Create SNS topic for monitoring alerts"""

177 topic = sns.Topic(

178 self,

179 "GCOAlertTopic",

180 display_name="GCO (Global Capacity Orchestrator on AWS) Monitoring Alerts",

181 enforce_ssl=True,

182 )

183 return topic

184

185 def _create_cost_report_storage(self) -> None:

186 """Create the central cost report bucket for the cost monitoring pipeline.

187

188 Every regional cost-monitor service writes Hive-partitioned Parquet

189 allocation reports here:

190

191 - ``reports/region=<region>/date=<YYYY-MM-DD>/...`` — scheduled

192 reports; the Glue table's partition projection reads this layout.

193 - ``adhoc/region=<region>/date=<YYYY-MM-DD>/...`` — user-requested

194 reports, kept out of the scheduled table so overlapping windows can

195 never double-count in Athena aggregations.

196 - ``athena-results/`` — Athena query results for the cost workgroup.

197

198 Three constructs mirror the regional-shared bucket pattern:

199

200 1. ``cost_report_kms_key`` — customer-managed KMS key with annual

201 rotation and a 7-day pending window on destroy. Its key policy

202 grants every regional cost-monitor role ``GenerateDataKey`` /

203 ``Decrypt`` / ``DescribeKey`` through S3 in this region.

204 2. ``cost_report_access_logs_bucket`` — the dedicated S3 access-logs

205 destination for the primary bucket.

206 3. ``cost_report_bucket`` — the primary bucket. Its physical name is

207 CloudFormation-generated: S3 bucket names are a global namespace

208 and a deleted name is not reliably reusable, so the previous fixed

209 ``<project>-cost-reports-<account>-<region>`` name made every

210 destroy-and-redeploy a collision hazard (and did fail one). The

211 regional stacks deploy *before* this stack, so instead of granting

212 by reconstructed ARN they are granted here, principal based, via

213 the bucket policy; the bucket's identity is published as SSM

214 parameters under ``cost_report_ssm_parameter_prefix`` for the

215 cost-monitor service, the CLI, and release validation to resolve.

216

217 Lifecycle policy comes from ``cdk.json`` (``cost_monitoring.reports``):

218 report objects transition to STANDARD_IA after

219 ``transition_to_infrequent_access_days`` and expire after

220 ``retention_days``; Athena results expire after

221 ``athena.query_results_retention_days``.

222 """

223 cost_config = self.config.get_cost_monitoring_config()

224 reports_config = cost_config["reports"]

225 athena_config = cost_config["athena"]

226

227 # KMS key for the cost report bucket. Annual rotation, 7-day pending

228 # window, destroy-on-teardown — matching the shared-bucket posture.

229 self.cost_report_kms_key = kms.Key(

230 self,

231 "CostReportKmsKey",

232 description=(

233 "Customer-managed KMS key for the GCO cost report bucket in the monitoring stack."

234 ),

235 enable_key_rotation=True,

236 pending_window=Duration.days(7),

237 removal_policy=RemovalPolicy.DESTROY,

238 )

239

240 kms_actions = [

241 "kms:Encrypt",

242 "kms:Decrypt",

243 "kms:ReEncrypt*",

244 "kms:GenerateDataKey*",

245 "kms:DescribeKey",

246 ]

247 self.cost_report_kms_key.add_to_resource_policy(

248 iam.PolicyStatement(

249 sid="AllowS3ServiceEncryptDecrypt",

250 effect=iam.Effect.ALLOW,

251 principals=[iam.ServicePrincipal("s3.amazonaws.com")],

252 actions=kms_actions,

253 resources=["*"],

254 )

255 )

256

257 # Every regional cost-monitor role writes KMS-encrypted objects into

258 # the bucket below. The roles live in other regions and their stacks

259 # deploy first, so the grant direction is inverted relative to the

260 # in-region buckets: this stack resolves each role ARN (a cross-region

261 # reference CDK routes through the same export mechanism the SQS and

262 # cluster widgets already use) and admits it here, scoped to use via

263 # S3 in this region. Nothing in the regional stacks needs the key ARN.

264 cost_monitor_principals = [

265 iam.ArnPrincipal(regional_stack.cost_monitor_role.role_arn)

266 for regional_stack in self.regional_stacks

267 ]

268 if cost_monitor_principals:

269 self.cost_report_kms_key.add_to_resource_policy(

270 iam.PolicyStatement(

271 sid="AllowRegionalCostMonitorsViaS3",

272 effect=iam.Effect.ALLOW,

273 principals=cost_monitor_principals,

274 actions=["kms:GenerateDataKey", "kms:Decrypt", "kms:DescribeKey"],

275 resources=["*"],

276 conditions={

277 "StringEquals": {"kms:ViaService": f"s3.{self.region}.{self.url_suffix}"}

278 },

279 )

280 )

281

282 # Retention for the access-logs bucket honors the same `s3_access_logs`

283 # context field used by the central buckets (default 90 days).

284 s3_access_logs_ctx = self.node.try_get_context("s3_access_logs") or {}

285 access_logs_retention_days = int(s3_access_logs_ctx.get("retention_days", 90))

286

287 self.cost_report_access_logs_bucket = s3.Bucket(

288 self,

289 "CostReportAccessLogsBucket",

290 encryption=s3.BucketEncryption.KMS,

291 encryption_key=self.cost_report_kms_key,

292 block_public_access=s3.BlockPublicAccess.BLOCK_ALL,

293 enforce_ssl=True,

294 versioned=True,

295 removal_policy=RemovalPolicy.DESTROY,

296 auto_delete_objects=True,

297 lifecycle_rules=[

298 s3.LifecycleRule(

299 id="ExpireAccessLogs",

300 enabled=True,

301 expiration=Duration.days(access_logs_retention_days),

302 )

303 ],

304 )

305

306 # No ``bucket_name``: see the S3 bucket naming policy in

307 # ``gco.stacks.constants`` — a CloudFormation-generated name can never

308 # collide with a previous deployment's deleted bucket.

309 self.cost_report_bucket = s3.Bucket(

310 self,

311 "CostReportBucket",

312 encryption=s3.BucketEncryption.KMS,

313 encryption_key=self.cost_report_kms_key,

314 bucket_key_enabled=True,

315 block_public_access=s3.BlockPublicAccess.BLOCK_ALL,

316 enforce_ssl=True,

317 versioned=True,

318 removal_policy=RemovalPolicy.DESTROY,

319 auto_delete_objects=True,

320 server_access_logs_bucket=self.cost_report_access_logs_bucket,

321 server_access_logs_prefix="cost-reports/",

322 lifecycle_rules=[

323 # Scheduled + ad-hoc reports share one policy: IA after the

324 # configured transition, expiry after the retention window.

325 s3.LifecycleRule(

326 id="CostReportRetention",

327 enabled=True,

328 prefix=f"{COST_REPORT_SCHEDULED_PREFIX}/",

329 transitions=[

330 s3.Transition(

331 storage_class=s3.StorageClass.INFREQUENT_ACCESS,

332 transition_after=Duration.days(

333 int(reports_config["transition_to_infrequent_access_days"])

334 ),

335 )

336 ],

337 expiration=Duration.days(int(reports_config["retention_days"])),

338 ),

339 s3.LifecycleRule(

340 id="AdhocReportRetention",

341 enabled=True,

342 prefix="adhoc/",

343 transitions=[

344 s3.Transition(

345 storage_class=s3.StorageClass.INFREQUENT_ACCESS,

346 transition_after=Duration.days(

347 int(reports_config["transition_to_infrequent_access_days"])

348 ),

349 )

350 ],

351 expiration=Duration.days(int(reports_config["retention_days"])),

352 ),

353 s3.LifecycleRule(

354 id="ExpireAthenaResults",

355 enabled=True,

356 prefix=f"{COST_ATHENA_RESULTS_PREFIX}/",

357 expiration=Duration.days(int(athena_config["query_results_retention_days"])),

358 ),

359 ],

360 )

361

362 # Explicit Deny for insecure transport with a verifiable SID,

363 # duplicating enforce_ssl=True per the central-bucket pattern.

364 self.cost_report_bucket.add_to_resource_policy(

365 iam.PolicyStatement(

366 sid="DenyInsecureTransport",

367 effect=iam.Effect.DENY,

368 principals=[iam.AnyPrincipal()],

369 actions=["s3:*"],

370 resources=[

371 self.cost_report_bucket.bucket_arn,

372 f"{self.cost_report_bucket.bucket_arn}/*",

373 ],

374 conditions={"Bool": {"aws:SecureTransport": "false"}},

375 )

376 )

377

378 # Principal-based write grant for the regional cost-monitor roles.

379 # Same-account bucket-policy allows are sufficient on their own, so

380 # the regional stacks carry no S3 statement for this bucket at all —

381 # the service writes scheduled/ad-hoc Parquet reports, lists recent

382 # report objects for the API surface, and needs GetBucketLocation for

383 # the cross-region client redirect. Object actions are scoped to this

384 # bucket's key space; nothing here is ``Principal: "*"``.

385 if cost_monitor_principals:

386 self.cost_report_bucket.add_to_resource_policy(

387 iam.PolicyStatement(

388 sid="AllowRegionalCostMonitorReports",

389 effect=iam.Effect.ALLOW,

390 principals=cost_monitor_principals,

391 actions=[

392 "s3:PutObject",

393 "s3:GetObject",

394 "s3:ListBucket",

395 "s3:GetBucketLocation",

396 ],

397 resources=[

398 self.cost_report_bucket.bucket_arn,

399 f"{self.cost_report_bucket.bucket_arn}/*",

400 ],

401 )

402 )

403

404 # Publish the bucket's identity in this region's parameter store,

405 # mirroring the model, cluster-shared and regional-shared buckets. The

406 # regional cost-monitor services resolve ``<prefix>/name`` at runtime

407 # (their stacks deploy before this one, so the parameter is the

408 # rendezvous point, not a synth-time value); the CLI and release

409 # validation read ``/name`` and ``/arn``.

410 cost_report_prefix = cost_report_ssm_parameter_prefix(self.project_name)

411 ssm.StringParameter(

412 self,

413 "CostReportBucketNameParam",

414 parameter_name=f"{cost_report_prefix}/name",

415 string_value=self.cost_report_bucket.bucket_name,

416 description="Name of the central cost report bucket written by every region.",

417 )

418 ssm.StringParameter(

419 self,

420 "CostReportBucketArnParam",

421 parameter_name=f"{cost_report_prefix}/arn",

422 string_value=self.cost_report_bucket.bucket_arn,

423 description="ARN of the central cost report bucket written by every region.",

424 )

425 ssm.StringParameter(

426 self,

427 "CostReportBucketRegionParam",

428 parameter_name=f"{cost_report_prefix}/region",

429 string_value=self.region,

430 description="Home region of the central cost report bucket.",

431 )

432

433 from gco.stacks.nag_suppressions import acknowledge_nag_findings

434

435 cost_replication_reason = (

436 "Cost reports are derived analytics data regenerated continuously "

437 "by the per-region cost-monitor services; there is no durability "

438 "requirement that warrants cross-region replication. Access logs "

439 "do not require replication for the same reason."

440 )

441 acknowledge_nag_findings(

442 self.cost_report_bucket,

443 [

444 {

445 "id": "HIPAA.Security-S3BucketReplicationEnabled",

446 "reason": cost_replication_reason,

447 },

448 {

449 "id": "NIST.800.53.R5-S3BucketReplicationEnabled",

450 "reason": cost_replication_reason,

451 },

452 {

453 "id": "PCI.DSS.321-S3BucketReplicationEnabled",

454 "reason": cost_replication_reason,

455 },

456 ],

457 )

458

459 access_logs_is_self_target_reason = (

460 "This is the server access logs destination bucket for the cost report bucket."

461 )

462 acknowledge_nag_findings(

463 self.cost_report_access_logs_bucket,

464 [

465 {

466 "id": "AwsSolutions-S1",

467 "reason": access_logs_is_self_target_reason,

468 },

469 {

470 "id": "HIPAA.Security-S3BucketLoggingEnabled",

471 "reason": access_logs_is_self_target_reason,

472 },

473 {

474 "id": "NIST.800.53.R5-S3BucketLoggingEnabled",

475 "reason": access_logs_is_self_target_reason,

476 },

477 {

478 "id": "PCI.DSS.321-S3BucketLoggingEnabled",

479 "reason": access_logs_is_self_target_reason,

480 },

481 {

482 "id": "HIPAA.Security-S3BucketReplicationEnabled",

483 "reason": cost_replication_reason,

484 },

485 {

486 "id": "NIST.800.53.R5-S3BucketReplicationEnabled",

487 "reason": cost_replication_reason,

488 },

489 {

490 "id": "PCI.DSS.321-S3BucketReplicationEnabled",

491 "reason": cost_replication_reason,

492 },

493 ],

494 )

495

496 CfnOutput(

497 self,

498 "CostReportBucketName",

499 value=self.cost_report_bucket.bucket_name,

500 description="Central S3 bucket receiving per-region Parquet cost reports",

501 )

502

503 def _create_cost_analytics(self) -> None:

504 """Create the Glue database/table and Athena workgroup for cost queries.

505

506 The Glue table reads the scheduled report layout

507 (``reports/region=<region>/date=<YYYY-MM-DD>/*.parquet``) using

508 **partition projection**, so there is no crawler, no scheduled

509 ``MSCK REPAIR``, and no partition-management Lambda — new partitions

510 are queryable the moment an object lands. The ``region`` partition is

511 projected from the configured deployment regions; ``date`` is a native

512 date projection from 2026-01-01 to NOW.

513

514 The Athena workgroup pins query results to ``athena-results/`` in the

515 cost report bucket (KMS-encrypted, lifecycle-expired) and enforces its

516 configuration so callers cannot redirect results elsewhere.

517

518 The table schema is the write-side contract of

519 ``gco.services.cost_monitor`` — the columns below must stay in

520 lockstep with the Parquet fields the service emits.

521 """

522 database_name = cost_glue_database_name(self.project_name)

523

524 self.cost_glue_database = glue.CfnDatabase(

525 self,

526 "CostGlueDatabase",

527 catalog_id=self.account,

528 database_input=glue.CfnDatabase.DatabaseInputProperty(

529 name=database_name,

530 description=(

531 "GCO cost analytics: per-region OpenCost allocation "

532 "reports written by the cost-monitor services."

533 ),

534 ),

535 )

536

537 # Columns mirror gco/services/cost_monitor.py::ALLOCATION_REPORT_FIELDS.

538 columns = [

539 glue.CfnTable.ColumnProperty(name="window_start", type="timestamp"),

540 glue.CfnTable.ColumnProperty(name="window_end", type="timestamp"),

541 glue.CfnTable.ColumnProperty(name="cluster", type="string"),

542 glue.CfnTable.ColumnProperty(name="namespace", type="string"),

543 glue.CfnTable.ColumnProperty(name="cpu_core_hours", type="double"),

544 glue.CfnTable.ColumnProperty(name="cpu_cost", type="double"),

545 glue.CfnTable.ColumnProperty(name="ram_gib_hours", type="double"),

546 glue.CfnTable.ColumnProperty(name="ram_cost", type="double"),

547 glue.CfnTable.ColumnProperty(name="gpu_hours", type="double"),

548 glue.CfnTable.ColumnProperty(name="gpu_cost", type="double"),

549 glue.CfnTable.ColumnProperty(name="pv_cost", type="double"),

550 glue.CfnTable.ColumnProperty(name="network_cost", type="double"),

551 glue.CfnTable.ColumnProperty(name="load_balancer_cost", type="double"),

552 glue.CfnTable.ColumnProperty(name="shared_cost", type="double"),

553 glue.CfnTable.ColumnProperty(name="external_cost", type="double"),

554 glue.CfnTable.ColumnProperty(name="total_cost", type="double"),

555 glue.CfnTable.ColumnProperty(name="total_efficiency", type="double"),

556 ]

557

558 scheduled_location = (

559 f"s3://{self.cost_report_bucket.bucket_name}/{COST_REPORT_SCHEDULED_PREFIX}/"

560 )

561 self.cost_glue_table = glue.CfnTable(

562 self,

563 "CostAllocationTable",

564 catalog_id=self.account,

565 database_name=database_name,

566 table_input=glue.CfnTable.TableInputProperty(

567 name=COST_GLUE_ALLOCATION_TABLE,

568 description="Scheduled OpenCost allocation reports (Parquet)",

569 table_type="EXTERNAL_TABLE",

570 parameters={

571 "classification": "parquet",

572 "EXTERNAL": "TRUE",

573 # Partition projection: no crawler, no MSCK. The region

574 # projection is pinned to the deployment's configured

575 # regions; extend deployment_regions.regional and redeploy

576 # to pick up new regions.

577 "projection.enabled": "true",

578 "projection.region.type": "enum",

579 "projection.region.values": ",".join(self.regions),

580 "projection.date.type": "date",

581 "projection.date.range": "2026-01-01,NOW",

582 "projection.date.format": "yyyy-MM-dd",

583 "projection.date.interval": "1",

584 "projection.date.interval.unit": "DAYS",

585 "storage.location.template": (

586 f"{scheduled_location}region=${{region}}/date=${{date}}"

587 ),

588 },

589 partition_keys=[

590 glue.CfnTable.ColumnProperty(name="region", type="string"),

591 glue.CfnTable.ColumnProperty(name="date", type="string"),

592 ],

593 storage_descriptor=glue.CfnTable.StorageDescriptorProperty(

594 location=scheduled_location,

595 input_format=("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"),

596 output_format=(

597 "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"

598 ),

599 serde_info=glue.CfnTable.SerdeInfoProperty(

600 serialization_library=(

601 "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"

602 ),

603 ),

604 columns=columns,

605 ),

606 ),

607 )

608 self.cost_glue_table.add_resource_dependency(self.cost_glue_database)

609

610 self.cost_athena_workgroup = athena.CfnWorkGroup(

611 self,

612 "CostAthenaWorkGroup",

613 name=cost_athena_workgroup_name(self.project_name),

614 description="GCO cost analytics queries over the cost report bucket",

615 recursive_delete_option=True,

616 work_group_configuration=athena.CfnWorkGroup.WorkGroupConfigurationProperty(

617 enforce_work_group_configuration=True,

618 publish_cloud_watch_metrics_enabled=True,

619 result_configuration=athena.CfnWorkGroup.ResultConfigurationProperty(

620 output_location=(

621 f"s3://{self.cost_report_bucket.bucket_name}/{COST_ATHENA_RESULTS_PREFIX}/"

622 ),

623 encryption_configuration=(

624 athena.CfnWorkGroup.EncryptionConfigurationProperty(

625 encryption_option="SSE_KMS",

626 kms_key=self.cost_report_kms_key.key_arn,

627 )

628 ),

629 ),

630 ),

631 )

632

633 CfnOutput(

634 self,

635 "CostAthenaWorkGroupName",

636 value=cost_athena_workgroup_name(self.project_name),

637 description="Athena workgroup for GCO cost analytics queries",

638 )

639 CfnOutput(

640 self,

641 "CostGlueDatabaseName",

642 value=cost_glue_database_name(self.project_name),

643 description="Glue database containing the cost allocation table",

644 )

645

646 def _create_dashboard(self) -> cloudwatch.Dashboard:

647 """Create comprehensive CloudWatch dashboard for monitoring"""

648 dashboard = cloudwatch.Dashboard(

649 self,

650 "GCODashboard",

651 period_override=cloudwatch.PeriodOverride.AUTO,

652 )

653

654 # Add widgets in logical order. The control-plane sections (Global

655 # Accelerator, API Gateway, Lambda, DynamoDB) are always present. The

656 # per-Region sections exist only when at least one workload Region is

657 # configured: a control-plane-only deployment has no cluster, queue,

658 # or file system to chart, and a single-value widget whose metric list

659 # is empty is rejected by CloudWatch's dashboard validation at deploy

660 # time, so the sections are replaced by one explanatory note instead.

661 dashboard.add_widgets(*self._create_global_accelerator_widgets())

662 dashboard.add_widgets(*self._create_api_gateway_widgets())

663 dashboard.add_widgets(*self._create_lambda_widgets())

664 if self.regional_stacks:

665 dashboard.add_widgets(*self._create_sqs_widgets())

666 dashboard.add_widgets(*self._create_dynamodb_widgets())

667 if self.regional_stacks:

668 dashboard.add_widgets(*self._create_eks_widgets())

669 dashboard.add_widgets(*self._create_gpu_widgets())

670 dashboard.add_widgets(*self._create_fsx_widgets())

671 dashboard.add_widgets(*self._create_valkey_widgets())

672 dashboard.add_widgets(*self._create_aurora_pgvector_widgets())

673 dashboard.add_widgets(*self._create_alb_widgets())

674 dashboard.add_widgets(*self._create_application_widgets())

675 else:

676 dashboard.add_widgets(self._create_no_workload_regions_widget())

677

678 return dashboard

679

680 def _create_no_workload_regions_widget(self) -> cloudwatch.IWidget:

681 """Explain the missing per-Region sections of a control-plane-only deployment."""

682 return cloudwatch.TextWidget(

683 markdown=(

684 "# Workload Regions\n"

685 "No workload Regions are configured (`deployment_regions.regional` is "

686 "empty), so this deployment has no EKS cluster, job queue, GPU, storage, "

687 "ALB, or application metrics to show. Add a Region with "

688 "`gco stacks regions add <region>` and run `gco stacks deploy-all` to "

689 "bring the per-Region sections back."

690 ),

691 width=24,

692 height=2,

693 )

694

695 def _create_global_accelerator_widgets(self) -> list[cloudwatch.IWidget]:

696 """Create Global Accelerator monitoring widgets.

697

698 Note: Global Accelerator metrics are only available in us-west-2,

699 regardless of where the accelerator endpoints are located.

700 CloudWatch uses the Accelerator ID (UUID), not the name.

701 """

702 widgets: list[cloudwatch.IWidget] = []

703 if self.global_stack.accelerator_id is None:

704 return widgets

705

706 # Get the accelerator ID from the global stack (CloudWatch uses ID, not name)

707 accelerator_id = self.global_stack.accelerator_id

708

709 # Global Accelerator metrics are always in us-west-2

710 ga_metrics_region = "us-west-2"

711

712 # Section header

713 widgets.append(

714 cloudwatch.TextWidget(

715 markdown="# Global Accelerator\nTraffic distribution and connectivity metrics",

716 width=24,

717 height=1,

718 )

719 )

720

721 # Flow count with anomaly detection

722 flow_count_widget = cloudwatch.GraphWidget(

723 title="Global Accelerator - New Flows",

724 left=[

725 cloudwatch.Metric(

726 namespace="AWS/GlobalAccelerator",

727 metric_name="NewFlowCount",

728 dimensions_map={"Accelerator": accelerator_id},

729 statistic="Sum",

730 period=Duration.minutes(5),

731 region=ga_metrics_region,

732 )

733 ],

734 width=12,

735 height=6,

736 region=ga_metrics_region,

737 )

738 widgets.append(flow_count_widget)

739

740 # Processed bytes

741 bytes_widget = cloudwatch.GraphWidget(

742 title="Global Accelerator - Processed Bytes",

743 left=[

744 cloudwatch.Metric(

745 namespace="AWS/GlobalAccelerator",

746 metric_name="ProcessedBytesIn",

747 dimensions_map={"Accelerator": accelerator_id},

748 statistic="Sum",

749 period=Duration.minutes(5),

750 region=ga_metrics_region,

751 ),

752 cloudwatch.Metric(

753 namespace="AWS/GlobalAccelerator",

754 metric_name="ProcessedBytesOut",

755 dimensions_map={"Accelerator": accelerator_id},

756 statistic="Sum",

757 period=Duration.minutes(5),

758 region=ga_metrics_region,

759 ),

760 ],

761 width=12,

762 height=6,

763 region=ga_metrics_region,

764 )

765 widgets.append(bytes_widget)

766

767 return widgets

768

769 def _create_api_gateway_widgets(self) -> list[cloudwatch.IWidget]:

770 """Create API Gateway monitoring widgets"""

771 widgets: list[cloudwatch.IWidget] = []

772

773 # Get the actual API name from the api_gateway_stack

774 api_name = (

775 self.api_gateway_stack.api.rest_api_name

776 if self.api_gateway_stack

777 else f"{self.project_name}-global-api"

778 )

779

780 # API Gateway metrics are in the region where the API is deployed

781 api_gw_region = self.config.get_api_gateway_region()

782

783 # Section header

784 widgets.append(

785 cloudwatch.TextWidget(

786 markdown="# API Gateway\nRequest metrics, latency, and error rates",

787 width=24,

788 height=1,

789 )

790 )

791

792 # Request count and latency

793 request_widget = cloudwatch.GraphWidget(

794 title="API Gateway - Requests & Latency",

795 left=[

796 cloudwatch.Metric(

797 namespace="AWS/ApiGateway",

798 metric_name="Count",

799 dimensions_map={"ApiName": api_name},

800 statistic="Sum",

801 period=Duration.minutes(5),

802 region=api_gw_region,

803 )

804 ],

805 right=[

806 cloudwatch.Metric(

807 namespace="AWS/ApiGateway",

808 metric_name="Latency",

809 dimensions_map={"ApiName": api_name},

810 statistic="Average",

811 period=Duration.minutes(5),

812 region=api_gw_region,

813 ),

814 cloudwatch.Metric(

815 namespace="AWS/ApiGateway",

816 metric_name="Latency",

817 dimensions_map={"ApiName": api_name},

818 statistic="p99",

819 period=Duration.minutes(5),

820 region=api_gw_region,

821 ),

822 ],

823 width=12,

824 height=6,

825 region=api_gw_region,

826 )

827 widgets.append(request_widget)

828

829 # Error rates (4XX and 5XX)

830 error_widget = cloudwatch.GraphWidget(

831 title="API Gateway - Error Rates",

832 left=[

833 cloudwatch.Metric(

834 namespace="AWS/ApiGateway",

835 metric_name="4XXError",

836 dimensions_map={"ApiName": api_name},

837 statistic="Sum",

838 period=Duration.minutes(5),

839 color="#ff7f0e",

840 region=api_gw_region,

841 ),

842 cloudwatch.Metric(

843 namespace="AWS/ApiGateway",

844 metric_name="5XXError",

845 dimensions_map={"ApiName": api_name},

846 statistic="Sum",

847 period=Duration.minutes(5),

848 color="#d62728",

849 region=api_gw_region,

850 ),

851 ],

852 width=12,

853 height=6,

854 region=api_gw_region,

855 )

856 widgets.append(error_widget)

857

858 return widgets

859

860 def _create_lambda_widgets(self) -> list[cloudwatch.IWidget]:

861 """Create Lambda function monitoring widgets"""

862 widgets: list[cloudwatch.IWidget] = []

863

864 # Section header

865 widgets.append(

866 cloudwatch.TextWidget(

867 markdown="# Lambda Functions\nProxy, rotation, and regional Lambda metrics",

868 width=24,

869 height=1,

870 )

871 )

872

873 # Get API Gateway region for global Lambda functions

874 api_gw_region = self.config.get_api_gateway_region()

875

876 # Build Lambda function list: (function_name, label, region)

877 lambda_functions: list[tuple[str, str, str]] = []

878

879 # Add API Gateway Lambda functions if available

880 if self.api_gateway_stack:

881 if self.api_gateway_stack.proxy_lambda is not None:

882 lambda_functions.append(

883 (

884 self.api_gateway_stack.proxy_lambda.function_name,

885 "API Gateway Proxy",

886 api_gw_region,

887 )

888 )

889 lambda_functions.append(

890 (

891 self.api_gateway_stack.rotation_lambda.function_name,

892 "Secret Rotation",

893 api_gw_region,

894 )

895 )

896

897 # Add regional Lambda functions from each regional stack

898 for regional_stack in self.regional_stacks:

899 region = regional_stack.deployment_region

900 lambda_functions.extend(

901 [

902 (

903 regional_stack.kubectl_lambda_function_name,

904 f"Kubectl Applier ({region})",

905 region,

906 ),

907 (

908 regional_stack.helm_installer_lambda_function_name,

909 f"Helm Installer ({region})",

910 region,

911 ),

912 ]

913 )

914

915 # Invocations widget

916 invocations_widget = cloudwatch.GraphWidget(

917 title="Lambda - Invocations",

918 left=[

919 cloudwatch.Metric(

920 namespace="AWS/Lambda",

921 metric_name="Invocations",

922 dimensions_map={"FunctionName": func_name},

923 statistic="Sum",

924 period=Duration.minutes(5),

925 label=label,

926 region=region,

927 )

928 for func_name, label, region in lambda_functions[:5]

929 ],

930 width=12,

931 height=6,

932 )

933 widgets.append(invocations_widget)

934

935 errors_widget = cloudwatch.GraphWidget(

936 title="Lambda - Errors",

937 left=[

938 cloudwatch.Metric(

939 namespace="AWS/Lambda",

940 metric_name="Errors",

941 dimensions_map={"FunctionName": func_name},

942 statistic="Sum",

943 period=Duration.minutes(5),

944 label=label,

945 color="#d62728",

946 region=region,

947 )

948 for func_name, label, region in lambda_functions[:5]

949 ],

950 width=12,

951 height=6,

952 )

953 widgets.append(errors_widget)

954

955 # Duration widget

956 duration_widget = cloudwatch.GraphWidget(

957 title="Lambda - Duration (ms)",

958 left=[

959 cloudwatch.Metric(

960 namespace="AWS/Lambda",

961 metric_name="Duration",

962 dimensions_map={"FunctionName": func_name},

963 statistic="Average",

964 period=Duration.minutes(5),

965 label=label,

966 region=region,

967 )

968 for func_name, label, region in lambda_functions[:5]

969 ],

970 width=12,

971 height=6,

972 )

973 widgets.append(duration_widget)

974

975 # Throttles widget

976 throttles_widget = cloudwatch.GraphWidget(

977 title="Lambda - Throttles & Concurrent Executions",

978 left=[

979 cloudwatch.Metric(

980 namespace="AWS/Lambda",

981 metric_name="Throttles",

982 dimensions_map={"FunctionName": func_name},

983 statistic="Sum",

984 period=Duration.minutes(5),

985 label=f"{label} Throttles",

986 region=region,

987 )

988 for func_name, label, region in lambda_functions[:3]

989 ],

990 right=[

991 cloudwatch.Metric(

992 namespace="AWS/Lambda",

993 metric_name="ConcurrentExecutions",

994 dimensions_map={"FunctionName": func_name},

995 statistic="Maximum",

996 period=Duration.minutes(5),

997 label=f"{label} Concurrent",

998 region=region,

999 )

1000 for func_name, label, region in lambda_functions[:3]

1001 ],

1002 width=12,

1003 height=6,

1004 )

1005 widgets.append(throttles_widget)

1006

1007 return widgets

1008

1009 def _create_sqs_widgets(self) -> list[cloudwatch.IWidget]:

1010 """Create SQS queue monitoring widgets"""

1011 widgets: list[cloudwatch.IWidget] = []

1012

1013 # Section header

1014 widgets.append(

1015 cloudwatch.TextWidget(

1016 markdown="# SQS Queues\nJob submission queue metrics and dead letter queue",

1017 width=24,

1018 height=1,

1019 )

1020 )

1021

1022 # Build queue info from regional stacks: (queue_name, dlq_name, region)

1023 queue_info = [

1024 (

1025 regional_stack.job_queue.queue_name,

1026 regional_stack.job_dlq.queue_name,

1027 regional_stack.deployment_region,

1028 )

1029 for regional_stack in self.regional_stacks

1030 ]

1031

1032 # Messages visible and in-flight per region

1033 messages_widget = cloudwatch.GraphWidget(

1034 title="SQS - Messages (Visible & In-Flight)",

1035 left=[

1036 cloudwatch.Metric(

1037 namespace="AWS/SQS",

1038 metric_name="ApproximateNumberOfMessagesVisible",

1039 dimensions_map={"QueueName": queue_name},

1040 statistic="Average",

1041 period=Duration.minutes(1),

1042 label=f"{region} Visible",

1043 region=region,

1044 )

1045 for queue_name, _, region in queue_info

1046 ],

1047 right=[

1048 cloudwatch.Metric(

1049 namespace="AWS/SQS",

1050 metric_name="ApproximateNumberOfMessagesNotVisible",

1051 dimensions_map={"QueueName": queue_name},

1052 statistic="Average",

1053 period=Duration.minutes(1),

1054 label=f"{region} In-Flight",

1055 region=region,

1056 )

1057 for queue_name, _, region in queue_info

1058 ],

1059 width=12,

1060 height=6,

1061 )

1062 widgets.append(messages_widget)

1063

1064 # Age of oldest message (critical for detecting stuck jobs)

1065 age_widget = cloudwatch.GraphWidget(

1066 title="SQS - Age of Oldest Message (seconds)",

1067 left=[

1068 cloudwatch.Metric(

1069 namespace="AWS/SQS",

1070 metric_name="ApproximateAgeOfOldestMessage",

1071 dimensions_map={"QueueName": queue_name},

1072 statistic="Maximum",

1073 period=Duration.minutes(1),

1074 label=region,

1075 region=region,

1076 )

1077 for queue_name, _, region in queue_info

1078 ],

1079 width=12,

1080 height=6,

1081 )

1082 widgets.append(age_widget)

1083

1084 # Dead letter queue depth

1085 dlq_widget = cloudwatch.GraphWidget(

1086 title="SQS - Dead Letter Queue Depth",

1087 left=[

1088 cloudwatch.Metric(

1089 namespace="AWS/SQS",

1090 metric_name="ApproximateNumberOfMessagesVisible",

1091 dimensions_map={"QueueName": dlq_name},

1092 statistic="Average",

1093 period=Duration.minutes(1),

1094 label=f"{region} DLQ",

1095 color="#d62728",

1096 region=region,

1097 )

1098 for _, dlq_name, region in queue_info

1099 ],

1100 width=12,

1101 height=6,

1102 )

1103 widgets.append(dlq_widget)

1104

1105 # Messages sent/received/deleted

1106 throughput_widget = cloudwatch.GraphWidget(

1107 title="SQS - Throughput",

1108 left=[

1109 cloudwatch.Metric(

1110 namespace="AWS/SQS",

1111 metric_name="NumberOfMessagesSent",

1112 dimensions_map={"QueueName": queue_name},

1113 statistic="Sum",

1114 period=Duration.minutes(5),

1115 label=f"{region} Sent",

1116 region=region,

1117 )

1118 for queue_name, _, region in queue_info

1119 ],

1120 right=[

1121 cloudwatch.Metric(

1122 namespace="AWS/SQS",

1123 metric_name="NumberOfMessagesDeleted",

1124 dimensions_map={"QueueName": queue_name},

1125 statistic="Sum",

1126 period=Duration.minutes(5),

1127 label=f"{region} Processed",

1128 region=region,

1129 )

1130 for queue_name, _, region in queue_info

1131 ],

1132 width=12,

1133 height=6,

1134 )

1135 widgets.append(throughput_widget)

1136

1137 return widgets

1138

1139 def _create_dynamodb_widgets(self) -> list[cloudwatch.IWidget]:

1140 """Create DynamoDB monitoring widgets for job queue, templates, and webhooks tables."""

1141 widgets: list[cloudwatch.IWidget] = []

1142

1143 # Get table names from global stack

1144 templates_table = self.global_stack.templates_table.table_name

1145 webhooks_table = self.global_stack.webhooks_table.table_name

1146 jobs_table = self.global_stack.jobs_table.table_name

1147

1148 # DynamoDB tables are in the global region

1149 global_region = self.config.get_global_region()

1150

1151 # Section header

1152 widgets.append(

1153 cloudwatch.TextWidget(

1154 markdown="# DynamoDB Tables\nJob queue, templates, and webhooks storage metrics",

1155 width=24,

1156 height=1,

1157 )

1158 )

1159

1160 # Read/Write capacity consumed

1161 capacity_widget = cloudwatch.GraphWidget(

1162 title="DynamoDB - Consumed Capacity",

1163 left=[

1164 cloudwatch.Metric(

1165 namespace="AWS/DynamoDB",

1166 metric_name="ConsumedReadCapacityUnits",

1167 dimensions_map={"TableName": jobs_table},

1168 statistic="Sum",

1169 period=Duration.minutes(5),

1170 label="Jobs Read",

1171 region=global_region,

1172 ),

1173 cloudwatch.Metric(

1174 namespace="AWS/DynamoDB",

1175 metric_name="ConsumedReadCapacityUnits",

1176 dimensions_map={"TableName": templates_table},

1177 statistic="Sum",

1178 period=Duration.minutes(5),

1179 label="Templates Read",

1180 region=global_region,

1181 ),

1182 cloudwatch.Metric(

1183 namespace="AWS/DynamoDB",

1184 metric_name="ConsumedReadCapacityUnits",

1185 dimensions_map={"TableName": webhooks_table},

1186 statistic="Sum",

1187 period=Duration.minutes(5),

1188 label="Webhooks Read",

1189 region=global_region,

1190 ),

1191 ],

1192 right=[

1193 cloudwatch.Metric(

1194 namespace="AWS/DynamoDB",

1195 metric_name="ConsumedWriteCapacityUnits",

1196 dimensions_map={"TableName": jobs_table},

1197 statistic="Sum",

1198 period=Duration.minutes(5),

1199 label="Jobs Write",

1200 region=global_region,

1201 ),

1202 cloudwatch.Metric(

1203 namespace="AWS/DynamoDB",

1204 metric_name="ConsumedWriteCapacityUnits",

1205 dimensions_map={"TableName": templates_table},

1206 statistic="Sum",

1207 period=Duration.minutes(5),

1208 label="Templates Write",

1209 region=global_region,

1210 ),

1211 ],

1212 width=12,

1213 height=6,

1214 region=global_region,

1215 )

1216 widgets.append(capacity_widget)

1217

1218 # Latency metrics

1219 latency_widget = cloudwatch.GraphWidget(

1220 title="DynamoDB - Latency (ms)",

1221 left=[

1222 cloudwatch.Metric(

1223 namespace="AWS/DynamoDB",

1224 metric_name="SuccessfulRequestLatency",

1225 dimensions_map={"TableName": jobs_table, "Operation": "GetItem"},

1226 statistic="Average",

1227 period=Duration.minutes(5),

1228 label="Jobs GetItem",

1229 region=global_region,

1230 ),

1231 cloudwatch.Metric(

1232 namespace="AWS/DynamoDB",

1233 metric_name="SuccessfulRequestLatency",

1234 dimensions_map={"TableName": jobs_table, "Operation": "PutItem"},

1235 statistic="Average",

1236 period=Duration.minutes(5),

1237 label="Jobs PutItem",

1238 region=global_region,

1239 ),

1240 cloudwatch.Metric(

1241 namespace="AWS/DynamoDB",

1242 metric_name="SuccessfulRequestLatency",

1243 dimensions_map={"TableName": jobs_table, "Operation": "Query"},

1244 statistic="Average",

1245 period=Duration.minutes(5),

1246 label="Jobs Query",

1247 region=global_region,

1248 ),

1249 ],

1250 width=12,

1251 height=6,

1252 region=global_region,

1253 )

1254 widgets.append(latency_widget)

1255

1256 # Throttled requests

1257 throttle_widget = cloudwatch.GraphWidget(

1258 title="DynamoDB - Throttled Requests",

1259 left=[

1260 cloudwatch.Metric(

1261 namespace="AWS/DynamoDB",

1262 metric_name="ThrottledRequests",

1263 dimensions_map={"TableName": jobs_table},

1264 statistic="Sum",

1265 period=Duration.minutes(5),

1266 label="Jobs",

1267 color="#d62728",

1268 region=global_region,

1269 ),

1270 cloudwatch.Metric(

1271 namespace="AWS/DynamoDB",

1272 metric_name="ThrottledRequests",

1273 dimensions_map={"TableName": templates_table},

1274 statistic="Sum",

1275 period=Duration.minutes(5),

1276 label="Templates",

1277 color="#ff7f0e",

1278 region=global_region,

1279 ),

1280 cloudwatch.Metric(

1281 namespace="AWS/DynamoDB",

1282 metric_name="ThrottledRequests",

1283 dimensions_map={"TableName": webhooks_table},

1284 statistic="Sum",

1285 period=Duration.minutes(5),

1286 label="Webhooks",

1287 color="#9467bd",

1288 region=global_region,

1289 ),

1290 ],

1291 width=12,

1292 height=6,

1293 region=global_region,

1294 )

1295 widgets.append(throttle_widget)

1296

1297 # System errors

1298 errors_widget = cloudwatch.GraphWidget(

1299 title="DynamoDB - System Errors",

1300 left=[

1301 cloudwatch.Metric(

1302 namespace="AWS/DynamoDB",

1303 metric_name="SystemErrors",

1304 dimensions_map={"TableName": jobs_table},

1305 statistic="Sum",

1306 period=Duration.minutes(5),

1307 label="Jobs",

1308 color="#d62728",

1309 region=global_region,

1310 ),

1311 cloudwatch.Metric(

1312 namespace="AWS/DynamoDB",

1313 metric_name="SystemErrors",

1314 dimensions_map={"TableName": templates_table},

1315 statistic="Sum",

1316 period=Duration.minutes(5),

1317 label="Templates",

1318 color="#ff7f0e",

1319 region=global_region,

1320 ),

1321 ],

1322 width=12,

1323 height=6,

1324 region=global_region,

1325 )

1326 widgets.append(errors_widget)

1327

1328 return widgets

1329

1330 def _create_eks_widgets(self) -> list[cloudwatch.IWidget]:

1331 """Create EKS cluster monitoring widgets"""

1332 widgets: list[cloudwatch.IWidget] = []

1333

1334 # Section header

1335 widgets.append(

1336 cloudwatch.TextWidget(

1337 markdown="# EKS Clusters\nCluster resource utilization and node metrics",

1338 width=24,

1339 height=1,

1340 )

1341 )

1342

1343 # Build cluster info from regional stacks: (cluster_name, region)

1344 cluster_info = [

1345 (regional_stack.cluster.cluster_name, regional_stack.deployment_region)

1346 for regional_stack in self.regional_stacks

1347 ]

1348

1349 # EKS cluster status

1350 cluster_status_widget = cloudwatch.SingleValueWidget(

1351 title="EKS Clusters - Failed Requests",

1352 metrics=[

1353 cloudwatch.Metric(

1354 namespace="AWS/EKS",

1355 metric_name="cluster_failed_request_count",

1356 dimensions_map={"cluster_name": cluster_name},

1357 statistic="Sum",

1358 period=Duration.minutes(5),

1359 region=region,

1360 )

1361 for cluster_name, region in cluster_info

1362 ],

1363 width=12,

1364 height=6,

1365 )

1366 widgets.append(cluster_status_widget)

1367

1368 # Container Insights - Node CPU utilization (aggregated across all nodes)

1369 # Note: region parameter enables cross-region metrics in dashboard

1370 cpu_widget = cloudwatch.GraphWidget(

1371 title="EKS Clusters - Node CPU Utilization (%)",

1372 left=[

1373 cloudwatch.Metric(

1374 namespace="ContainerInsights",

1375 metric_name="node_cpu_utilization",

1376 dimensions_map={"ClusterName": cluster_name},

1377 statistic="Average",

1378 period=Duration.minutes(5),

1379 label=region,

1380 region=region,

1381 )

1382 for cluster_name, region in cluster_info

1383 ],

1384 width=12,

1385 height=6,

1386 )

1387 widgets.append(cpu_widget)

1388

1389 # Container Insights - Node Memory utilization (aggregated across all nodes)

1390 memory_widget = cloudwatch.GraphWidget(

1391 title="EKS Clusters - Node Memory Utilization (%)",

1392 left=[

1393 cloudwatch.Metric(

1394 namespace="ContainerInsights",

1395 metric_name="node_memory_utilization",

1396 dimensions_map={"ClusterName": cluster_name},

1397 statistic="Average",

1398 period=Duration.minutes(5),

1399 label=region,

1400 region=region,

1401 )

1402 for cluster_name, region in cluster_info

1403 ],

1404 width=12,

1405 height=6,

1406 )

1407 widgets.append(memory_widget)

1408

1409 # Node status - running pods capacity

1410 node_widget = cloudwatch.GraphWidget(

1411 title="EKS Clusters - Node Pod Capacity",

1412 left=[

1413 cloudwatch.Metric(

1414 namespace="ContainerInsights",

1415 metric_name="node_status_capacity_pods",

1416 dimensions_map={"ClusterName": cluster_name},

1417 statistic="Sum",

1418 period=Duration.minutes(5),

1419 label=f"{region} Capacity",

1420 region=region,

1421 )

1422 for cluster_name, region in cluster_info

1423 ],

1424 right=[

1425 cloudwatch.Metric(

1426 namespace="ContainerInsights",

1427 metric_name="node_number_of_running_pods",

1428 dimensions_map={"ClusterName": cluster_name},

1429 statistic="Sum",

1430 period=Duration.minutes(5),

1431 label=f"{region} Running",

1432 region=region,

1433 )

1434 for cluster_name, region in cluster_info

1435 ],

1436 width=12,

1437 height=6,

1438 )

1439 widgets.append(node_widget)

1440

1441 return widgets

1442

1443 def _create_gpu_widgets(self) -> list[cloudwatch.IWidget]:

1444 """Create GPU monitoring widgets using DCGM Exporter metrics via ContainerInsights."""

1445 widgets: list[cloudwatch.IWidget] = []

1446

1447 widgets.append(

1448 cloudwatch.TextWidget(

1449 markdown="# GPU Metrics\nGPU utilization, memory, and temperature from DCGM Exporter",

1450 width=24,

1451 height=1,

1452 )

1453 )

1454

1455 cluster_info = [

1456 (regional_stack.cluster.cluster_name, regional_stack.deployment_region)

1457 for regional_stack in self.regional_stacks

1458 ]

1459

1460 # GPU utilization percentage

1461 gpu_util_widget = cloudwatch.GraphWidget(

1462 title="GPU Utilization (%)",

1463 left=[

1464 cloudwatch.Metric(

1465 namespace="ContainerInsights",

1466 metric_name="node_gpu_utilization",

1467 dimensions_map={"ClusterName": cluster_name},

1468 statistic="Average",

1469 period=Duration.minutes(5),

1470 label=region,

1471 region=region,

1472 )

1473 for cluster_name, region in cluster_info

1474 ],

1475 width=12,

1476 height=6,

1477 )

1478 widgets.append(gpu_util_widget)

1479

1480 # GPU memory utilization

1481 gpu_mem_widget = cloudwatch.GraphWidget(

1482 title="GPU Memory Utilization (%)",

1483 left=[

1484 cloudwatch.Metric(

1485 namespace="ContainerInsights",

1486 metric_name="node_gpu_memory_utilization",

1487 dimensions_map={"ClusterName": cluster_name},

1488 statistic="Average",

1489 period=Duration.minutes(5),

1490 label=region,

1491 region=region,

1492 )

1493 for cluster_name, region in cluster_info

1494 ],

1495 width=12,

1496 height=6,

1497 )

1498 widgets.append(gpu_mem_widget)

1499

1500 # GPU temperature

1501 gpu_temp_widget = cloudwatch.GraphWidget(

1502 title="GPU Temperature (°C)",

1503 left=[

1504 cloudwatch.Metric(

1505 namespace="ContainerInsights",

1506 metric_name="node_gpu_temperature",

1507 dimensions_map={"ClusterName": cluster_name},

1508 statistic="Maximum",

1509 period=Duration.minutes(5),

1510 label=region,

1511 region=region,

1512 )

1513 for cluster_name, region in cluster_info

1514 ],

1515 width=12,

1516 height=6,

1517 )

1518 widgets.append(gpu_temp_widget)

1519

1520 # GPU count (active GPUs)

1521 gpu_count_widget = cloudwatch.GraphWidget(

1522 title="Active GPU Count",

1523 left=[

1524 cloudwatch.Metric(

1525 namespace="ContainerInsights",

1526 metric_name="node_gpu_limit",

1527 dimensions_map={"ClusterName": cluster_name},

1528 statistic="Sum",

1529 period=Duration.minutes(5),

1530 label=region,

1531 region=region,

1532 )

1533 for cluster_name, region in cluster_info

1534 ],

1535 width=12,

1536 height=6,

1537 )

1538 widgets.append(gpu_count_widget)

1539

1540 return widgets

1541

1542 def _create_fsx_widgets(self) -> list[cloudwatch.IWidget]:

1543 """Create FSx for Lustre monitoring widgets.

1544

1545 Only emits widgets for regions where the FSx file system is actually

1546 provisioned (``regional_stack.fsx_file_system`` is non-None). The

1547 dimension ``FileSystemId`` is the CDK-generated CloudFormation ref

1548 from each regional stack — CDK's ``cross_region_references=True``

1549 (enabled on this stack's constructor) plumbs the value across

1550 regions via SSM + custom resources.

1551

1552 Returns an empty list if no region has FSx enabled — the dashboard

1553 skips the section entirely.

1554 """

1555 # Collect (file_system_id, region) tuples for regions that have FSx on.

1556 # fsx_file_system is either a CfnFileSystem or None; the local

1557 # assignment + is-not-None check lets mypy narrow the type so

1558 # ``.ref`` access typechecks cleanly (a list comprehension with

1559 # the guard in the ``if`` clause does not narrow the value clause).

1560 fsx_info: list[tuple[str, str]] = []

1561 for regional_stack in self.regional_stacks:

1562 fsx = getattr(regional_stack, "fsx_file_system", None)

1563 if fsx is None:

1564 continue

1565 fsx_info.append((fsx.ref, regional_stack.deployment_region))

1566 if not fsx_info:

1567 return []

1568

1569 widgets: list[cloudwatch.IWidget] = []

1570

1571 # Section header

1572 widgets.append(

1573 cloudwatch.TextWidget(

1574 markdown=(

1575 "# FSx for Lustre\n"

1576 "Parallel file system throughput, IOPS, and free storage "

1577 "capacity. Each line below is scoped to the exact GCO "

1578 "file system in its region — so unrelated FSx file "

1579 "systems in the same account do not appear on the "

1580 "dashboard."

1581 ),

1582 width=24,

1583 height=1,

1584 )

1585 )

1586

1587 # Throughput: bytes read vs written

1588 throughput_widget = cloudwatch.GraphWidget(

1589 title="FSx - Throughput (Bytes/sec)",

1590 left=[

1591 cloudwatch.Metric(

1592 namespace="AWS/FSx",

1593 metric_name="DataReadBytes",

1594 dimensions_map={"FileSystemId": fs_id},

1595 statistic="Sum",

1596 period=Duration.minutes(1),

1597 label=f"{region} Read",

1598 region=region,

1599 )

1600 for fs_id, region in fsx_info

1601 ],

1602 right=[

1603 cloudwatch.Metric(

1604 namespace="AWS/FSx",

1605 metric_name="DataWriteBytes",

1606 dimensions_map={"FileSystemId": fs_id},

1607 statistic="Sum",

1608 period=Duration.minutes(1),

1609 label=f"{region} Write",

1610 region=region,

1611 )

1612 for fs_id, region in fsx_info

1613 ],

1614 width=12,

1615 height=6,

1616 )

1617 widgets.append(throughput_widget)

1618

1619 # IOPS: read vs write operations

1620 iops_widget = cloudwatch.GraphWidget(

1621 title="FSx - IOPS",

1622 left=[

1623 cloudwatch.Metric(

1624 namespace="AWS/FSx",

1625 metric_name="DataReadOperations",

1626 dimensions_map={"FileSystemId": fs_id},

1627 statistic="Sum",

1628 period=Duration.minutes(1),

1629 label=f"{region} Read",

1630 region=region,

1631 )

1632 for fs_id, region in fsx_info

1633 ],

1634 right=[

1635 cloudwatch.Metric(

1636 namespace="AWS/FSx",

1637 metric_name="DataWriteOperations",

1638 dimensions_map={"FileSystemId": fs_id},

1639 statistic="Sum",

1640 period=Duration.minutes(1),

1641 label=f"{region} Write",

1642 region=region,

1643 )

1644 for fs_id, region in fsx_info

1645 ],

1646 width=12,

1647 height=6,

1648 )

1649 widgets.append(iops_widget)

1650

1651 # Free storage capacity — the classic "running out of space" signal.

1652 # FreeDataStorageCapacity is emitted in bytes.

1653 free_storage_widget = cloudwatch.GraphWidget(

1654 title="FSx - Free Storage Capacity (Bytes)",

1655 left=[

1656 cloudwatch.Metric(

1657 namespace="AWS/FSx",

1658 metric_name="FreeDataStorageCapacity",

1659 dimensions_map={"FileSystemId": fs_id},

1660 statistic="Minimum",

1661 period=Duration.minutes(5),

1662 label=region,

1663 region=region,

1664 )

1665 for fs_id, region in fsx_info

1666 ],

1667 width=24,

1668 height=6,

1669 )

1670 widgets.append(free_storage_widget)

1671

1672 return widgets

1673

1674 def _create_valkey_widgets(self) -> list[cloudwatch.IWidget]:

1675 """Create Valkey (ElastiCache Serverless) monitoring widgets.

1676

1677 Uses explicit ``clusterId`` dimension values (camelCase — the

1678 ElastiCache Serverless variant; distinct from the node-based

1679 ``CacheClusterId``). The regional stack names its cache

1680 deterministically as ``gco-{deployment_region}``, so we reproduce

1681 that name here and pin each widget to the exact cache in its

1682 region. No SEARCH expression, so the dashboard ignores every

1683 unrelated ElastiCache cluster in the account.

1684 """

1685 valkey_enabled = self.config.get_valkey_config().get("enabled", False)

1686 if not valkey_enabled or not self.regions:

1687 return []

1688

1689 widgets: list[cloudwatch.IWidget] = []

1690

1691 widgets.append(

1692 cloudwatch.TextWidget(

1693 markdown=(

1694 "# Valkey Serverless Cache\n"

1695 "ECPU consumption, storage, hit rate, and request "

1696 "latency — scoped to each region's ``gco-{region}`` "

1697 "cache exactly (no SEARCH)."

1698 ),

1699 width=24,

1700 height=1,

1701 )

1702 )

1703

1704 # Build (cache_name, region) pairs. cache_name is the literal

1705 # ``serverless_cache_name`` the regional stack passes to the

1706 # CfnServerlessCache.

1707 cache_info = [(f"{self.project_name}-{region}", region) for region in self.regions]

1708

1709 # ECPU consumption and cache size per region

1710 for cache_name, region in cache_info:

1711 widgets.append(

1712 cloudwatch.GraphWidget(

1713 title=f"Valkey - ECPU & Cache Size ({region})",

1714 left=[

1715 cloudwatch.Metric(

1716 namespace="AWS/ElastiCache",

1717 metric_name="ElastiCacheProcessingUnits",

1718 dimensions_map={"clusterId": cache_name},

1719 statistic="Sum",

1720 period=Duration.minutes(1),

1721 label="ECPUs",

1722 region=region,

1723 ),

1724 ],

1725 right=[

1726 cloudwatch.Metric(

1727 namespace="AWS/ElastiCache",

1728 metric_name="BytesUsedForCache",

1729 dimensions_map={"clusterId": cache_name},

1730 statistic="Average",

1731 period=Duration.minutes(5),

1732 label="Bytes",

1733 region=region,

1734 ),

1735 ],

1736 width=12,

1737 height=6,

1738 region=region,

1739 )

1740 )

1741

1742 # Hit rate and p99 read/write latency per region

1743 for cache_name, region in cache_info:

1744 widgets.append(

1745 cloudwatch.GraphWidget(

1746 title=f"Valkey - Hit Rate & Latency ({region})",

1747 left=[

1748 cloudwatch.Metric(

1749 namespace="AWS/ElastiCache",

1750 metric_name="CacheHitRate",

1751 dimensions_map={"clusterId": cache_name},

1752 statistic="Average",

1753 period=Duration.minutes(5),

1754 label="Hit Rate %",

1755 region=region,

1756 ),

1757 ],

1758 right=[

1759 cloudwatch.Metric(

1760 namespace="AWS/ElastiCache",

1761 metric_name="SuccessfulReadRequestLatency",

1762 dimensions_map={"clusterId": cache_name},

1763 statistic="p99",

1764 period=Duration.minutes(1),

1765 label="Read p99 µs",

1766 region=region,

1767 ),

1768 cloudwatch.Metric(

1769 namespace="AWS/ElastiCache",

1770 metric_name="SuccessfulWriteRequestLatency",

1771 dimensions_map={"clusterId": cache_name},

1772 statistic="p99",

1773 period=Duration.minutes(1),

1774 label="Write p99 µs",

1775 region=region,

1776 ),

1777 ],

1778 width=12,

1779 height=6,

1780 region=region,

1781 )

1782 )

1783

1784 return widgets

1785

1786 def _create_aurora_pgvector_widgets(self) -> list[cloudwatch.IWidget]:

1787 """Create Aurora Serverless v2 (pgvector) monitoring widgets.

1788

1789 Pins each widget to the exact Aurora cluster provisioned by the

1790 regional stack via ``regional_stack.aurora_cluster.cluster_identifier``.

1791 CDK-generated cluster IDs are CloudFormation tokens; the

1792 ``cross_region_references=True`` flag on this stack handles

1793 plumbing them from each regional stack into the monitoring stack

1794 (us-east-2 by default) through SSM + custom resources.

1795

1796 Returns an empty list when every region has Aurora pgvector

1797 disabled so the dashboard skips the section entirely.

1798 """

1799 # (cluster_identifier, region) pairs for regions with Aurora on.

1800 # Use a guarded loop (not a comprehension) so mypy can narrow the

1801 # Optional[DatabaseCluster] to a real cluster before dereferencing.

1802 aurora_info: list[tuple[str, str]] = []

1803 for regional_stack in self.regional_stacks:

1804 aurora = getattr(regional_stack, "aurora_cluster", None)

1805 if aurora is None:

1806 continue

1807 aurora_info.append((aurora.cluster_identifier, regional_stack.deployment_region))

1808 if not aurora_info:

1809 return []

1810

1811 widgets: list[cloudwatch.IWidget] = []

1812

1813 widgets.append(

1814 cloudwatch.TextWidget(

1815 markdown=(

1816 "# Aurora pgvector (Serverless v2)\n"

1817 "ACU utilization, database connections, query latency, "

1818 "and CPU utilization — pinned to each regional GCO "

1819 "Aurora cluster by ID. ACU utilization is the primary "

1820 "scale/cost signal for Serverless v2."

1821 ),

1822 width=24,

1823 height=1,

1824 )

1825 )

1826

1827 # ACU utilization and capacity

1828 for cluster_id, region in aurora_info:

1829 widgets.append(

1830 cloudwatch.GraphWidget(

1831 title=f"Aurora - ACU Utilization & Capacity ({region})",

1832 left=[

1833 cloudwatch.Metric(

1834 namespace="AWS/RDS",

1835 metric_name="ACUUtilization",

1836 dimensions_map={"DBClusterIdentifier": cluster_id},

1837 statistic="Average",

1838 period=Duration.minutes(1),

1839 label="ACU %",

1840 region=region,

1841 ),

1842 ],

1843 right=[

1844 cloudwatch.Metric(

1845 namespace="AWS/RDS",

1846 metric_name="ServerlessDatabaseCapacity",

1847 dimensions_map={"DBClusterIdentifier": cluster_id},

1848 statistic="Average",

1849 period=Duration.minutes(1),

1850 label="ACUs",

1851 region=region,

1852 ),

1853 ],

1854 width=12,

1855 height=6,

1856 region=region,

1857 )

1858 )

1859

1860 # Database connections and CPU utilization

1861 for cluster_id, region in aurora_info:

1862 widgets.append(

1863 cloudwatch.GraphWidget(

1864 title=f"Aurora - Connections & CPU ({region})",

1865 left=[

1866 cloudwatch.Metric(

1867 namespace="AWS/RDS",

1868 metric_name="DatabaseConnections",

1869 dimensions_map={"DBClusterIdentifier": cluster_id},

1870 statistic="Average",

1871 period=Duration.minutes(1),

1872 label="Connections",

1873 region=region,

1874 ),

1875 ],

1876 right=[

1877 cloudwatch.Metric(

1878 namespace="AWS/RDS",

1879 metric_name="CPUUtilization",

1880 dimensions_map={"DBClusterIdentifier": cluster_id},

1881 statistic="Average",

1882 period=Duration.minutes(1),

1883 label="CPU %",

1884 region=region,

1885 ),

1886 ],

1887 width=12,

1888 height=6,

1889 region=region,

1890 )

1891 )

1892

1893 # Read and write latency p99

1894 for cluster_id, region in aurora_info:

1895 widgets.append(

1896 cloudwatch.GraphWidget(

1897 title=f"Aurora - Query Latency p99 ({region})",

1898 left=[

1899 cloudwatch.Metric(

1900 namespace="AWS/RDS",

1901 metric_name="ReadLatency",

1902 dimensions_map={"DBClusterIdentifier": cluster_id},

1903 statistic="p99",

1904 period=Duration.minutes(1),

1905 label="Read p99",

1906 region=region,

1907 ),

1908 ],

1909 right=[

1910 cloudwatch.Metric(

1911 namespace="AWS/RDS",

1912 metric_name="WriteLatency",

1913 dimensions_map={"DBClusterIdentifier": cluster_id},

1914 statistic="p99",

1915 period=Duration.minutes(1),

1916 label="Write p99",

1917 region=region,

1918 ),

1919 ],

1920 width=24,

1921 height=6,

1922 region=region,

1923 )

1924 )

1925

1926 return widgets

1927

1928 def _create_alb_widgets(self) -> list[cloudwatch.IWidget]:

1929 """Create ALB monitoring widgets scoped to the GCO platform ALB.

1930

1931 ALBs are created by the AWS Load Balancer Controller at runtime

1932 from an Ingress resource (not by CDK), so the exact ALB name

1933 isn't known at synth time. We originally tried reading the ARN

1934 off the regional stack's ``GaRegistration`` custom resource via

1935 ``cross_region_references=True``, but that path races the

1936 custom-resource response pipeline: CDK's cross-region

1937 ``ExportsWriter`` executes ``Fn::GetAtt: [GaRegistration, AlbArn]``

1938 before CloudFormation has the updated response data stored, and

1939 errors with "Vendor response doesn't contain AlbArn attribute".

1940

1941 Instead we use a SEARCH expression with a composite-token

1942 filter. The ALB Controller names the platform ALB

1943 ``k8s-gco-<hash>`` (the namespace is shortened because the

1944 controller enforces a 32-char total name limit); CloudWatch's

1945 ``LoadBalancer`` dimension is the ARN suffix ``app/<name>/<hash>``,

1946 so an unquoted filter ``LoadBalancer=app/k8s-gco-`` performs a

1947 composite-token match (the sequence ``app``, ``k``, ``8``, ``s``,

1948 ``gco`` must appear consecutively in the dimension value).

1949 Double-quoted filters would be exact matches and return nothing

1950 because no ALB's dimension value is literally ``app/k8s-gco-``.

1951 """

1952 widgets: list[cloudwatch.IWidget] = []

1953

1954 # Section header

1955 widgets.append(

1956 cloudwatch.TextWidget(

1957 markdown=(

1958 "# Application Load Balancers\n"

1959 "Request metrics, response time, HTTP errors, and "

1960 "connection counts — scoped via SEARCH composite-token "

1961 "match to ALBs named ``app/k8s-gco-*`` so only the GCO "

1962 "platform ALB in each region appears. Inference ALBs "

1963 "(named per endpoint) and unrelated ALBs in the "

1964 "account are excluded."

1965 ),

1966 width=24,

1967 height=1,

1968 )

1969 )

1970

1971 # Per-region request count

1972 for region in self.regions:

1973 widgets.append(

1974 cloudwatch.GraphWidget(

1975 title=f"ALB - Request Count ({region})",

1976 left=[

1977 cloudwatch.MathExpression(

1978 expression=(

1979 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "

1980 'MetricName="RequestCount" '

1981 'LoadBalancer=app/k8s-gco-\', "Sum", 300)'

1982 ),

1983 label="Request Count",

1984 period=Duration.minutes(5),

1985 ),

1986 ],

1987 width=12,

1988 height=6,

1989 region=region,

1990 )

1991 )

1992

1993 # Per-region response time (average and p99)

1994 for region in self.regions:

1995 widgets.append(

1996 cloudwatch.GraphWidget(

1997 title=f"ALB - Response Time ({region})",

1998 left=[

1999 cloudwatch.MathExpression(

2000 expression=(

2001 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "

2002 'MetricName="TargetResponseTime" '

2003 'LoadBalancer=app/k8s-gco-\', "Average", 300)'

2004 ),

2005 label="Avg Response Time",

2006 period=Duration.minutes(5),

2007 ),

2008 cloudwatch.MathExpression(

2009 expression=(

2010 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "

2011 'MetricName="TargetResponseTime" '

2012 'LoadBalancer=app/k8s-gco-\', "p99", 300)'

2013 ),

2014 label="p99 Response Time",

2015 period=Duration.minutes(5),

2016 ),

2017 ],

2018 width=12,

2019 height=6,

2020 region=region,

2021 )

2022 )

2023

2024 # Per-region HTTP errors (4XX + 5XX from targets)

2025 for region in self.regions:

2026 widgets.append(

2027 cloudwatch.GraphWidget(

2028 title=f"ALB - HTTP Errors ({region})",

2029 left=[

2030 cloudwatch.MathExpression(

2031 expression=(

2032 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "

2033 'MetricName="HTTPCode_Target_4XX_Count" '

2034 'LoadBalancer=app/k8s-gco-\', "Sum", 300)'

2035 ),

2036 label="4XX Errors",

2037 period=Duration.minutes(5),

2038 ),

2039 ],

2040 right=[

2041 cloudwatch.MathExpression(

2042 expression=(

2043 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "

2044 'MetricName="HTTPCode_Target_5XX_Count" '

2045 'LoadBalancer=app/k8s-gco-\', "Sum", 300)'

2046 ),

2047 label="5XX Errors",

2048 period=Duration.minutes(5),

2049 ),

2050 ],

2051 width=12,

2052 height=6,

2053 region=region,

2054 )

2055 )

2056

2057 # Per-region active connections

2058 for region in self.regions:

2059 widgets.append(

2060 cloudwatch.GraphWidget(

2061 title=f"ALB - Active Connections ({region})",

2062 left=[

2063 cloudwatch.MathExpression(

2064 expression=(

2065 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "

2066 'MetricName="ActiveConnectionCount" '

2067 'LoadBalancer=app/k8s-gco-\', "Sum", 300)'

2068 ),

2069 label="Active Connections",

2070 period=Duration.minutes(5),

2071 ),

2072 ],

2073 width=12,

2074 height=6,

2075 region=region,

2076 )

2077 )

2078

2079 return widgets

2080

2081 def _create_application_widgets(self) -> list[cloudwatch.IWidget]:

2082 """Create custom application monitoring widgets"""

2083 widgets: list[cloudwatch.IWidget] = []

2084

2085 # Section header

2086 widgets.append(

2087 cloudwatch.TextWidget(

2088 markdown="# Application Metrics\n"

2089 "Health monitor and manifest processor metrics. "

2090 "Application logs are available in Container Insights at "

2091 "`/aws/containerinsights/<cluster>/application`.",

2092 width=24,

2093 height=1,

2094 )

2095 )

2096

2097 # Build cluster info from regional stacks: (cluster_name, region)

2098 cluster_info = [

2099 (regional_stack.cluster.cluster_name, regional_stack.deployment_region)

2100 for regional_stack in self.regional_stacks

2101 ]

2102

2103 # Health monitor metrics

2104 health_monitor_widget = cloudwatch.GraphWidget(

2105 title="Health Monitor - Resource Utilization",

2106 left=[

2107 cloudwatch.Metric(

2108 namespace="GCO/HealthMonitor",

2109 metric_name="ClusterCpuUtilization",

2110 dimensions_map={

2111 "ClusterName": cluster_name,

2112 "Region": region,

2113 },

2114 statistic="Average",

2115 period=Duration.minutes(5),

2116 label=f"{region} CPU",

2117 region=region,

2118 )

2119 for cluster_name, region in cluster_info

2120 ],

2121 right=[

2122 cloudwatch.Metric(

2123 namespace="GCO/HealthMonitor",

2124 metric_name="ClusterMemoryUtilization",

2125 dimensions_map={

2126 "ClusterName": cluster_name,

2127 "Region": region,

2128 },

2129 statistic="Average",

2130 period=Duration.minutes(5),

2131 label=f"{region} Memory",

2132 region=region,

2133 )

2134 for cluster_name, region in cluster_info

2135 ],

2136 width=12,

2137 height=6,

2138 )

2139 widgets.append(health_monitor_widget)

2140

2141 # Manifest processor metrics

2142 manifest_processor_widget = cloudwatch.GraphWidget(

2143 title="Manifest Processor - Submissions",

2144 left=[

2145 cloudwatch.Metric(

2146 namespace="GCO/ManifestProcessor",

2147 metric_name="ManifestSubmissions",

2148 dimensions_map={

2149 "ClusterName": cluster_name,

2150 "Region": region,

2151 },

2152 statistic="Sum",

2153 period=Duration.minutes(5),

2154 label=f"{region} Submissions",

2155 region=region,

2156 )

2157 for cluster_name, region in cluster_info

2158 ],

2159 right=[

2160 cloudwatch.Metric(

2161 namespace="GCO/ManifestProcessor",

2162 metric_name="ManifestFailures",

2163 dimensions_map={

2164 "ClusterName": cluster_name,

2165 "Region": region,

2166 },

2167 statistic="Sum",

2168 period=Duration.minutes(5),

2169 label=f"{region} Failures",

2170 color="#d62728",

2171 region=region,

2172 )

2173 for cluster_name, region in cluster_info

2174 ],

2175 width=12,

2176 height=6,

2177 )

2178 widgets.append(manifest_processor_widget)

2179

2180 # Container Insights - Pod restarts (indicates application issues)

2181 pod_restarts_widget = cloudwatch.GraphWidget(

2182 title="Container Insights - Pod Restarts",

2183 left=[

2184 cloudwatch.Metric(

2185 namespace="ContainerInsights",

2186 metric_name="pod_number_of_container_restarts",

2187 dimensions_map={"ClusterName": cluster_name},

2188 statistic="Sum",

2189 period=Duration.minutes(5),

2190 label=f"{region}",

2191 region=region,

2192 )

2193 for cluster_name, region in cluster_info

2194 ],

2195 width=12,

2196 height=6,

2197 )

2198 widgets.append(pod_restarts_widget)

2199

2200 # Secret rotation Lambda metrics (Secrets Manager doesn't publish rotation metrics,

2201 # so we monitor the rotation Lambda function instead)

2202 if self.api_gateway_stack:

2203 rotation_function_name = self.api_gateway_stack.rotation_lambda.function_name

2204 api_gw_region = self.config.get_api_gateway_region()

2205

2206 rotation_widget = cloudwatch.GraphWidget(

2207 title="Secret Rotation Lambda - Invocations & Errors",

2208 left=[

2209 cloudwatch.Metric(

2210 namespace="AWS/Lambda",

2211 metric_name="Invocations",

2212 dimensions_map={"FunctionName": rotation_function_name},

2213 statistic="Sum",

2214 period=Duration.hours(1),

2215 label="Invocations",

2216 color="#2ca02c",

2217 region=api_gw_region,

2218 ),

2219 ],

2220 right=[

2221 cloudwatch.Metric(

2222 namespace="AWS/Lambda",

2223 metric_name="Errors",

2224 dimensions_map={"FunctionName": rotation_function_name},

2225 statistic="Sum",

2226 period=Duration.hours(1),

2227 label="Errors",

2228 color="#d62728",

2229 region=api_gw_region,

2230 ),

2231 ],

2232 width=12,

2233 height=6,

2234 )

2235 widgets.append(rotation_widget)

2236 else:

2237 # Fallback text widget if api_gateway_stack not available

2238 fallback_widget = cloudwatch.TextWidget(

2239 markdown="**Secret Rotation:** API Gateway stack not configured. "

2240 "Rotation Lambda metrics unavailable.",

2241 width=12,

2242 height=6,

2243 )

2244 widgets.append(fallback_widget)

2245

2246 return widgets

2247

2248 def _create_alarms(self) -> None:

2249 """Create CloudWatch alarms"""

2250 self._create_global_accelerator_alarms()

2251 self._create_api_gateway_alarms()

2252 self._create_lambda_alarms()

2253 self._create_sqs_alarms()

2254 self._create_dynamodb_alarms()

2255 self._create_eks_alarms()

2256 self._create_alb_alarms()

2257 self._create_application_alarms()

2258

2259 def _create_global_accelerator_alarms(self) -> None:

2260 """Create Global Accelerator alarms.

2261

2262 Note: Global Accelerator metrics are only available in us-west-2.

2263 CloudWatch Alarms must be in the same region as the metrics they monitor.

2264 Since this monitoring stack may be deployed in a different region,

2265 we skip GA alarms here. To monitor GA, either:

2266 1. Create alarms manually in us-west-2

2267 2. Use CloudWatch cross-region dashboard widgets (which we do)

2268 3. Deploy a separate alarm stack in us-west-2

2269 """

2270 # GA alarms skipped - metrics only available in us-west-2

2271 # Dashboard widgets use region parameter to display GA metrics correctly

2272

2273 def _create_api_gateway_alarms(self) -> None:

2274 """Create API Gateway alarms"""

2275 # Get the actual API name from the api_gateway_stack

2276 api_name = (

2277 self.api_gateway_stack.api.rest_api_name

2278 if self.api_gateway_stack

2279 else f"{self.project_name}-global-api"

2280 )

2281

2282 # High 5XX error rate

2283 api_5xx_alarm = cloudwatch.Alarm(

2284 self,

2285 "ApiGateway5xxAlarm",

2286 alarm_description="API Gateway has high 5XX error rate",

2287 metric=cloudwatch.Metric(

2288 namespace="AWS/ApiGateway",

2289 metric_name="5XXError",

2290 dimensions_map={"ApiName": api_name},

2291 statistic="Sum",

2292 period=Duration.minutes(5),

2293 ),

2294 threshold=10,

2295 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,

2296 evaluation_periods=2,

2297 datapoints_to_alarm=2,

2298 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2299 )

2300 api_5xx_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))

2301

2302 # High latency

2303 api_latency_alarm = cloudwatch.Alarm(

2304 self,

2305 "ApiGatewayHighLatencyAlarm",

2306 alarm_description="API Gateway has high latency",

2307 metric=cloudwatch.Metric(

2308 namespace="AWS/ApiGateway",

2309 metric_name="Latency",

2310 dimensions_map={"ApiName": api_name},

2311 statistic="p99",

2312 period=Duration.minutes(5),

2313 ),

2314 threshold=10000, # 10 seconds

2315 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,

2316 evaluation_periods=3,

2317 datapoints_to_alarm=2,

2318 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2319 )

2320 api_latency_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))

2321

2322 def _create_lambda_alarms(self) -> None:

2323 """Create Lambda function alarms."""

2324 if self.api_gateway_stack is None:

2325 return

2326

2327 proxy_lambda = self.api_gateway_stack.proxy_lambda

2328 if proxy_lambda is not None:

2329 proxy_function_name = proxy_lambda.function_name

2330 proxy_errors_alarm = cloudwatch.Alarm(

2331 self,

2332 "ProxyLambdaErrorsAlarm",

2333 alarm_description="API Gateway proxy Lambda has errors",

2334 metric=cloudwatch.Metric(

2335 namespace="AWS/Lambda",

2336 metric_name="Errors",

2337 dimensions_map={"FunctionName": proxy_function_name},

2338 statistic="Sum",

2339 period=Duration.minutes(5),

2340 ),

2341 threshold=5,

2342 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,

2343 evaluation_periods=2,

2344 datapoints_to_alarm=2,

2345 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2346 )

2347 proxy_errors_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))

2348

2349 proxy_throttles_alarm = cloudwatch.Alarm(

2350 self,

2351 "ProxyLambdaThrottlesAlarm",

2352 alarm_description="API Gateway proxy Lambda is being throttled",

2353 metric=cloudwatch.Metric(

2354 namespace="AWS/Lambda",

2355 metric_name="Throttles",

2356 dimensions_map={"FunctionName": proxy_function_name},

2357 statistic="Sum",

2358 period=Duration.minutes(5),

2359 ),

2360 threshold=1,

2361 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,

2362 evaluation_periods=2,

2363 datapoints_to_alarm=2,

2364 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2365 )

2366 proxy_throttles_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))

2367

2368 rotation_errors_alarm = cloudwatch.Alarm(

2369 self,

2370 "RotationLambdaErrorsAlarm",

2371 alarm_description="Secret rotation Lambda has errors",

2372 metric=cloudwatch.Metric(

2373 namespace="AWS/Lambda",

2374 metric_name="Errors",

2375 dimensions_map={

2376 "FunctionName": self.api_gateway_stack.rotation_lambda.function_name

2377 },

2378 statistic="Sum",

2379 period=Duration.hours(1),

2380 ),

2381 threshold=1,

2382 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,

2383 evaluation_periods=1,

2384 datapoints_to_alarm=1,

2385 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2386 )

2387 rotation_errors_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))

2388

2389 def _create_sqs_alarms(self) -> None:

2390 """Create SQS queue alarms"""

2391 for regional_stack in self.regional_stacks:

2392 region = regional_stack.deployment_region

2393 queue_name = regional_stack.job_queue.queue_name

2394 dlq_name = regional_stack.job_dlq.queue_name

2395 region_id = region.replace("-", "").title()

2396

2397 # Old message alarm (stuck jobs)

2398 old_message_alarm = cloudwatch.Alarm(

2399 self,

2400 f"SqsOldMessageAlarm{region_id}",

2401 alarm_description=f"SQS queue in {region} has old messages (potential stuck jobs)",

2402 metric=cloudwatch.Metric(

2403 namespace="AWS/SQS",

2404 metric_name="ApproximateAgeOfOldestMessage",

2405 dimensions_map={"QueueName": queue_name},

2406 statistic="Maximum",

2407 period=Duration.minutes(5),

2408 ),

2409 threshold=3600, # 1 hour

2410 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,

2411 evaluation_periods=2,

2412 datapoints_to_alarm=2,

2413 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2414 )

2415 old_message_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))

2416

2417 # Dead letter queue alarm

2418 dlq_alarm = cloudwatch.Alarm(

2419 self,

2420 f"SqsDlqAlarm{region_id}",

2421 alarm_description=f"SQS dead letter queue in {region} has messages",

2422 metric=cloudwatch.Metric(

2423 namespace="AWS/SQS",

2424 metric_name="ApproximateNumberOfMessagesVisible",

2425 dimensions_map={"QueueName": dlq_name},

2426 statistic="Sum",

2427 period=Duration.minutes(5),

2428 ),

2429 threshold=1,

2430 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,

2431 evaluation_periods=1,

2432 datapoints_to_alarm=1,

2433 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2434 )

2435 dlq_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))

2436

2437 def _create_dynamodb_alarms(self) -> None:

2438 """Create DynamoDB alarms for job queue, templates, and webhooks tables."""

2439 # Get table names from global stack

2440 jobs_table = self.global_stack.jobs_table.table_name

2441

2442 # DynamoDB tables are in the global region

2443 global_region = self.config.get_global_region()

2444

2445 # Jobs table throttling alarm

2446 jobs_throttle_alarm = cloudwatch.Alarm(

2447 self,

2448 "DynamoDBJobsThrottleAlarm",

2449 alarm_description="DynamoDB jobs table is being throttled",

2450 metric=cloudwatch.Metric(

2451 namespace="AWS/DynamoDB",

2452 metric_name="ThrottledRequests",

2453 dimensions_map={"TableName": jobs_table},

2454 statistic="Sum",

2455 period=Duration.minutes(5),

2456 region=global_region,

2457 ),

2458 threshold=1,

2459 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,

2460 evaluation_periods=2,

2461 datapoints_to_alarm=2,

2462 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2463 )

2464 jobs_throttle_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))

2465

2466 # Jobs table system errors alarm

2467 jobs_errors_alarm = cloudwatch.Alarm(

2468 self,

2469 "DynamoDBJobsErrorsAlarm",

2470 alarm_description="DynamoDB jobs table has system errors",

2471 metric=cloudwatch.Metric(

2472 namespace="AWS/DynamoDB",

2473 metric_name="SystemErrors",

2474 dimensions_map={"TableName": jobs_table},

2475 statistic="Sum",

2476 period=Duration.minutes(5),

2477 region=global_region,

2478 ),

2479 threshold=1,

2480 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,

2481 evaluation_periods=1,

2482 datapoints_to_alarm=1,

2483 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2484 )

2485 jobs_errors_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))

2486

2487 def _create_eks_alarms(self) -> None:

2488 """Create EKS cluster alarms"""

2489 for regional_stack in self.regional_stacks:

2490 region = regional_stack.deployment_region

2491 cluster_name = regional_stack.cluster.cluster_name

2492 region_id = region.replace("-", "").title()

2493

2494 # High CPU utilization alarm (node-level metric)

2495 high_cpu_alarm = cloudwatch.Alarm(

2496 self,

2497 f"EksHighCpuAlarm{region_id}",

2498 alarm_description=f"EKS cluster {cluster_name} has high CPU utilization",

2499 metric=cloudwatch.Metric(

2500 namespace="ContainerInsights",

2501 metric_name="node_cpu_utilization",

2502 dimensions_map={"ClusterName": cluster_name},

2503 statistic="Average",

2504 period=Duration.minutes(5),

2505 ),

2506 threshold=80,

2507 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,

2508 evaluation_periods=3,

2509 datapoints_to_alarm=2,

2510 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2511 )

2512 high_cpu_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))

2513

2514 # High memory utilization alarm (node-level metric)

2515 high_memory_alarm = cloudwatch.Alarm(

2516 self,

2517 f"EksHighMemoryAlarm{region_id}",

2518 alarm_description=f"EKS cluster {cluster_name} has high memory utilization",

2519 metric=cloudwatch.Metric(

2520 namespace="ContainerInsights",

2521 metric_name="node_memory_utilization",

2522 dimensions_map={"ClusterName": cluster_name},

2523 statistic="Average",

2524 period=Duration.minutes(5),

2525 ),

2526 threshold=85,

2527 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,

2528 evaluation_periods=3,

2529 datapoints_to_alarm=2,

2530 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2531 )

2532 high_memory_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))

2533

2534 def _create_alb_alarms(self) -> None:

2535 """Create ALB alarms.

2536

2537 Status: no alarms created yet, even though we now have the ALB

2538 ARN at deploy time via the GA registration custom resource (which

2539 also feeds the dashboard widgets). Adding per-ALB alarms here is

2540 a straightforward enhancement — derive the ``LoadBalancer``

2541 dimension the same way ``_create_alb_widgets`` does

2542 (``Fn.split(":loadbalancer/", alb_arn)[1]``) and wire it into

2543 ``cloudwatch.Alarm`` constructs.

2544

2545 For now we rely on:

2546 1. Dashboard widgets pinned to each platform ALB (see

2547 ``_create_alb_widgets``)

2548 2. EKS Container Insights alarms for pod/node health

2549 3. API Gateway alarms for request-level monitoring

2550 """

2551 # TODO: Add UnHealthyHostCount / 5XXCount alarms using the ARN

2552 # returned by regional_stack.ga_registration.get_att_string("AlbArn").

2553 # The test suite explicitly documents that the ALB alarm count is

2554 # currently zero (test_alb_unhealthy_hosts_alarm_skipped); update

2555 # that test when adding real alarms.

2556

2557 def _create_application_alarms(self) -> None:

2558 """Create application-specific alarms"""

2559 for regional_stack in self.regional_stacks:

2560 region = regional_stack.deployment_region

2561 cluster_name = regional_stack.cluster.cluster_name

2562 region_id = region.replace("-", "").title()

2563

2564 # High manifest failure rate alarm

2565 high_failure_rate_alarm = cloudwatch.Alarm(

2566 self,

2567 f"ManifestHighFailureRateAlarm{region_id}",

2568 alarm_description=f"Manifest processor in {region} has high failure rate",

2569 metric=cloudwatch.Metric(

2570 namespace="GCO/ManifestProcessor",

2571 metric_name="ManifestFailures",

2572 dimensions_map={"ClusterName": cluster_name, "Region": region},

2573 statistic="Sum",

2574 period=Duration.minutes(5),

2575 ),

2576 threshold=10,

2577 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,

2578 evaluation_periods=2,

2579 datapoints_to_alarm=2,

2580 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2581 )

2582 high_failure_rate_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))

2583

2584 def _create_composite_alarms(self) -> None:

2585 """Create composite alarms for better signal-to-noise ratio"""

2586

2587 # Store individual alarms for composite alarm references

2588 regional_alarms: dict[str, list[cloudwatch.Alarm]] = {}

2589

2590 for regional_stack in self.regional_stacks:

2591 region = regional_stack.deployment_region

2592 cluster_name = regional_stack.cluster.cluster_name

2593 region_id = region.replace("-", "").title()

2594 regional_alarms[region] = []

2595

2596 # Create regional health composite alarm

2597 # Triggers when multiple issues occur in the same region

2598 eks_cpu_alarm = cloudwatch.Alarm(

2599 self,

2600 f"CompositeEksCpu{region_id}",

2601 metric=cloudwatch.Metric(

2602 namespace="ContainerInsights",

2603 metric_name="node_cpu_utilization",

2604 dimensions_map={"ClusterName": cluster_name},

2605 statistic="Average",

2606 period=Duration.minutes(5),

2607 ),

2608 threshold=90,

2609 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,

2610 evaluation_periods=2,

2611 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2612 )

2613 regional_alarms[region].append(eks_cpu_alarm)

2614

2615 eks_memory_alarm = cloudwatch.Alarm(

2616 self,

2617 f"CompositeEksMemory{region_id}",

2618 metric=cloudwatch.Metric(

2619 namespace="ContainerInsights",

2620 metric_name="node_memory_utilization",

2621 dimensions_map={"ClusterName": cluster_name},

2622 statistic="Average",

2623 period=Duration.minutes(5),

2624 ),

2625 threshold=90,

2626 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,

2627 evaluation_periods=2,

2628 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2629 )

2630 regional_alarms[region].append(eks_memory_alarm)

2631

2632 # Create composite alarm for critical regional issues. Every region

2633 # above appends exactly the CPU and memory alarms (two), so this

2634 # always fires; no `len(alarms) >= 2` guard is needed here.

2635 for region, alarms in regional_alarms.items():

2636 region_id = region.replace("-", "").title()

2637 composite_alarm = cloudwatch.CompositeAlarm(

2638 self,

2639 f"RegionalCriticalAlarm{region_id}",

2640 alarm_description=f"Critical: Multiple issues detected in {region}",

2641 alarm_rule=cloudwatch.AlarmRule.all_of(*alarms),

2642 )

2643 composite_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))

2644

2645 # API Gateway + Lambda composite alarm (only if api_gateway_stack is available)

2646 if self.api_gateway_stack and self.api_gateway_stack.proxy_lambda is not None:

2647 api_name = self.api_gateway_stack.api.rest_api_name

2648 proxy_function_name = self.api_gateway_stack.proxy_lambda.function_name

2649

2650 api_error_alarm = cloudwatch.Alarm(

2651 self,

2652 "CompositeApiErrors",

2653 metric=cloudwatch.Metric(

2654 namespace="AWS/ApiGateway",

2655 metric_name="5XXError",

2656 dimensions_map={"ApiName": api_name},

2657 statistic="Sum",

2658 period=Duration.minutes(5),

2659 ),

2660 threshold=5,

2661 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,

2662 evaluation_periods=2,

2663 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2664 )

2665

2666 lambda_error_alarm = cloudwatch.Alarm(

2667 self,

2668 "CompositeLambdaErrors",

2669 metric=cloudwatch.Metric(

2670 namespace="AWS/Lambda",

2671 metric_name="Errors",

2672 dimensions_map={"FunctionName": proxy_function_name},

2673 statistic="Sum",

2674 period=Duration.minutes(5),

2675 ),

2676 threshold=3,

2677 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,

2678 evaluation_periods=2,

2679 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,

2680 )

2681

2682 api_lambda_composite = cloudwatch.CompositeAlarm(

2683 self,

2684 "ApiLambdaCompositeAlarm",

2685 alarm_description="Critical: Both API Gateway and Lambda proxy have errors",

2686 alarm_rule=cloudwatch.AlarmRule.all_of(api_error_alarm, lambda_error_alarm),

2687 )

2688 api_lambda_composite.add_alarm_action(cw_actions.SnsAction(self.alert_topic))

2689

2690 def _create_custom_metrics(self) -> None:

2691 """Create custom metric filters and log groups"""

2692 for regional_stack in self.regional_stacks:

2693 region = regional_stack.deployment_region

2694 region_id = region.replace("-", "").title()

2695

2696 # Health monitor log group

2697 # log_group_name intentionally omitted - let CDK generate unique name

2698 logs.LogGroup(

2699 self,

2700 f"HealthMonitorLogGroup{region_id}",

2701 retention=logs.RetentionDays.ONE_MONTH,

2702 removal_policy=RemovalPolicy.DESTROY,

2703 )

2704

2705 # Manifest processor log group

2706 # log_group_name intentionally omitted - let CDK generate unique name

2707 logs.LogGroup(

2708 self,

2709 f"ManifestProcessorLogGroup{region_id}",

2710 retention=logs.RetentionDays.ONE_MONTH,

2711 removal_policy=RemovalPolicy.DESTROY,

2712 )

2713

2714 def _create_outputs(self) -> None:

2715 """Create CloudFormation outputs"""

2716 CfnOutput(

2717 self,

2718 "DashboardUrl",

2719 value=f"https://console.aws.amazon.com/cloudwatch/home?region={self.region}#dashboards:name={self.dashboard.dashboard_name}",

2720 description="CloudWatch Dashboard URL",

2721 )

2722

2723 CfnOutput(

2724 self,

2725 "AlertTopicArn",

2726 value=self.alert_topic.topic_arn,

2727 description="SNS Topic ARN for monitoring alerts",

2728 )

2729

2730 CfnOutput(

2731 self,

2732 "AlarmCount",

2733 value="See CloudWatch Alarms console for full list",

2734 description="Monitoring alarms created",

2735 )

Coverage for gco / stacks / monitoring_stack.py: 100.00%

368 statements