Coverage for gco / stacks / monitoring_stack.py: 100.00%
365 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-09-14 22:07 +0000
1"""
2Monitoring stack for GCO (Global Capacity Orchestrator on AWS) - Cross-region monitoring and observability.
4This stack creates centralized monitoring resources for all GCO deployments:
5- CloudWatch Dashboard with comprehensive widgets for all regions
6- SNS topic for alerting
7- CloudWatch Alarms for critical metrics
8- Log groups for application logs
9- Anomaly detection for traffic patterns
10- Composite alarms for better signal-to-noise
12Dashboard Sections:
13- Global Accelerator: Flow counts, processed bytes
14- API Gateway: Request counts, latency, error rates
15- Lambda Functions: Invocations, errors, duration, throttles
16- SQS Queues: Message counts, age, dead letter queue depth
17- DynamoDB Tables: Capacity, latency, throttles, errors
18- EKS Clusters: CPU/memory utilization per region
19- FSx for Lustre (when enabled): Throughput, IOPS, free storage
20- Valkey Serverless (when enabled): ECPU, hit rate, latency, bytes used
21- Aurora pgvector (when enabled): ACU utilization, connections, latency, CPU
22- ALBs: Request counts, response times, healthy hosts
23- Applications: Custom metrics from health monitor and manifest processor
25Cross-Region Metrics:
26 CloudWatch metrics are region-specific. This stack handles cross-region
27 monitoring by specifying the `region` parameter on metrics:
28 - Global Accelerator metrics: Always in us-west-2
29 - DynamoDB metrics: In the global region (where tables are deployed)
30 - Regional metrics: In each cluster's region
32Alarms:
33- High CPU/memory utilization on EKS clusters
34- Unhealthy hosts in ALB target groups
35- High response times
36- Manifest processing failures
37- Lambda errors and throttles
38- SQS message age (stuck jobs)
39- DynamoDB throttling and system errors
40- API Gateway 5XX errors
41- Secret rotation failures
42"""
44from typing import TYPE_CHECKING, Any
46from aws_cdk import (
47 CfnOutput,
48 Duration,
49 RemovalPolicy,
50 Stack,
51)
52from aws_cdk import aws_athena as athena
53from aws_cdk import aws_cloudwatch as cloudwatch
54from aws_cdk import aws_cloudwatch_actions as cw_actions
55from aws_cdk import aws_glue as glue
56from aws_cdk import aws_iam as iam
57from aws_cdk import aws_kms as kms
58from aws_cdk import aws_logs as logs
59from aws_cdk import aws_s3 as s3
60from aws_cdk import aws_sns as sns
61from aws_cdk import aws_ssm as ssm
62from constructs import Construct
64from gco.config.config_loader import ConfigLoader
65from gco.stacks.constants import (
66 COST_ATHENA_RESULTS_PREFIX,
67 COST_GLUE_ALLOCATION_TABLE,
68 COST_REPORT_SCHEDULED_PREFIX,
69 cost_athena_workgroup_name,
70 cost_glue_database_name,
71 cost_report_ssm_parameter_prefix,
72)
74# <pyflowchart-code-diagram> BEGIN - auto-inserted, do not edit
75# Generated at (UTC): 2026-09-12T06:04:03Z
76# Generated from Git commit: e96e2c39c3626a5088651f43873dfade6a346850
77# Flowchart(s) generated from this file:
78# * ``GCOMonitoringStack.__init__`` -> ``diagrams/code_diagrams/gco/stacks/monitoring_stack.GCOMonitoringStack___init__.html``
79# (PNG: ``diagrams/code_diagrams/gco/stacks/monitoring_stack.GCOMonitoringStack___init__.png``)
80# Regenerate with ``SOURCE_DATE_EPOCH=<unix-seconds> GCO_DIAGRAM_SOURCE_COMMIT=<40-char-sha> python diagrams/generate.py --code-only``.
81# <pyflowchart-code-diagram> END
84if TYPE_CHECKING:
85 from gco.stacks.api_gateway_global_stack import GCOApiGatewayGlobalStack
86 from gco.stacks.global_stack import GCOGlobalStack
87 from gco.stacks.regional_stack import GCORegionalStack
90class GCOMonitoringStack(Stack):
91 """
92 Cross-region monitoring and observability stack.
94 Creates a centralized CloudWatch dashboard and alarms that aggregate
95 metrics from all regional deployments.
97 Attributes:
98 alert_topic: SNS topic for alarm notifications
99 dashboard: CloudWatch dashboard with all monitoring widgets
100 """
102 def __init__(
103 self,
104 scope: Construct,
105 construct_id: str,
106 config: ConfigLoader,
107 global_stack: GCOGlobalStack,
108 regional_stacks: list[GCORegionalStack],
109 api_gateway_stack: GCOApiGatewayGlobalStack | None = None,
110 **kwargs: Any,
111 ) -> None:
112 # Enable CDK's native cross-region references. The monitoring stack
113 # lives in the monitoring region (by default us-east-2) and needs
114 # resource identifiers from the regional stacks for dashboard
115 # dimensions — specifically the auto-generated FSx file system IDs,
116 # whose values aren't known until deploy time.
117 #
118 # CDK implements this by provisioning a small Lambda-backed custom
119 # resource in each source stack that writes the referenced value to
120 # an SSM parameter in the target region, plus a reader custom
121 # resource in the target stack. Cost is negligible (the Lambdas run
122 # once per deploy) and the pattern is the documented canonical
123 # answer for ``CrossRegionReferencesNotEnabled`` errors.
124 kwargs.setdefault("cross_region_references", True)
125 super().__init__(scope, construct_id, **kwargs)
127 self.config = config
128 self.global_stack = global_stack
129 self.regional_stacks = regional_stacks
130 self.api_gateway_stack = api_gateway_stack
131 self.project_name = config.get_project_name()
132 self.regions = config.get_regions()
134 # Create SNS topic for alerts
135 self.alert_topic = self._create_alert_topic()
137 # Cost monitoring pipeline (on by default): the cost report bucket the
138 # regional cost-monitor services write Parquet allocation reports to,
139 # plus the Glue database/table and Athena workgroup that make the
140 # cross-region data queryable from the CLI.
141 if self.config.get_cost_monitoring_enabled():
142 self._create_cost_report_storage()
143 self._create_cost_analytics()
145 # Create CloudWatch dashboard
146 self.dashboard = self._create_dashboard()
148 # Create alarms
149 self._create_alarms()
151 # Create composite alarms
152 self._create_composite_alarms()
154 # Create custom metrics
155 self._create_custom_metrics()
157 # Export monitoring resources
158 self._create_outputs()
160 # Apply cdk-nag suppressions
161 self._apply_nag_suppressions()
163 def _apply_nag_suppressions(self) -> None:
164 """Apply cdk-nag suppressions for this stack."""
165 from gco.stacks.nag_suppressions import apply_all_suppressions
167 apply_all_suppressions(
168 self,
169 stack_type="monitoring",
170 regions=self.config.get_regions(),
171 global_region=self.config.get_global_region(),
172 project_name=self.project_name,
173 )
175 def _create_alert_topic(self) -> sns.Topic:
176 """Create SNS topic for monitoring alerts"""
177 topic = sns.Topic(
178 self,
179 "GCOAlertTopic",
180 display_name="GCO (Global Capacity Orchestrator on AWS) Monitoring Alerts",
181 enforce_ssl=True,
182 )
183 return topic
185 def _create_cost_report_storage(self) -> None:
186 """Create the central cost report bucket for the cost monitoring pipeline.
188 Every regional cost-monitor service writes Hive-partitioned Parquet
189 allocation reports here:
191 - ``reports/region=<region>/date=<YYYY-MM-DD>/...`` — scheduled
192 reports; the Glue table's partition projection reads this layout.
193 - ``adhoc/region=<region>/date=<YYYY-MM-DD>/...`` — user-requested
194 reports, kept out of the scheduled table so overlapping windows can
195 never double-count in Athena aggregations.
196 - ``athena-results/`` — Athena query results for the cost workgroup.
198 Three constructs mirror the regional-shared bucket pattern:
200 1. ``cost_report_kms_key`` — customer-managed KMS key with annual
201 rotation and a 7-day pending window on destroy. Its key policy
202 grants every regional cost-monitor role ``GenerateDataKey`` /
203 ``Decrypt`` / ``DescribeKey`` through S3 in this region.
204 2. ``cost_report_access_logs_bucket`` — the dedicated S3 access-logs
205 destination for the primary bucket.
206 3. ``cost_report_bucket`` — the primary bucket. Its physical name is
207 CloudFormation-generated: S3 bucket names are a global namespace
208 and a deleted name is not reliably reusable, so the previous fixed
209 ``<project>-cost-reports-<account>-<region>`` name made every
210 destroy-and-redeploy a collision hazard (and did fail one). The
211 regional stacks deploy *before* this stack, so instead of granting
212 by reconstructed ARN they are granted here, principal based, via
213 the bucket policy; the bucket's identity is published as SSM
214 parameters under ``cost_report_ssm_parameter_prefix`` for the
215 cost-monitor service, the CLI, and release validation to resolve.
217 Lifecycle policy comes from ``cdk.json`` (``cost_monitoring.reports``):
218 report objects transition to STANDARD_IA after
219 ``transition_to_infrequent_access_days`` and expire after
220 ``retention_days``; Athena results expire after
221 ``athena.query_results_retention_days``.
222 """
223 cost_config = self.config.get_cost_monitoring_config()
224 reports_config = cost_config["reports"]
225 athena_config = cost_config["athena"]
227 # KMS key for the cost report bucket. Annual rotation, 7-day pending
228 # window, destroy-on-teardown — matching the shared-bucket posture.
229 self.cost_report_kms_key = kms.Key(
230 self,
231 "CostReportKmsKey",
232 description=(
233 "Customer-managed KMS key for the GCO cost report bucket in the monitoring stack."
234 ),
235 enable_key_rotation=True,
236 pending_window=Duration.days(7),
237 removal_policy=RemovalPolicy.DESTROY,
238 )
240 kms_actions = [
241 "kms:Encrypt",
242 "kms:Decrypt",
243 "kms:ReEncrypt*",
244 "kms:GenerateDataKey*",
245 "kms:DescribeKey",
246 ]
247 self.cost_report_kms_key.add_to_resource_policy(
248 iam.PolicyStatement(
249 sid="AllowS3ServiceEncryptDecrypt",
250 effect=iam.Effect.ALLOW,
251 principals=[iam.ServicePrincipal("s3.amazonaws.com")],
252 actions=kms_actions,
253 resources=["*"],
254 )
255 )
257 # Every regional cost-monitor role writes KMS-encrypted objects into
258 # the bucket below. The roles live in other regions and their stacks
259 # deploy first, so the grant direction is inverted relative to the
260 # in-region buckets: this stack resolves each role ARN (a cross-region
261 # reference CDK routes through the same export mechanism the SQS and
262 # cluster widgets already use) and admits it here, scoped to use via
263 # S3 in this region. Nothing in the regional stacks needs the key ARN.
264 cost_monitor_principals = [
265 iam.ArnPrincipal(regional_stack.cost_monitor_role.role_arn)
266 for regional_stack in self.regional_stacks
267 ]
268 if cost_monitor_principals:
269 self.cost_report_kms_key.add_to_resource_policy(
270 iam.PolicyStatement(
271 sid="AllowRegionalCostMonitorsViaS3",
272 effect=iam.Effect.ALLOW,
273 principals=cost_monitor_principals,
274 actions=["kms:GenerateDataKey", "kms:Decrypt", "kms:DescribeKey"],
275 resources=["*"],
276 conditions={
277 "StringEquals": {"kms:ViaService": f"s3.{self.region}.{self.url_suffix}"}
278 },
279 )
280 )
282 # Retention for the access-logs bucket honors the same `s3_access_logs`
283 # context field used by the central buckets (default 90 days).
284 s3_access_logs_ctx = self.node.try_get_context("s3_access_logs") or {}
285 access_logs_retention_days = int(s3_access_logs_ctx.get("retention_days", 90))
287 self.cost_report_access_logs_bucket = s3.Bucket(
288 self,
289 "CostReportAccessLogsBucket",
290 encryption=s3.BucketEncryption.KMS,
291 encryption_key=self.cost_report_kms_key,
292 block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
293 enforce_ssl=True,
294 versioned=True,
295 removal_policy=RemovalPolicy.DESTROY,
296 auto_delete_objects=True,
297 lifecycle_rules=[
298 s3.LifecycleRule(
299 id="ExpireAccessLogs",
300 enabled=True,
301 expiration=Duration.days(access_logs_retention_days),
302 )
303 ],
304 )
306 # No ``bucket_name``: see the S3 bucket naming policy in
307 # ``gco.stacks.constants`` — a CloudFormation-generated name can never
308 # collide with a previous deployment's deleted bucket.
309 self.cost_report_bucket = s3.Bucket(
310 self,
311 "CostReportBucket",
312 encryption=s3.BucketEncryption.KMS,
313 encryption_key=self.cost_report_kms_key,
314 bucket_key_enabled=True,
315 block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
316 enforce_ssl=True,
317 versioned=True,
318 removal_policy=RemovalPolicy.DESTROY,
319 auto_delete_objects=True,
320 server_access_logs_bucket=self.cost_report_access_logs_bucket,
321 server_access_logs_prefix="cost-reports/",
322 lifecycle_rules=[
323 # Scheduled + ad-hoc reports share one policy: IA after the
324 # configured transition, expiry after the retention window.
325 s3.LifecycleRule(
326 id="CostReportRetention",
327 enabled=True,
328 prefix=f"{COST_REPORT_SCHEDULED_PREFIX}/",
329 transitions=[
330 s3.Transition(
331 storage_class=s3.StorageClass.INFREQUENT_ACCESS,
332 transition_after=Duration.days(
333 int(reports_config["transition_to_infrequent_access_days"])
334 ),
335 )
336 ],
337 expiration=Duration.days(int(reports_config["retention_days"])),
338 ),
339 s3.LifecycleRule(
340 id="AdhocReportRetention",
341 enabled=True,
342 prefix="adhoc/",
343 transitions=[
344 s3.Transition(
345 storage_class=s3.StorageClass.INFREQUENT_ACCESS,
346 transition_after=Duration.days(
347 int(reports_config["transition_to_infrequent_access_days"])
348 ),
349 )
350 ],
351 expiration=Duration.days(int(reports_config["retention_days"])),
352 ),
353 s3.LifecycleRule(
354 id="ExpireAthenaResults",
355 enabled=True,
356 prefix=f"{COST_ATHENA_RESULTS_PREFIX}/",
357 expiration=Duration.days(int(athena_config["query_results_retention_days"])),
358 ),
359 ],
360 )
362 # Explicit Deny for insecure transport with a verifiable SID,
363 # duplicating enforce_ssl=True per the central-bucket pattern.
364 self.cost_report_bucket.add_to_resource_policy(
365 iam.PolicyStatement(
366 sid="DenyInsecureTransport",
367 effect=iam.Effect.DENY,
368 principals=[iam.AnyPrincipal()],
369 actions=["s3:*"],
370 resources=[
371 self.cost_report_bucket.bucket_arn,
372 f"{self.cost_report_bucket.bucket_arn}/*",
373 ],
374 conditions={"Bool": {"aws:SecureTransport": "false"}},
375 )
376 )
378 # Principal-based write grant for the regional cost-monitor roles.
379 # Same-account bucket-policy allows are sufficient on their own, so
380 # the regional stacks carry no S3 statement for this bucket at all —
381 # the service writes scheduled/ad-hoc Parquet reports, lists recent
382 # report objects for the API surface, and needs GetBucketLocation for
383 # the cross-region client redirect. Object actions are scoped to this
384 # bucket's key space; nothing here is ``Principal: "*"``.
385 if cost_monitor_principals:
386 self.cost_report_bucket.add_to_resource_policy(
387 iam.PolicyStatement(
388 sid="AllowRegionalCostMonitorReports",
389 effect=iam.Effect.ALLOW,
390 principals=cost_monitor_principals,
391 actions=[
392 "s3:PutObject",
393 "s3:GetObject",
394 "s3:ListBucket",
395 "s3:GetBucketLocation",
396 ],
397 resources=[
398 self.cost_report_bucket.bucket_arn,
399 f"{self.cost_report_bucket.bucket_arn}/*",
400 ],
401 )
402 )
404 # Publish the bucket's identity in this region's parameter store,
405 # mirroring the model, cluster-shared and regional-shared buckets. The
406 # regional cost-monitor services resolve ``<prefix>/name`` at runtime
407 # (their stacks deploy before this one, so the parameter is the
408 # rendezvous point, not a synth-time value); the CLI and release
409 # validation read ``/name`` and ``/arn``.
410 cost_report_prefix = cost_report_ssm_parameter_prefix(self.project_name)
411 ssm.StringParameter(
412 self,
413 "CostReportBucketNameParam",
414 parameter_name=f"{cost_report_prefix}/name",
415 string_value=self.cost_report_bucket.bucket_name,
416 description="Name of the central cost report bucket written by every region.",
417 )
418 ssm.StringParameter(
419 self,
420 "CostReportBucketArnParam",
421 parameter_name=f"{cost_report_prefix}/arn",
422 string_value=self.cost_report_bucket.bucket_arn,
423 description="ARN of the central cost report bucket written by every region.",
424 )
425 ssm.StringParameter(
426 self,
427 "CostReportBucketRegionParam",
428 parameter_name=f"{cost_report_prefix}/region",
429 string_value=self.region,
430 description="Home region of the central cost report bucket.",
431 )
433 from gco.stacks.nag_suppressions import acknowledge_nag_findings
435 cost_replication_reason = (
436 "Cost reports are derived analytics data regenerated continuously "
437 "by the per-region cost-monitor services; there is no durability "
438 "requirement that warrants cross-region replication. Access logs "
439 "do not require replication for the same reason."
440 )
441 acknowledge_nag_findings(
442 self.cost_report_bucket,
443 [
444 {
445 "id": "HIPAA.Security-S3BucketReplicationEnabled",
446 "reason": cost_replication_reason,
447 },
448 {
449 "id": "NIST.800.53.R5-S3BucketReplicationEnabled",
450 "reason": cost_replication_reason,
451 },
452 {
453 "id": "PCI.DSS.321-S3BucketReplicationEnabled",
454 "reason": cost_replication_reason,
455 },
456 ],
457 )
459 access_logs_is_self_target_reason = (
460 "This is the server access logs destination bucket for the cost report bucket."
461 )
462 acknowledge_nag_findings(
463 self.cost_report_access_logs_bucket,
464 [
465 {
466 "id": "AwsSolutions-S1",
467 "reason": access_logs_is_self_target_reason,
468 },
469 {
470 "id": "HIPAA.Security-S3BucketLoggingEnabled",
471 "reason": access_logs_is_self_target_reason,
472 },
473 {
474 "id": "NIST.800.53.R5-S3BucketLoggingEnabled",
475 "reason": access_logs_is_self_target_reason,
476 },
477 {
478 "id": "PCI.DSS.321-S3BucketLoggingEnabled",
479 "reason": access_logs_is_self_target_reason,
480 },
481 {
482 "id": "HIPAA.Security-S3BucketReplicationEnabled",
483 "reason": cost_replication_reason,
484 },
485 {
486 "id": "NIST.800.53.R5-S3BucketReplicationEnabled",
487 "reason": cost_replication_reason,
488 },
489 {
490 "id": "PCI.DSS.321-S3BucketReplicationEnabled",
491 "reason": cost_replication_reason,
492 },
493 ],
494 )
496 CfnOutput(
497 self,
498 "CostReportBucketName",
499 value=self.cost_report_bucket.bucket_name,
500 description="Central S3 bucket receiving per-region Parquet cost reports",
501 )
503 def _create_cost_analytics(self) -> None:
504 """Create the Glue database/table and Athena workgroup for cost queries.
506 The Glue table reads the scheduled report layout
507 (``reports/region=<region>/date=<YYYY-MM-DD>/*.parquet``) using
508 **partition projection**, so there is no crawler, no scheduled
509 ``MSCK REPAIR``, and no partition-management Lambda — new partitions
510 are queryable the moment an object lands. The ``region`` partition is
511 projected from the configured deployment regions; ``date`` is a native
512 date projection from 2026-01-01 to NOW.
514 The Athena workgroup pins query results to ``athena-results/`` in the
515 cost report bucket (KMS-encrypted, lifecycle-expired) and enforces its
516 configuration so callers cannot redirect results elsewhere.
518 The table schema is the write-side contract of
519 ``gco.services.cost_monitor`` — the columns below must stay in
520 lockstep with the Parquet fields the service emits.
521 """
522 database_name = cost_glue_database_name(self.project_name)
524 self.cost_glue_database = glue.CfnDatabase(
525 self,
526 "CostGlueDatabase",
527 catalog_id=self.account,
528 database_input=glue.CfnDatabase.DatabaseInputProperty(
529 name=database_name,
530 description=(
531 "GCO cost analytics: per-region OpenCost allocation "
532 "reports written by the cost-monitor services."
533 ),
534 ),
535 )
537 # Columns mirror gco/services/cost_monitor.py::ALLOCATION_REPORT_FIELDS.
538 columns = [
539 glue.CfnTable.ColumnProperty(name="window_start", type="timestamp"),
540 glue.CfnTable.ColumnProperty(name="window_end", type="timestamp"),
541 glue.CfnTable.ColumnProperty(name="cluster", type="string"),
542 glue.CfnTable.ColumnProperty(name="namespace", type="string"),
543 glue.CfnTable.ColumnProperty(name="cpu_core_hours", type="double"),
544 glue.CfnTable.ColumnProperty(name="cpu_cost", type="double"),
545 glue.CfnTable.ColumnProperty(name="ram_gib_hours", type="double"),
546 glue.CfnTable.ColumnProperty(name="ram_cost", type="double"),
547 glue.CfnTable.ColumnProperty(name="gpu_hours", type="double"),
548 glue.CfnTable.ColumnProperty(name="gpu_cost", type="double"),
549 glue.CfnTable.ColumnProperty(name="pv_cost", type="double"),
550 glue.CfnTable.ColumnProperty(name="network_cost", type="double"),
551 glue.CfnTable.ColumnProperty(name="load_balancer_cost", type="double"),
552 glue.CfnTable.ColumnProperty(name="shared_cost", type="double"),
553 glue.CfnTable.ColumnProperty(name="external_cost", type="double"),
554 glue.CfnTable.ColumnProperty(name="total_cost", type="double"),
555 glue.CfnTable.ColumnProperty(name="total_efficiency", type="double"),
556 ]
558 scheduled_location = (
559 f"s3://{self.cost_report_bucket.bucket_name}/{COST_REPORT_SCHEDULED_PREFIX}/"
560 )
561 self.cost_glue_table = glue.CfnTable(
562 self,
563 "CostAllocationTable",
564 catalog_id=self.account,
565 database_name=database_name,
566 table_input=glue.CfnTable.TableInputProperty(
567 name=COST_GLUE_ALLOCATION_TABLE,
568 description="Scheduled OpenCost allocation reports (Parquet)",
569 table_type="EXTERNAL_TABLE",
570 parameters={
571 "classification": "parquet",
572 "EXTERNAL": "TRUE",
573 # Partition projection: no crawler, no MSCK. The region
574 # projection is pinned to the deployment's configured
575 # regions; extend deployment_regions.regional and redeploy
576 # to pick up new regions.
577 "projection.enabled": "true",
578 "projection.region.type": "enum",
579 "projection.region.values": ",".join(self.regions),
580 "projection.date.type": "date",
581 "projection.date.range": "2026-01-01,NOW",
582 "projection.date.format": "yyyy-MM-dd",
583 "projection.date.interval": "1",
584 "projection.date.interval.unit": "DAYS",
585 "storage.location.template": (
586 f"{scheduled_location}region=${{region}}/date=${{date}}"
587 ),
588 },
589 partition_keys=[
590 glue.CfnTable.ColumnProperty(name="region", type="string"),
591 glue.CfnTable.ColumnProperty(name="date", type="string"),
592 ],
593 storage_descriptor=glue.CfnTable.StorageDescriptorProperty(
594 location=scheduled_location,
595 input_format=("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"),
596 output_format=(
597 "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"
598 ),
599 serde_info=glue.CfnTable.SerdeInfoProperty(
600 serialization_library=(
601 "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"
602 ),
603 ),
604 columns=columns,
605 ),
606 ),
607 )
608 self.cost_glue_table.add_resource_dependency(self.cost_glue_database)
610 self.cost_athena_workgroup = athena.CfnWorkGroup(
611 self,
612 "CostAthenaWorkGroup",
613 name=cost_athena_workgroup_name(self.project_name),
614 description="GCO cost analytics queries over the cost report bucket",
615 recursive_delete_option=True,
616 work_group_configuration=athena.CfnWorkGroup.WorkGroupConfigurationProperty(
617 enforce_work_group_configuration=True,
618 publish_cloud_watch_metrics_enabled=True,
619 result_configuration=athena.CfnWorkGroup.ResultConfigurationProperty(
620 output_location=(
621 f"s3://{self.cost_report_bucket.bucket_name}/{COST_ATHENA_RESULTS_PREFIX}/"
622 ),
623 encryption_configuration=(
624 athena.CfnWorkGroup.EncryptionConfigurationProperty(
625 encryption_option="SSE_KMS",
626 kms_key=self.cost_report_kms_key.key_arn,
627 )
628 ),
629 ),
630 ),
631 )
633 CfnOutput(
634 self,
635 "CostAthenaWorkGroupName",
636 value=cost_athena_workgroup_name(self.project_name),
637 description="Athena workgroup for GCO cost analytics queries",
638 )
639 CfnOutput(
640 self,
641 "CostGlueDatabaseName",
642 value=cost_glue_database_name(self.project_name),
643 description="Glue database containing the cost allocation table",
644 )
646 def _create_dashboard(self) -> cloudwatch.Dashboard:
647 """Create comprehensive CloudWatch dashboard for monitoring"""
648 dashboard = cloudwatch.Dashboard(
649 self,
650 "GCODashboard",
651 period_override=cloudwatch.PeriodOverride.AUTO,
652 )
654 # Add widgets in logical order
655 dashboard.add_widgets(*self._create_global_accelerator_widgets())
656 dashboard.add_widgets(*self._create_api_gateway_widgets())
657 dashboard.add_widgets(*self._create_lambda_widgets())
658 dashboard.add_widgets(*self._create_sqs_widgets())
659 dashboard.add_widgets(*self._create_dynamodb_widgets())
660 dashboard.add_widgets(*self._create_eks_widgets())
661 dashboard.add_widgets(*self._create_gpu_widgets())
662 dashboard.add_widgets(*self._create_fsx_widgets())
663 dashboard.add_widgets(*self._create_valkey_widgets())
664 dashboard.add_widgets(*self._create_aurora_pgvector_widgets())
665 dashboard.add_widgets(*self._create_alb_widgets())
666 dashboard.add_widgets(*self._create_application_widgets())
668 return dashboard
670 def _create_global_accelerator_widgets(self) -> list[cloudwatch.IWidget]:
671 """Create Global Accelerator monitoring widgets.
673 Note: Global Accelerator metrics are only available in us-west-2,
674 regardless of where the accelerator endpoints are located.
675 CloudWatch uses the Accelerator ID (UUID), not the name.
676 """
677 widgets: list[cloudwatch.IWidget] = []
678 if self.global_stack.accelerator_id is None:
679 return widgets
681 # Get the accelerator ID from the global stack (CloudWatch uses ID, not name)
682 accelerator_id = self.global_stack.accelerator_id
684 # Global Accelerator metrics are always in us-west-2
685 ga_metrics_region = "us-west-2"
687 # Section header
688 widgets.append(
689 cloudwatch.TextWidget(
690 markdown="# Global Accelerator\nTraffic distribution and connectivity metrics",
691 width=24,
692 height=1,
693 )
694 )
696 # Flow count with anomaly detection
697 flow_count_widget = cloudwatch.GraphWidget(
698 title="Global Accelerator - New Flows",
699 left=[
700 cloudwatch.Metric(
701 namespace="AWS/GlobalAccelerator",
702 metric_name="NewFlowCount",
703 dimensions_map={"Accelerator": accelerator_id},
704 statistic="Sum",
705 period=Duration.minutes(5),
706 region=ga_metrics_region,
707 )
708 ],
709 width=12,
710 height=6,
711 region=ga_metrics_region,
712 )
713 widgets.append(flow_count_widget)
715 # Processed bytes
716 bytes_widget = cloudwatch.GraphWidget(
717 title="Global Accelerator - Processed Bytes",
718 left=[
719 cloudwatch.Metric(
720 namespace="AWS/GlobalAccelerator",
721 metric_name="ProcessedBytesIn",
722 dimensions_map={"Accelerator": accelerator_id},
723 statistic="Sum",
724 period=Duration.minutes(5),
725 region=ga_metrics_region,
726 ),
727 cloudwatch.Metric(
728 namespace="AWS/GlobalAccelerator",
729 metric_name="ProcessedBytesOut",
730 dimensions_map={"Accelerator": accelerator_id},
731 statistic="Sum",
732 period=Duration.minutes(5),
733 region=ga_metrics_region,
734 ),
735 ],
736 width=12,
737 height=6,
738 region=ga_metrics_region,
739 )
740 widgets.append(bytes_widget)
742 return widgets
744 def _create_api_gateway_widgets(self) -> list[cloudwatch.IWidget]:
745 """Create API Gateway monitoring widgets"""
746 widgets: list[cloudwatch.IWidget] = []
748 # Get the actual API name from the api_gateway_stack
749 api_name = (
750 self.api_gateway_stack.api.rest_api_name
751 if self.api_gateway_stack
752 else f"{self.project_name}-global-api"
753 )
755 # API Gateway metrics are in the region where the API is deployed
756 api_gw_region = self.config.get_api_gateway_region()
758 # Section header
759 widgets.append(
760 cloudwatch.TextWidget(
761 markdown="# API Gateway\nRequest metrics, latency, and error rates",
762 width=24,
763 height=1,
764 )
765 )
767 # Request count and latency
768 request_widget = cloudwatch.GraphWidget(
769 title="API Gateway - Requests & Latency",
770 left=[
771 cloudwatch.Metric(
772 namespace="AWS/ApiGateway",
773 metric_name="Count",
774 dimensions_map={"ApiName": api_name},
775 statistic="Sum",
776 period=Duration.minutes(5),
777 region=api_gw_region,
778 )
779 ],
780 right=[
781 cloudwatch.Metric(
782 namespace="AWS/ApiGateway",
783 metric_name="Latency",
784 dimensions_map={"ApiName": api_name},
785 statistic="Average",
786 period=Duration.minutes(5),
787 region=api_gw_region,
788 ),
789 cloudwatch.Metric(
790 namespace="AWS/ApiGateway",
791 metric_name="Latency",
792 dimensions_map={"ApiName": api_name},
793 statistic="p99",
794 period=Duration.minutes(5),
795 region=api_gw_region,
796 ),
797 ],
798 width=12,
799 height=6,
800 region=api_gw_region,
801 )
802 widgets.append(request_widget)
804 # Error rates (4XX and 5XX)
805 error_widget = cloudwatch.GraphWidget(
806 title="API Gateway - Error Rates",
807 left=[
808 cloudwatch.Metric(
809 namespace="AWS/ApiGateway",
810 metric_name="4XXError",
811 dimensions_map={"ApiName": api_name},
812 statistic="Sum",
813 period=Duration.minutes(5),
814 color="#ff7f0e",
815 region=api_gw_region,
816 ),
817 cloudwatch.Metric(
818 namespace="AWS/ApiGateway",
819 metric_name="5XXError",
820 dimensions_map={"ApiName": api_name},
821 statistic="Sum",
822 period=Duration.minutes(5),
823 color="#d62728",
824 region=api_gw_region,
825 ),
826 ],
827 width=12,
828 height=6,
829 region=api_gw_region,
830 )
831 widgets.append(error_widget)
833 return widgets
835 def _create_lambda_widgets(self) -> list[cloudwatch.IWidget]:
836 """Create Lambda function monitoring widgets"""
837 widgets: list[cloudwatch.IWidget] = []
839 # Section header
840 widgets.append(
841 cloudwatch.TextWidget(
842 markdown="# Lambda Functions\nProxy, rotation, and regional Lambda metrics",
843 width=24,
844 height=1,
845 )
846 )
848 # Get API Gateway region for global Lambda functions
849 api_gw_region = self.config.get_api_gateway_region()
851 # Build Lambda function list: (function_name, label, region)
852 lambda_functions: list[tuple[str, str, str]] = []
854 # Add API Gateway Lambda functions if available
855 if self.api_gateway_stack:
856 if self.api_gateway_stack.proxy_lambda is not None:
857 lambda_functions.append(
858 (
859 self.api_gateway_stack.proxy_lambda.function_name,
860 "API Gateway Proxy",
861 api_gw_region,
862 )
863 )
864 lambda_functions.append(
865 (
866 self.api_gateway_stack.rotation_lambda.function_name,
867 "Secret Rotation",
868 api_gw_region,
869 )
870 )
872 # Add regional Lambda functions from each regional stack
873 for regional_stack in self.regional_stacks:
874 region = regional_stack.deployment_region
875 lambda_functions.extend(
876 [
877 (
878 regional_stack.kubectl_lambda_function_name,
879 f"Kubectl Applier ({region})",
880 region,
881 ),
882 (
883 regional_stack.helm_installer_lambda_function_name,
884 f"Helm Installer ({region})",
885 region,
886 ),
887 ]
888 )
890 # Invocations widget
891 invocations_widget = cloudwatch.GraphWidget(
892 title="Lambda - Invocations",
893 left=[
894 cloudwatch.Metric(
895 namespace="AWS/Lambda",
896 metric_name="Invocations",
897 dimensions_map={"FunctionName": func_name},
898 statistic="Sum",
899 period=Duration.minutes(5),
900 label=label,
901 region=region,
902 )
903 for func_name, label, region in lambda_functions[:5]
904 ],
905 width=12,
906 height=6,
907 )
908 widgets.append(invocations_widget)
910 errors_widget = cloudwatch.GraphWidget(
911 title="Lambda - Errors",
912 left=[
913 cloudwatch.Metric(
914 namespace="AWS/Lambda",
915 metric_name="Errors",
916 dimensions_map={"FunctionName": func_name},
917 statistic="Sum",
918 period=Duration.minutes(5),
919 label=label,
920 color="#d62728",
921 region=region,
922 )
923 for func_name, label, region in lambda_functions[:5]
924 ],
925 width=12,
926 height=6,
927 )
928 widgets.append(errors_widget)
930 # Duration widget
931 duration_widget = cloudwatch.GraphWidget(
932 title="Lambda - Duration (ms)",
933 left=[
934 cloudwatch.Metric(
935 namespace="AWS/Lambda",
936 metric_name="Duration",
937 dimensions_map={"FunctionName": func_name},
938 statistic="Average",
939 period=Duration.minutes(5),
940 label=label,
941 region=region,
942 )
943 for func_name, label, region in lambda_functions[:5]
944 ],
945 width=12,
946 height=6,
947 )
948 widgets.append(duration_widget)
950 # Throttles widget
951 throttles_widget = cloudwatch.GraphWidget(
952 title="Lambda - Throttles & Concurrent Executions",
953 left=[
954 cloudwatch.Metric(
955 namespace="AWS/Lambda",
956 metric_name="Throttles",
957 dimensions_map={"FunctionName": func_name},
958 statistic="Sum",
959 period=Duration.minutes(5),
960 label=f"{label} Throttles",
961 region=region,
962 )
963 for func_name, label, region in lambda_functions[:3]
964 ],
965 right=[
966 cloudwatch.Metric(
967 namespace="AWS/Lambda",
968 metric_name="ConcurrentExecutions",
969 dimensions_map={"FunctionName": func_name},
970 statistic="Maximum",
971 period=Duration.minutes(5),
972 label=f"{label} Concurrent",
973 region=region,
974 )
975 for func_name, label, region in lambda_functions[:3]
976 ],
977 width=12,
978 height=6,
979 )
980 widgets.append(throttles_widget)
982 return widgets
984 def _create_sqs_widgets(self) -> list[cloudwatch.IWidget]:
985 """Create SQS queue monitoring widgets"""
986 widgets: list[cloudwatch.IWidget] = []
988 # Section header
989 widgets.append(
990 cloudwatch.TextWidget(
991 markdown="# SQS Queues\nJob submission queue metrics and dead letter queue",
992 width=24,
993 height=1,
994 )
995 )
997 # Build queue info from regional stacks: (queue_name, dlq_name, region)
998 queue_info = [
999 (
1000 regional_stack.job_queue.queue_name,
1001 regional_stack.job_dlq.queue_name,
1002 regional_stack.deployment_region,
1003 )
1004 for regional_stack in self.regional_stacks
1005 ]
1007 # Messages visible and in-flight per region
1008 messages_widget = cloudwatch.GraphWidget(
1009 title="SQS - Messages (Visible & In-Flight)",
1010 left=[
1011 cloudwatch.Metric(
1012 namespace="AWS/SQS",
1013 metric_name="ApproximateNumberOfMessagesVisible",
1014 dimensions_map={"QueueName": queue_name},
1015 statistic="Average",
1016 period=Duration.minutes(1),
1017 label=f"{region} Visible",
1018 region=region,
1019 )
1020 for queue_name, _, region in queue_info
1021 ],
1022 right=[
1023 cloudwatch.Metric(
1024 namespace="AWS/SQS",
1025 metric_name="ApproximateNumberOfMessagesNotVisible",
1026 dimensions_map={"QueueName": queue_name},
1027 statistic="Average",
1028 period=Duration.minutes(1),
1029 label=f"{region} In-Flight",
1030 region=region,
1031 )
1032 for queue_name, _, region in queue_info
1033 ],
1034 width=12,
1035 height=6,
1036 )
1037 widgets.append(messages_widget)
1039 # Age of oldest message (critical for detecting stuck jobs)
1040 age_widget = cloudwatch.GraphWidget(
1041 title="SQS - Age of Oldest Message (seconds)",
1042 left=[
1043 cloudwatch.Metric(
1044 namespace="AWS/SQS",
1045 metric_name="ApproximateAgeOfOldestMessage",
1046 dimensions_map={"QueueName": queue_name},
1047 statistic="Maximum",
1048 period=Duration.minutes(1),
1049 label=region,
1050 region=region,
1051 )
1052 for queue_name, _, region in queue_info
1053 ],
1054 width=12,
1055 height=6,
1056 )
1057 widgets.append(age_widget)
1059 # Dead letter queue depth
1060 dlq_widget = cloudwatch.GraphWidget(
1061 title="SQS - Dead Letter Queue Depth",
1062 left=[
1063 cloudwatch.Metric(
1064 namespace="AWS/SQS",
1065 metric_name="ApproximateNumberOfMessagesVisible",
1066 dimensions_map={"QueueName": dlq_name},
1067 statistic="Average",
1068 period=Duration.minutes(1),
1069 label=f"{region} DLQ",
1070 color="#d62728",
1071 region=region,
1072 )
1073 for _, dlq_name, region in queue_info
1074 ],
1075 width=12,
1076 height=6,
1077 )
1078 widgets.append(dlq_widget)
1080 # Messages sent/received/deleted
1081 throughput_widget = cloudwatch.GraphWidget(
1082 title="SQS - Throughput",
1083 left=[
1084 cloudwatch.Metric(
1085 namespace="AWS/SQS",
1086 metric_name="NumberOfMessagesSent",
1087 dimensions_map={"QueueName": queue_name},
1088 statistic="Sum",
1089 period=Duration.minutes(5),
1090 label=f"{region} Sent",
1091 region=region,
1092 )
1093 for queue_name, _, region in queue_info
1094 ],
1095 right=[
1096 cloudwatch.Metric(
1097 namespace="AWS/SQS",
1098 metric_name="NumberOfMessagesDeleted",
1099 dimensions_map={"QueueName": queue_name},
1100 statistic="Sum",
1101 period=Duration.minutes(5),
1102 label=f"{region} Processed",
1103 region=region,
1104 )
1105 for queue_name, _, region in queue_info
1106 ],
1107 width=12,
1108 height=6,
1109 )
1110 widgets.append(throughput_widget)
1112 return widgets
1114 def _create_dynamodb_widgets(self) -> list[cloudwatch.IWidget]:
1115 """Create DynamoDB monitoring widgets for job queue, templates, and webhooks tables."""
1116 widgets: list[cloudwatch.IWidget] = []
1118 # Get table names from global stack
1119 templates_table = self.global_stack.templates_table.table_name
1120 webhooks_table = self.global_stack.webhooks_table.table_name
1121 jobs_table = self.global_stack.jobs_table.table_name
1123 # DynamoDB tables are in the global region
1124 global_region = self.config.get_global_region()
1126 # Section header
1127 widgets.append(
1128 cloudwatch.TextWidget(
1129 markdown="# DynamoDB Tables\nJob queue, templates, and webhooks storage metrics",
1130 width=24,
1131 height=1,
1132 )
1133 )
1135 # Read/Write capacity consumed
1136 capacity_widget = cloudwatch.GraphWidget(
1137 title="DynamoDB - Consumed Capacity",
1138 left=[
1139 cloudwatch.Metric(
1140 namespace="AWS/DynamoDB",
1141 metric_name="ConsumedReadCapacityUnits",
1142 dimensions_map={"TableName": jobs_table},
1143 statistic="Sum",
1144 period=Duration.minutes(5),
1145 label="Jobs Read",
1146 region=global_region,
1147 ),
1148 cloudwatch.Metric(
1149 namespace="AWS/DynamoDB",
1150 metric_name="ConsumedReadCapacityUnits",
1151 dimensions_map={"TableName": templates_table},
1152 statistic="Sum",
1153 period=Duration.minutes(5),
1154 label="Templates Read",
1155 region=global_region,
1156 ),
1157 cloudwatch.Metric(
1158 namespace="AWS/DynamoDB",
1159 metric_name="ConsumedReadCapacityUnits",
1160 dimensions_map={"TableName": webhooks_table},
1161 statistic="Sum",
1162 period=Duration.minutes(5),
1163 label="Webhooks Read",
1164 region=global_region,
1165 ),
1166 ],
1167 right=[
1168 cloudwatch.Metric(
1169 namespace="AWS/DynamoDB",
1170 metric_name="ConsumedWriteCapacityUnits",
1171 dimensions_map={"TableName": jobs_table},
1172 statistic="Sum",
1173 period=Duration.minutes(5),
1174 label="Jobs Write",
1175 region=global_region,
1176 ),
1177 cloudwatch.Metric(
1178 namespace="AWS/DynamoDB",
1179 metric_name="ConsumedWriteCapacityUnits",
1180 dimensions_map={"TableName": templates_table},
1181 statistic="Sum",
1182 period=Duration.minutes(5),
1183 label="Templates Write",
1184 region=global_region,
1185 ),
1186 ],
1187 width=12,
1188 height=6,
1189 region=global_region,
1190 )
1191 widgets.append(capacity_widget)
1193 # Latency metrics
1194 latency_widget = cloudwatch.GraphWidget(
1195 title="DynamoDB - Latency (ms)",
1196 left=[
1197 cloudwatch.Metric(
1198 namespace="AWS/DynamoDB",
1199 metric_name="SuccessfulRequestLatency",
1200 dimensions_map={"TableName": jobs_table, "Operation": "GetItem"},
1201 statistic="Average",
1202 period=Duration.minutes(5),
1203 label="Jobs GetItem",
1204 region=global_region,
1205 ),
1206 cloudwatch.Metric(
1207 namespace="AWS/DynamoDB",
1208 metric_name="SuccessfulRequestLatency",
1209 dimensions_map={"TableName": jobs_table, "Operation": "PutItem"},
1210 statistic="Average",
1211 period=Duration.minutes(5),
1212 label="Jobs PutItem",
1213 region=global_region,
1214 ),
1215 cloudwatch.Metric(
1216 namespace="AWS/DynamoDB",
1217 metric_name="SuccessfulRequestLatency",
1218 dimensions_map={"TableName": jobs_table, "Operation": "Query"},
1219 statistic="Average",
1220 period=Duration.minutes(5),
1221 label="Jobs Query",
1222 region=global_region,
1223 ),
1224 ],
1225 width=12,
1226 height=6,
1227 region=global_region,
1228 )
1229 widgets.append(latency_widget)
1231 # Throttled requests
1232 throttle_widget = cloudwatch.GraphWidget(
1233 title="DynamoDB - Throttled Requests",
1234 left=[
1235 cloudwatch.Metric(
1236 namespace="AWS/DynamoDB",
1237 metric_name="ThrottledRequests",
1238 dimensions_map={"TableName": jobs_table},
1239 statistic="Sum",
1240 period=Duration.minutes(5),
1241 label="Jobs",
1242 color="#d62728",
1243 region=global_region,
1244 ),
1245 cloudwatch.Metric(
1246 namespace="AWS/DynamoDB",
1247 metric_name="ThrottledRequests",
1248 dimensions_map={"TableName": templates_table},
1249 statistic="Sum",
1250 period=Duration.minutes(5),
1251 label="Templates",
1252 color="#ff7f0e",
1253 region=global_region,
1254 ),
1255 cloudwatch.Metric(
1256 namespace="AWS/DynamoDB",
1257 metric_name="ThrottledRequests",
1258 dimensions_map={"TableName": webhooks_table},
1259 statistic="Sum",
1260 period=Duration.minutes(5),
1261 label="Webhooks",
1262 color="#9467bd",
1263 region=global_region,
1264 ),
1265 ],
1266 width=12,
1267 height=6,
1268 region=global_region,
1269 )
1270 widgets.append(throttle_widget)
1272 # System errors
1273 errors_widget = cloudwatch.GraphWidget(
1274 title="DynamoDB - System Errors",
1275 left=[
1276 cloudwatch.Metric(
1277 namespace="AWS/DynamoDB",
1278 metric_name="SystemErrors",
1279 dimensions_map={"TableName": jobs_table},
1280 statistic="Sum",
1281 period=Duration.minutes(5),
1282 label="Jobs",
1283 color="#d62728",
1284 region=global_region,
1285 ),
1286 cloudwatch.Metric(
1287 namespace="AWS/DynamoDB",
1288 metric_name="SystemErrors",
1289 dimensions_map={"TableName": templates_table},
1290 statistic="Sum",
1291 period=Duration.minutes(5),
1292 label="Templates",
1293 color="#ff7f0e",
1294 region=global_region,
1295 ),
1296 ],
1297 width=12,
1298 height=6,
1299 region=global_region,
1300 )
1301 widgets.append(errors_widget)
1303 return widgets
1305 def _create_eks_widgets(self) -> list[cloudwatch.IWidget]:
1306 """Create EKS cluster monitoring widgets"""
1307 widgets: list[cloudwatch.IWidget] = []
1309 # Section header
1310 widgets.append(
1311 cloudwatch.TextWidget(
1312 markdown="# EKS Clusters\nCluster resource utilization and node metrics",
1313 width=24,
1314 height=1,
1315 )
1316 )
1318 # Build cluster info from regional stacks: (cluster_name, region)
1319 cluster_info = [
1320 (regional_stack.cluster.cluster_name, regional_stack.deployment_region)
1321 for regional_stack in self.regional_stacks
1322 ]
1324 # EKS cluster status
1325 cluster_status_widget = cloudwatch.SingleValueWidget(
1326 title="EKS Clusters - Failed Requests",
1327 metrics=[
1328 cloudwatch.Metric(
1329 namespace="AWS/EKS",
1330 metric_name="cluster_failed_request_count",
1331 dimensions_map={"cluster_name": cluster_name},
1332 statistic="Sum",
1333 period=Duration.minutes(5),
1334 region=region,
1335 )
1336 for cluster_name, region in cluster_info
1337 ],
1338 width=12,
1339 height=6,
1340 )
1341 widgets.append(cluster_status_widget)
1343 # Container Insights - Node CPU utilization (aggregated across all nodes)
1344 # Note: region parameter enables cross-region metrics in dashboard
1345 cpu_widget = cloudwatch.GraphWidget(
1346 title="EKS Clusters - Node CPU Utilization (%)",
1347 left=[
1348 cloudwatch.Metric(
1349 namespace="ContainerInsights",
1350 metric_name="node_cpu_utilization",
1351 dimensions_map={"ClusterName": cluster_name},
1352 statistic="Average",
1353 period=Duration.minutes(5),
1354 label=region,
1355 region=region,
1356 )
1357 for cluster_name, region in cluster_info
1358 ],
1359 width=12,
1360 height=6,
1361 )
1362 widgets.append(cpu_widget)
1364 # Container Insights - Node Memory utilization (aggregated across all nodes)
1365 memory_widget = cloudwatch.GraphWidget(
1366 title="EKS Clusters - Node Memory Utilization (%)",
1367 left=[
1368 cloudwatch.Metric(
1369 namespace="ContainerInsights",
1370 metric_name="node_memory_utilization",
1371 dimensions_map={"ClusterName": cluster_name},
1372 statistic="Average",
1373 period=Duration.minutes(5),
1374 label=region,
1375 region=region,
1376 )
1377 for cluster_name, region in cluster_info
1378 ],
1379 width=12,
1380 height=6,
1381 )
1382 widgets.append(memory_widget)
1384 # Node status - running pods capacity
1385 node_widget = cloudwatch.GraphWidget(
1386 title="EKS Clusters - Node Pod Capacity",
1387 left=[
1388 cloudwatch.Metric(
1389 namespace="ContainerInsights",
1390 metric_name="node_status_capacity_pods",
1391 dimensions_map={"ClusterName": cluster_name},
1392 statistic="Sum",
1393 period=Duration.minutes(5),
1394 label=f"{region} Capacity",
1395 region=region,
1396 )
1397 for cluster_name, region in cluster_info
1398 ],
1399 right=[
1400 cloudwatch.Metric(
1401 namespace="ContainerInsights",
1402 metric_name="node_number_of_running_pods",
1403 dimensions_map={"ClusterName": cluster_name},
1404 statistic="Sum",
1405 period=Duration.minutes(5),
1406 label=f"{region} Running",
1407 region=region,
1408 )
1409 for cluster_name, region in cluster_info
1410 ],
1411 width=12,
1412 height=6,
1413 )
1414 widgets.append(node_widget)
1416 return widgets
1418 def _create_gpu_widgets(self) -> list[cloudwatch.IWidget]:
1419 """Create GPU monitoring widgets using DCGM Exporter metrics via ContainerInsights."""
1420 widgets: list[cloudwatch.IWidget] = []
1422 widgets.append(
1423 cloudwatch.TextWidget(
1424 markdown="# GPU Metrics\nGPU utilization, memory, and temperature from DCGM Exporter",
1425 width=24,
1426 height=1,
1427 )
1428 )
1430 cluster_info = [
1431 (regional_stack.cluster.cluster_name, regional_stack.deployment_region)
1432 for regional_stack in self.regional_stacks
1433 ]
1435 # GPU utilization percentage
1436 gpu_util_widget = cloudwatch.GraphWidget(
1437 title="GPU Utilization (%)",
1438 left=[
1439 cloudwatch.Metric(
1440 namespace="ContainerInsights",
1441 metric_name="node_gpu_utilization",
1442 dimensions_map={"ClusterName": cluster_name},
1443 statistic="Average",
1444 period=Duration.minutes(5),
1445 label=region,
1446 region=region,
1447 )
1448 for cluster_name, region in cluster_info
1449 ],
1450 width=12,
1451 height=6,
1452 )
1453 widgets.append(gpu_util_widget)
1455 # GPU memory utilization
1456 gpu_mem_widget = cloudwatch.GraphWidget(
1457 title="GPU Memory Utilization (%)",
1458 left=[
1459 cloudwatch.Metric(
1460 namespace="ContainerInsights",
1461 metric_name="node_gpu_memory_utilization",
1462 dimensions_map={"ClusterName": cluster_name},
1463 statistic="Average",
1464 period=Duration.minutes(5),
1465 label=region,
1466 region=region,
1467 )
1468 for cluster_name, region in cluster_info
1469 ],
1470 width=12,
1471 height=6,
1472 )
1473 widgets.append(gpu_mem_widget)
1475 # GPU temperature
1476 gpu_temp_widget = cloudwatch.GraphWidget(
1477 title="GPU Temperature (°C)",
1478 left=[
1479 cloudwatch.Metric(
1480 namespace="ContainerInsights",
1481 metric_name="node_gpu_temperature",
1482 dimensions_map={"ClusterName": cluster_name},
1483 statistic="Maximum",
1484 period=Duration.minutes(5),
1485 label=region,
1486 region=region,
1487 )
1488 for cluster_name, region in cluster_info
1489 ],
1490 width=12,
1491 height=6,
1492 )
1493 widgets.append(gpu_temp_widget)
1495 # GPU count (active GPUs)
1496 gpu_count_widget = cloudwatch.GraphWidget(
1497 title="Active GPU Count",
1498 left=[
1499 cloudwatch.Metric(
1500 namespace="ContainerInsights",
1501 metric_name="node_gpu_limit",
1502 dimensions_map={"ClusterName": cluster_name},
1503 statistic="Sum",
1504 period=Duration.minutes(5),
1505 label=region,
1506 region=region,
1507 )
1508 for cluster_name, region in cluster_info
1509 ],
1510 width=12,
1511 height=6,
1512 )
1513 widgets.append(gpu_count_widget)
1515 return widgets
1517 def _create_fsx_widgets(self) -> list[cloudwatch.IWidget]:
1518 """Create FSx for Lustre monitoring widgets.
1520 Only emits widgets for regions where the FSx file system is actually
1521 provisioned (``regional_stack.fsx_file_system`` is non-None). The
1522 dimension ``FileSystemId`` is the CDK-generated CloudFormation ref
1523 from each regional stack — CDK's ``cross_region_references=True``
1524 (enabled on this stack's constructor) plumbs the value across
1525 regions via SSM + custom resources.
1527 Returns an empty list if no region has FSx enabled — the dashboard
1528 skips the section entirely.
1529 """
1530 # Collect (file_system_id, region) tuples for regions that have FSx on.
1531 # fsx_file_system is either a CfnFileSystem or None; the local
1532 # assignment + is-not-None check lets mypy narrow the type so
1533 # ``.ref`` access typechecks cleanly (a list comprehension with
1534 # the guard in the ``if`` clause does not narrow the value clause).
1535 fsx_info: list[tuple[str, str]] = []
1536 for regional_stack in self.regional_stacks:
1537 fsx = getattr(regional_stack, "fsx_file_system", None)
1538 if fsx is None:
1539 continue
1540 fsx_info.append((fsx.ref, regional_stack.deployment_region))
1541 if not fsx_info:
1542 return []
1544 widgets: list[cloudwatch.IWidget] = []
1546 # Section header
1547 widgets.append(
1548 cloudwatch.TextWidget(
1549 markdown=(
1550 "# FSx for Lustre\n"
1551 "Parallel file system throughput, IOPS, and free storage "
1552 "capacity. Each line below is scoped to the exact GCO "
1553 "file system in its region — so unrelated FSx file "
1554 "systems in the same account do not appear on the "
1555 "dashboard."
1556 ),
1557 width=24,
1558 height=1,
1559 )
1560 )
1562 # Throughput: bytes read vs written
1563 throughput_widget = cloudwatch.GraphWidget(
1564 title="FSx - Throughput (Bytes/sec)",
1565 left=[
1566 cloudwatch.Metric(
1567 namespace="AWS/FSx",
1568 metric_name="DataReadBytes",
1569 dimensions_map={"FileSystemId": fs_id},
1570 statistic="Sum",
1571 period=Duration.minutes(1),
1572 label=f"{region} Read",
1573 region=region,
1574 )
1575 for fs_id, region in fsx_info
1576 ],
1577 right=[
1578 cloudwatch.Metric(
1579 namespace="AWS/FSx",
1580 metric_name="DataWriteBytes",
1581 dimensions_map={"FileSystemId": fs_id},
1582 statistic="Sum",
1583 period=Duration.minutes(1),
1584 label=f"{region} Write",
1585 region=region,
1586 )
1587 for fs_id, region in fsx_info
1588 ],
1589 width=12,
1590 height=6,
1591 )
1592 widgets.append(throughput_widget)
1594 # IOPS: read vs write operations
1595 iops_widget = cloudwatch.GraphWidget(
1596 title="FSx - IOPS",
1597 left=[
1598 cloudwatch.Metric(
1599 namespace="AWS/FSx",
1600 metric_name="DataReadOperations",
1601 dimensions_map={"FileSystemId": fs_id},
1602 statistic="Sum",
1603 period=Duration.minutes(1),
1604 label=f"{region} Read",
1605 region=region,
1606 )
1607 for fs_id, region in fsx_info
1608 ],
1609 right=[
1610 cloudwatch.Metric(
1611 namespace="AWS/FSx",
1612 metric_name="DataWriteOperations",
1613 dimensions_map={"FileSystemId": fs_id},
1614 statistic="Sum",
1615 period=Duration.minutes(1),
1616 label=f"{region} Write",
1617 region=region,
1618 )
1619 for fs_id, region in fsx_info
1620 ],
1621 width=12,
1622 height=6,
1623 )
1624 widgets.append(iops_widget)
1626 # Free storage capacity — the classic "running out of space" signal.
1627 # FreeDataStorageCapacity is emitted in bytes.
1628 free_storage_widget = cloudwatch.GraphWidget(
1629 title="FSx - Free Storage Capacity (Bytes)",
1630 left=[
1631 cloudwatch.Metric(
1632 namespace="AWS/FSx",
1633 metric_name="FreeDataStorageCapacity",
1634 dimensions_map={"FileSystemId": fs_id},
1635 statistic="Minimum",
1636 period=Duration.minutes(5),
1637 label=region,
1638 region=region,
1639 )
1640 for fs_id, region in fsx_info
1641 ],
1642 width=24,
1643 height=6,
1644 )
1645 widgets.append(free_storage_widget)
1647 return widgets
1649 def _create_valkey_widgets(self) -> list[cloudwatch.IWidget]:
1650 """Create Valkey (ElastiCache Serverless) monitoring widgets.
1652 Uses explicit ``clusterId`` dimension values (camelCase — the
1653 ElastiCache Serverless variant; distinct from the node-based
1654 ``CacheClusterId``). The regional stack names its cache
1655 deterministically as ``gco-{deployment_region}``, so we reproduce
1656 that name here and pin each widget to the exact cache in its
1657 region. No SEARCH expression, so the dashboard ignores every
1658 unrelated ElastiCache cluster in the account.
1659 """
1660 valkey_enabled = self.config.get_valkey_config().get("enabled", False)
1661 if not valkey_enabled or not self.regions:
1662 return []
1664 widgets: list[cloudwatch.IWidget] = []
1666 widgets.append(
1667 cloudwatch.TextWidget(
1668 markdown=(
1669 "# Valkey Serverless Cache\n"
1670 "ECPU consumption, storage, hit rate, and request "
1671 "latency — scoped to each region's ``gco-{region}`` "
1672 "cache exactly (no SEARCH)."
1673 ),
1674 width=24,
1675 height=1,
1676 )
1677 )
1679 # Build (cache_name, region) pairs. cache_name is the literal
1680 # ``serverless_cache_name`` the regional stack passes to the
1681 # CfnServerlessCache.
1682 cache_info = [(f"{self.project_name}-{region}", region) for region in self.regions]
1684 # ECPU consumption and cache size per region
1685 for cache_name, region in cache_info:
1686 widgets.append(
1687 cloudwatch.GraphWidget(
1688 title=f"Valkey - ECPU & Cache Size ({region})",
1689 left=[
1690 cloudwatch.Metric(
1691 namespace="AWS/ElastiCache",
1692 metric_name="ElastiCacheProcessingUnits",
1693 dimensions_map={"clusterId": cache_name},
1694 statistic="Sum",
1695 period=Duration.minutes(1),
1696 label="ECPUs",
1697 region=region,
1698 ),
1699 ],
1700 right=[
1701 cloudwatch.Metric(
1702 namespace="AWS/ElastiCache",
1703 metric_name="BytesUsedForCache",
1704 dimensions_map={"clusterId": cache_name},
1705 statistic="Average",
1706 period=Duration.minutes(5),
1707 label="Bytes",
1708 region=region,
1709 ),
1710 ],
1711 width=12,
1712 height=6,
1713 region=region,
1714 )
1715 )
1717 # Hit rate and p99 read/write latency per region
1718 for cache_name, region in cache_info:
1719 widgets.append(
1720 cloudwatch.GraphWidget(
1721 title=f"Valkey - Hit Rate & Latency ({region})",
1722 left=[
1723 cloudwatch.Metric(
1724 namespace="AWS/ElastiCache",
1725 metric_name="CacheHitRate",
1726 dimensions_map={"clusterId": cache_name},
1727 statistic="Average",
1728 period=Duration.minutes(5),
1729 label="Hit Rate %",
1730 region=region,
1731 ),
1732 ],
1733 right=[
1734 cloudwatch.Metric(
1735 namespace="AWS/ElastiCache",
1736 metric_name="SuccessfulReadRequestLatency",
1737 dimensions_map={"clusterId": cache_name},
1738 statistic="p99",
1739 period=Duration.minutes(1),
1740 label="Read p99 µs",
1741 region=region,
1742 ),
1743 cloudwatch.Metric(
1744 namespace="AWS/ElastiCache",
1745 metric_name="SuccessfulWriteRequestLatency",
1746 dimensions_map={"clusterId": cache_name},
1747 statistic="p99",
1748 period=Duration.minutes(1),
1749 label="Write p99 µs",
1750 region=region,
1751 ),
1752 ],
1753 width=12,
1754 height=6,
1755 region=region,
1756 )
1757 )
1759 return widgets
1761 def _create_aurora_pgvector_widgets(self) -> list[cloudwatch.IWidget]:
1762 """Create Aurora Serverless v2 (pgvector) monitoring widgets.
1764 Pins each widget to the exact Aurora cluster provisioned by the
1765 regional stack via ``regional_stack.aurora_cluster.cluster_identifier``.
1766 CDK-generated cluster IDs are CloudFormation tokens; the
1767 ``cross_region_references=True`` flag on this stack handles
1768 plumbing them from each regional stack into the monitoring stack
1769 (us-east-2 by default) through SSM + custom resources.
1771 Returns an empty list when every region has Aurora pgvector
1772 disabled so the dashboard skips the section entirely.
1773 """
1774 # (cluster_identifier, region) pairs for regions with Aurora on.
1775 # Use a guarded loop (not a comprehension) so mypy can narrow the
1776 # Optional[DatabaseCluster] to a real cluster before dereferencing.
1777 aurora_info: list[tuple[str, str]] = []
1778 for regional_stack in self.regional_stacks:
1779 aurora = getattr(regional_stack, "aurora_cluster", None)
1780 if aurora is None:
1781 continue
1782 aurora_info.append((aurora.cluster_identifier, regional_stack.deployment_region))
1783 if not aurora_info:
1784 return []
1786 widgets: list[cloudwatch.IWidget] = []
1788 widgets.append(
1789 cloudwatch.TextWidget(
1790 markdown=(
1791 "# Aurora pgvector (Serverless v2)\n"
1792 "ACU utilization, database connections, query latency, "
1793 "and CPU utilization — pinned to each regional GCO "
1794 "Aurora cluster by ID. ACU utilization is the primary "
1795 "scale/cost signal for Serverless v2."
1796 ),
1797 width=24,
1798 height=1,
1799 )
1800 )
1802 # ACU utilization and capacity
1803 for cluster_id, region in aurora_info:
1804 widgets.append(
1805 cloudwatch.GraphWidget(
1806 title=f"Aurora - ACU Utilization & Capacity ({region})",
1807 left=[
1808 cloudwatch.Metric(
1809 namespace="AWS/RDS",
1810 metric_name="ACUUtilization",
1811 dimensions_map={"DBClusterIdentifier": cluster_id},
1812 statistic="Average",
1813 period=Duration.minutes(1),
1814 label="ACU %",
1815 region=region,
1816 ),
1817 ],
1818 right=[
1819 cloudwatch.Metric(
1820 namespace="AWS/RDS",
1821 metric_name="ServerlessDatabaseCapacity",
1822 dimensions_map={"DBClusterIdentifier": cluster_id},
1823 statistic="Average",
1824 period=Duration.minutes(1),
1825 label="ACUs",
1826 region=region,
1827 ),
1828 ],
1829 width=12,
1830 height=6,
1831 region=region,
1832 )
1833 )
1835 # Database connections and CPU utilization
1836 for cluster_id, region in aurora_info:
1837 widgets.append(
1838 cloudwatch.GraphWidget(
1839 title=f"Aurora - Connections & CPU ({region})",
1840 left=[
1841 cloudwatch.Metric(
1842 namespace="AWS/RDS",
1843 metric_name="DatabaseConnections",
1844 dimensions_map={"DBClusterIdentifier": cluster_id},
1845 statistic="Average",
1846 period=Duration.minutes(1),
1847 label="Connections",
1848 region=region,
1849 ),
1850 ],
1851 right=[
1852 cloudwatch.Metric(
1853 namespace="AWS/RDS",
1854 metric_name="CPUUtilization",
1855 dimensions_map={"DBClusterIdentifier": cluster_id},
1856 statistic="Average",
1857 period=Duration.minutes(1),
1858 label="CPU %",
1859 region=region,
1860 ),
1861 ],
1862 width=12,
1863 height=6,
1864 region=region,
1865 )
1866 )
1868 # Read and write latency p99
1869 for cluster_id, region in aurora_info:
1870 widgets.append(
1871 cloudwatch.GraphWidget(
1872 title=f"Aurora - Query Latency p99 ({region})",
1873 left=[
1874 cloudwatch.Metric(
1875 namespace="AWS/RDS",
1876 metric_name="ReadLatency",
1877 dimensions_map={"DBClusterIdentifier": cluster_id},
1878 statistic="p99",
1879 period=Duration.minutes(1),
1880 label="Read p99",
1881 region=region,
1882 ),
1883 ],
1884 right=[
1885 cloudwatch.Metric(
1886 namespace="AWS/RDS",
1887 metric_name="WriteLatency",
1888 dimensions_map={"DBClusterIdentifier": cluster_id},
1889 statistic="p99",
1890 period=Duration.minutes(1),
1891 label="Write p99",
1892 region=region,
1893 ),
1894 ],
1895 width=24,
1896 height=6,
1897 region=region,
1898 )
1899 )
1901 return widgets
1903 def _create_alb_widgets(self) -> list[cloudwatch.IWidget]:
1904 """Create ALB monitoring widgets scoped to the GCO platform ALB.
1906 ALBs are created by the AWS Load Balancer Controller at runtime
1907 from an Ingress resource (not by CDK), so the exact ALB name
1908 isn't known at synth time. We originally tried reading the ARN
1909 off the regional stack's ``GaRegistration`` custom resource via
1910 ``cross_region_references=True``, but that path races the
1911 custom-resource response pipeline: CDK's cross-region
1912 ``ExportsWriter`` executes ``Fn::GetAtt: [GaRegistration, AlbArn]``
1913 before CloudFormation has the updated response data stored, and
1914 errors with "Vendor response doesn't contain AlbArn attribute".
1916 Instead we use a SEARCH expression with a composite-token
1917 filter. The ALB Controller names the platform ALB
1918 ``k8s-gco-<hash>`` (the namespace is shortened because the
1919 controller enforces a 32-char total name limit); CloudWatch's
1920 ``LoadBalancer`` dimension is the ARN suffix ``app/<name>/<hash>``,
1921 so an unquoted filter ``LoadBalancer=app/k8s-gco-`` performs a
1922 composite-token match (the sequence ``app``, ``k``, ``8``, ``s``,
1923 ``gco`` must appear consecutively in the dimension value).
1924 Double-quoted filters would be exact matches and return nothing
1925 because no ALB's dimension value is literally ``app/k8s-gco-``.
1926 """
1927 widgets: list[cloudwatch.IWidget] = []
1929 # Section header
1930 widgets.append(
1931 cloudwatch.TextWidget(
1932 markdown=(
1933 "# Application Load Balancers\n"
1934 "Request metrics, response time, HTTP errors, and "
1935 "connection counts — scoped via SEARCH composite-token "
1936 "match to ALBs named ``app/k8s-gco-*`` so only the GCO "
1937 "platform ALB in each region appears. Inference ALBs "
1938 "(named per endpoint) and unrelated ALBs in the "
1939 "account are excluded."
1940 ),
1941 width=24,
1942 height=1,
1943 )
1944 )
1946 # Per-region request count
1947 for region in self.regions:
1948 widgets.append(
1949 cloudwatch.GraphWidget(
1950 title=f"ALB - Request Count ({region})",
1951 left=[
1952 cloudwatch.MathExpression(
1953 expression=(
1954 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "
1955 'MetricName="RequestCount" '
1956 'LoadBalancer=app/k8s-gco-\', "Sum", 300)'
1957 ),
1958 label="Request Count",
1959 period=Duration.minutes(5),
1960 ),
1961 ],
1962 width=12,
1963 height=6,
1964 region=region,
1965 )
1966 )
1968 # Per-region response time (average and p99)
1969 for region in self.regions:
1970 widgets.append(
1971 cloudwatch.GraphWidget(
1972 title=f"ALB - Response Time ({region})",
1973 left=[
1974 cloudwatch.MathExpression(
1975 expression=(
1976 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "
1977 'MetricName="TargetResponseTime" '
1978 'LoadBalancer=app/k8s-gco-\', "Average", 300)'
1979 ),
1980 label="Avg Response Time",
1981 period=Duration.minutes(5),
1982 ),
1983 cloudwatch.MathExpression(
1984 expression=(
1985 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "
1986 'MetricName="TargetResponseTime" '
1987 'LoadBalancer=app/k8s-gco-\', "p99", 300)'
1988 ),
1989 label="p99 Response Time",
1990 period=Duration.minutes(5),
1991 ),
1992 ],
1993 width=12,
1994 height=6,
1995 region=region,
1996 )
1997 )
1999 # Per-region HTTP errors (4XX + 5XX from targets)
2000 for region in self.regions:
2001 widgets.append(
2002 cloudwatch.GraphWidget(
2003 title=f"ALB - HTTP Errors ({region})",
2004 left=[
2005 cloudwatch.MathExpression(
2006 expression=(
2007 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "
2008 'MetricName="HTTPCode_Target_4XX_Count" '
2009 'LoadBalancer=app/k8s-gco-\', "Sum", 300)'
2010 ),
2011 label="4XX Errors",
2012 period=Duration.minutes(5),
2013 ),
2014 ],
2015 right=[
2016 cloudwatch.MathExpression(
2017 expression=(
2018 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "
2019 'MetricName="HTTPCode_Target_5XX_Count" '
2020 'LoadBalancer=app/k8s-gco-\', "Sum", 300)'
2021 ),
2022 label="5XX Errors",
2023 period=Duration.minutes(5),
2024 ),
2025 ],
2026 width=12,
2027 height=6,
2028 region=region,
2029 )
2030 )
2032 # Per-region active connections
2033 for region in self.regions:
2034 widgets.append(
2035 cloudwatch.GraphWidget(
2036 title=f"ALB - Active Connections ({region})",
2037 left=[
2038 cloudwatch.MathExpression(
2039 expression=(
2040 "SEARCH('{AWS/ApplicationELB,LoadBalancer} "
2041 'MetricName="ActiveConnectionCount" '
2042 'LoadBalancer=app/k8s-gco-\', "Sum", 300)'
2043 ),
2044 label="Active Connections",
2045 period=Duration.minutes(5),
2046 ),
2047 ],
2048 width=12,
2049 height=6,
2050 region=region,
2051 )
2052 )
2054 return widgets
2056 def _create_application_widgets(self) -> list[cloudwatch.IWidget]:
2057 """Create custom application monitoring widgets"""
2058 widgets: list[cloudwatch.IWidget] = []
2060 # Section header
2061 widgets.append(
2062 cloudwatch.TextWidget(
2063 markdown="# Application Metrics\n"
2064 "Health monitor and manifest processor metrics. "
2065 "Application logs are available in Container Insights at "
2066 "`/aws/containerinsights/<cluster>/application`.",
2067 width=24,
2068 height=1,
2069 )
2070 )
2072 # Build cluster info from regional stacks: (cluster_name, region)
2073 cluster_info = [
2074 (regional_stack.cluster.cluster_name, regional_stack.deployment_region)
2075 for regional_stack in self.regional_stacks
2076 ]
2078 # Health monitor metrics
2079 health_monitor_widget = cloudwatch.GraphWidget(
2080 title="Health Monitor - Resource Utilization",
2081 left=[
2082 cloudwatch.Metric(
2083 namespace="GCO/HealthMonitor",
2084 metric_name="ClusterCpuUtilization",
2085 dimensions_map={
2086 "ClusterName": cluster_name,
2087 "Region": region,
2088 },
2089 statistic="Average",
2090 period=Duration.minutes(5),
2091 label=f"{region} CPU",
2092 region=region,
2093 )
2094 for cluster_name, region in cluster_info
2095 ],
2096 right=[
2097 cloudwatch.Metric(
2098 namespace="GCO/HealthMonitor",
2099 metric_name="ClusterMemoryUtilization",
2100 dimensions_map={
2101 "ClusterName": cluster_name,
2102 "Region": region,
2103 },
2104 statistic="Average",
2105 period=Duration.minutes(5),
2106 label=f"{region} Memory",
2107 region=region,
2108 )
2109 for cluster_name, region in cluster_info
2110 ],
2111 width=12,
2112 height=6,
2113 )
2114 widgets.append(health_monitor_widget)
2116 # Manifest processor metrics
2117 manifest_processor_widget = cloudwatch.GraphWidget(
2118 title="Manifest Processor - Submissions",
2119 left=[
2120 cloudwatch.Metric(
2121 namespace="GCO/ManifestProcessor",
2122 metric_name="ManifestSubmissions",
2123 dimensions_map={
2124 "ClusterName": cluster_name,
2125 "Region": region,
2126 },
2127 statistic="Sum",
2128 period=Duration.minutes(5),
2129 label=f"{region} Submissions",
2130 region=region,
2131 )
2132 for cluster_name, region in cluster_info
2133 ],
2134 right=[
2135 cloudwatch.Metric(
2136 namespace="GCO/ManifestProcessor",
2137 metric_name="ManifestFailures",
2138 dimensions_map={
2139 "ClusterName": cluster_name,
2140 "Region": region,
2141 },
2142 statistic="Sum",
2143 period=Duration.minutes(5),
2144 label=f"{region} Failures",
2145 color="#d62728",
2146 region=region,
2147 )
2148 for cluster_name, region in cluster_info
2149 ],
2150 width=12,
2151 height=6,
2152 )
2153 widgets.append(manifest_processor_widget)
2155 # Container Insights - Pod restarts (indicates application issues)
2156 pod_restarts_widget = cloudwatch.GraphWidget(
2157 title="Container Insights - Pod Restarts",
2158 left=[
2159 cloudwatch.Metric(
2160 namespace="ContainerInsights",
2161 metric_name="pod_number_of_container_restarts",
2162 dimensions_map={"ClusterName": cluster_name},
2163 statistic="Sum",
2164 period=Duration.minutes(5),
2165 label=f"{region}",
2166 region=region,
2167 )
2168 for cluster_name, region in cluster_info
2169 ],
2170 width=12,
2171 height=6,
2172 )
2173 widgets.append(pod_restarts_widget)
2175 # Secret rotation Lambda metrics (Secrets Manager doesn't publish rotation metrics,
2176 # so we monitor the rotation Lambda function instead)
2177 if self.api_gateway_stack:
2178 rotation_function_name = self.api_gateway_stack.rotation_lambda.function_name
2179 api_gw_region = self.config.get_api_gateway_region()
2181 rotation_widget = cloudwatch.GraphWidget(
2182 title="Secret Rotation Lambda - Invocations & Errors",
2183 left=[
2184 cloudwatch.Metric(
2185 namespace="AWS/Lambda",
2186 metric_name="Invocations",
2187 dimensions_map={"FunctionName": rotation_function_name},
2188 statistic="Sum",
2189 period=Duration.hours(1),
2190 label="Invocations",
2191 color="#2ca02c",
2192 region=api_gw_region,
2193 ),
2194 ],
2195 right=[
2196 cloudwatch.Metric(
2197 namespace="AWS/Lambda",
2198 metric_name="Errors",
2199 dimensions_map={"FunctionName": rotation_function_name},
2200 statistic="Sum",
2201 period=Duration.hours(1),
2202 label="Errors",
2203 color="#d62728",
2204 region=api_gw_region,
2205 ),
2206 ],
2207 width=12,
2208 height=6,
2209 )
2210 widgets.append(rotation_widget)
2211 else:
2212 # Fallback text widget if api_gateway_stack not available
2213 fallback_widget = cloudwatch.TextWidget(
2214 markdown="**Secret Rotation:** API Gateway stack not configured. "
2215 "Rotation Lambda metrics unavailable.",
2216 width=12,
2217 height=6,
2218 )
2219 widgets.append(fallback_widget)
2221 return widgets
2223 def _create_alarms(self) -> None:
2224 """Create CloudWatch alarms"""
2225 self._create_global_accelerator_alarms()
2226 self._create_api_gateway_alarms()
2227 self._create_lambda_alarms()
2228 self._create_sqs_alarms()
2229 self._create_dynamodb_alarms()
2230 self._create_eks_alarms()
2231 self._create_alb_alarms()
2232 self._create_application_alarms()
2234 def _create_global_accelerator_alarms(self) -> None:
2235 """Create Global Accelerator alarms.
2237 Note: Global Accelerator metrics are only available in us-west-2.
2238 CloudWatch Alarms must be in the same region as the metrics they monitor.
2239 Since this monitoring stack may be deployed in a different region,
2240 we skip GA alarms here. To monitor GA, either:
2241 1. Create alarms manually in us-west-2
2242 2. Use CloudWatch cross-region dashboard widgets (which we do)
2243 3. Deploy a separate alarm stack in us-west-2
2244 """
2245 # GA alarms skipped - metrics only available in us-west-2
2246 # Dashboard widgets use region parameter to display GA metrics correctly
2247 pass
2249 def _create_api_gateway_alarms(self) -> None:
2250 """Create API Gateway alarms"""
2251 # Get the actual API name from the api_gateway_stack
2252 api_name = (
2253 self.api_gateway_stack.api.rest_api_name
2254 if self.api_gateway_stack
2255 else f"{self.project_name}-global-api"
2256 )
2258 # High 5XX error rate
2259 api_5xx_alarm = cloudwatch.Alarm(
2260 self,
2261 "ApiGateway5xxAlarm",
2262 alarm_description="API Gateway has high 5XX error rate",
2263 metric=cloudwatch.Metric(
2264 namespace="AWS/ApiGateway",
2265 metric_name="5XXError",
2266 dimensions_map={"ApiName": api_name},
2267 statistic="Sum",
2268 period=Duration.minutes(5),
2269 ),
2270 threshold=10,
2271 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2272 evaluation_periods=2,
2273 datapoints_to_alarm=2,
2274 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2275 )
2276 api_5xx_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2278 # High latency
2279 api_latency_alarm = cloudwatch.Alarm(
2280 self,
2281 "ApiGatewayHighLatencyAlarm",
2282 alarm_description="API Gateway has high latency",
2283 metric=cloudwatch.Metric(
2284 namespace="AWS/ApiGateway",
2285 metric_name="Latency",
2286 dimensions_map={"ApiName": api_name},
2287 statistic="p99",
2288 period=Duration.minutes(5),
2289 ),
2290 threshold=10000, # 10 seconds
2291 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2292 evaluation_periods=3,
2293 datapoints_to_alarm=2,
2294 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2295 )
2296 api_latency_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2298 def _create_lambda_alarms(self) -> None:
2299 """Create Lambda function alarms."""
2300 if self.api_gateway_stack is None:
2301 return
2303 proxy_lambda = self.api_gateway_stack.proxy_lambda
2304 if proxy_lambda is not None:
2305 proxy_function_name = proxy_lambda.function_name
2306 proxy_errors_alarm = cloudwatch.Alarm(
2307 self,
2308 "ProxyLambdaErrorsAlarm",
2309 alarm_description="API Gateway proxy Lambda has errors",
2310 metric=cloudwatch.Metric(
2311 namespace="AWS/Lambda",
2312 metric_name="Errors",
2313 dimensions_map={"FunctionName": proxy_function_name},
2314 statistic="Sum",
2315 period=Duration.minutes(5),
2316 ),
2317 threshold=5,
2318 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2319 evaluation_periods=2,
2320 datapoints_to_alarm=2,
2321 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2322 )
2323 proxy_errors_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2325 proxy_throttles_alarm = cloudwatch.Alarm(
2326 self,
2327 "ProxyLambdaThrottlesAlarm",
2328 alarm_description="API Gateway proxy Lambda is being throttled",
2329 metric=cloudwatch.Metric(
2330 namespace="AWS/Lambda",
2331 metric_name="Throttles",
2332 dimensions_map={"FunctionName": proxy_function_name},
2333 statistic="Sum",
2334 period=Duration.minutes(5),
2335 ),
2336 threshold=1,
2337 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
2338 evaluation_periods=2,
2339 datapoints_to_alarm=2,
2340 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2341 )
2342 proxy_throttles_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2344 rotation_errors_alarm = cloudwatch.Alarm(
2345 self,
2346 "RotationLambdaErrorsAlarm",
2347 alarm_description="Secret rotation Lambda has errors",
2348 metric=cloudwatch.Metric(
2349 namespace="AWS/Lambda",
2350 metric_name="Errors",
2351 dimensions_map={
2352 "FunctionName": self.api_gateway_stack.rotation_lambda.function_name
2353 },
2354 statistic="Sum",
2355 period=Duration.hours(1),
2356 ),
2357 threshold=1,
2358 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
2359 evaluation_periods=1,
2360 datapoints_to_alarm=1,
2361 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2362 )
2363 rotation_errors_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2365 def _create_sqs_alarms(self) -> None:
2366 """Create SQS queue alarms"""
2367 for regional_stack in self.regional_stacks:
2368 region = regional_stack.deployment_region
2369 queue_name = regional_stack.job_queue.queue_name
2370 dlq_name = regional_stack.job_dlq.queue_name
2371 region_id = region.replace("-", "").title()
2373 # Old message alarm (stuck jobs)
2374 old_message_alarm = cloudwatch.Alarm(
2375 self,
2376 f"SqsOldMessageAlarm{region_id}",
2377 alarm_description=f"SQS queue in {region} has old messages (potential stuck jobs)",
2378 metric=cloudwatch.Metric(
2379 namespace="AWS/SQS",
2380 metric_name="ApproximateAgeOfOldestMessage",
2381 dimensions_map={"QueueName": queue_name},
2382 statistic="Maximum",
2383 period=Duration.minutes(5),
2384 ),
2385 threshold=3600, # 1 hour
2386 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2387 evaluation_periods=2,
2388 datapoints_to_alarm=2,
2389 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2390 )
2391 old_message_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2393 # Dead letter queue alarm
2394 dlq_alarm = cloudwatch.Alarm(
2395 self,
2396 f"SqsDlqAlarm{region_id}",
2397 alarm_description=f"SQS dead letter queue in {region} has messages",
2398 metric=cloudwatch.Metric(
2399 namespace="AWS/SQS",
2400 metric_name="ApproximateNumberOfMessagesVisible",
2401 dimensions_map={"QueueName": dlq_name},
2402 statistic="Sum",
2403 period=Duration.minutes(5),
2404 ),
2405 threshold=1,
2406 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
2407 evaluation_periods=1,
2408 datapoints_to_alarm=1,
2409 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2410 )
2411 dlq_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2413 def _create_dynamodb_alarms(self) -> None:
2414 """Create DynamoDB alarms for job queue, templates, and webhooks tables."""
2415 # Get table names from global stack
2416 jobs_table = self.global_stack.jobs_table.table_name
2418 # DynamoDB tables are in the global region
2419 global_region = self.config.get_global_region()
2421 # Jobs table throttling alarm
2422 jobs_throttle_alarm = cloudwatch.Alarm(
2423 self,
2424 "DynamoDBJobsThrottleAlarm",
2425 alarm_description="DynamoDB jobs table is being throttled",
2426 metric=cloudwatch.Metric(
2427 namespace="AWS/DynamoDB",
2428 metric_name="ThrottledRequests",
2429 dimensions_map={"TableName": jobs_table},
2430 statistic="Sum",
2431 period=Duration.minutes(5),
2432 region=global_region,
2433 ),
2434 threshold=1,
2435 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
2436 evaluation_periods=2,
2437 datapoints_to_alarm=2,
2438 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2439 )
2440 jobs_throttle_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2442 # Jobs table system errors alarm
2443 jobs_errors_alarm = cloudwatch.Alarm(
2444 self,
2445 "DynamoDBJobsErrorsAlarm",
2446 alarm_description="DynamoDB jobs table has system errors",
2447 metric=cloudwatch.Metric(
2448 namespace="AWS/DynamoDB",
2449 metric_name="SystemErrors",
2450 dimensions_map={"TableName": jobs_table},
2451 statistic="Sum",
2452 period=Duration.minutes(5),
2453 region=global_region,
2454 ),
2455 threshold=1,
2456 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
2457 evaluation_periods=1,
2458 datapoints_to_alarm=1,
2459 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2460 )
2461 jobs_errors_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2463 def _create_eks_alarms(self) -> None:
2464 """Create EKS cluster alarms"""
2465 for regional_stack in self.regional_stacks:
2466 region = regional_stack.deployment_region
2467 cluster_name = regional_stack.cluster.cluster_name
2468 region_id = region.replace("-", "").title()
2470 # High CPU utilization alarm (node-level metric)
2471 high_cpu_alarm = cloudwatch.Alarm(
2472 self,
2473 f"EksHighCpuAlarm{region_id}",
2474 alarm_description=f"EKS cluster {cluster_name} has high CPU utilization",
2475 metric=cloudwatch.Metric(
2476 namespace="ContainerInsights",
2477 metric_name="node_cpu_utilization",
2478 dimensions_map={"ClusterName": cluster_name},
2479 statistic="Average",
2480 period=Duration.minutes(5),
2481 ),
2482 threshold=80,
2483 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2484 evaluation_periods=3,
2485 datapoints_to_alarm=2,
2486 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2487 )
2488 high_cpu_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2490 # High memory utilization alarm (node-level metric)
2491 high_memory_alarm = cloudwatch.Alarm(
2492 self,
2493 f"EksHighMemoryAlarm{region_id}",
2494 alarm_description=f"EKS cluster {cluster_name} has high memory utilization",
2495 metric=cloudwatch.Metric(
2496 namespace="ContainerInsights",
2497 metric_name="node_memory_utilization",
2498 dimensions_map={"ClusterName": cluster_name},
2499 statistic="Average",
2500 period=Duration.minutes(5),
2501 ),
2502 threshold=85,
2503 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2504 evaluation_periods=3,
2505 datapoints_to_alarm=2,
2506 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2507 )
2508 high_memory_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2510 def _create_alb_alarms(self) -> None:
2511 """Create ALB alarms.
2513 Status: no alarms created yet, even though we now have the ALB
2514 ARN at deploy time via the GA registration custom resource (which
2515 also feeds the dashboard widgets). Adding per-ALB alarms here is
2516 a straightforward enhancement — derive the ``LoadBalancer``
2517 dimension the same way ``_create_alb_widgets`` does
2518 (``Fn.split(":loadbalancer/", alb_arn)[1]``) and wire it into
2519 ``cloudwatch.Alarm`` constructs.
2521 For now we rely on:
2522 1. Dashboard widgets pinned to each platform ALB (see
2523 ``_create_alb_widgets``)
2524 2. EKS Container Insights alarms for pod/node health
2525 3. API Gateway alarms for request-level monitoring
2526 """
2527 # TODO: Add UnHealthyHostCount / 5XXCount alarms using the ARN
2528 # returned by regional_stack.ga_registration.get_att_string("AlbArn").
2529 # The test suite explicitly documents that the ALB alarm count is
2530 # currently zero (test_alb_unhealthy_hosts_alarm_skipped); update
2531 # that test when adding real alarms.
2532 pass
2534 def _create_application_alarms(self) -> None:
2535 """Create application-specific alarms"""
2536 for regional_stack in self.regional_stacks:
2537 region = regional_stack.deployment_region
2538 cluster_name = regional_stack.cluster.cluster_name
2539 region_id = region.replace("-", "").title()
2541 # High manifest failure rate alarm
2542 high_failure_rate_alarm = cloudwatch.Alarm(
2543 self,
2544 f"ManifestHighFailureRateAlarm{region_id}",
2545 alarm_description=f"Manifest processor in {region} has high failure rate",
2546 metric=cloudwatch.Metric(
2547 namespace="GCO/ManifestProcessor",
2548 metric_name="ManifestFailures",
2549 dimensions_map={"ClusterName": cluster_name, "Region": region},
2550 statistic="Sum",
2551 period=Duration.minutes(5),
2552 ),
2553 threshold=10,
2554 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2555 evaluation_periods=2,
2556 datapoints_to_alarm=2,
2557 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2558 )
2559 high_failure_rate_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2561 def _create_composite_alarms(self) -> None:
2562 """Create composite alarms for better signal-to-noise ratio"""
2564 # Store individual alarms for composite alarm references
2565 regional_alarms: dict[str, list[cloudwatch.Alarm]] = {}
2567 for regional_stack in self.regional_stacks:
2568 region = regional_stack.deployment_region
2569 cluster_name = regional_stack.cluster.cluster_name
2570 region_id = region.replace("-", "").title()
2571 regional_alarms[region] = []
2573 # Create regional health composite alarm
2574 # Triggers when multiple issues occur in the same region
2575 eks_cpu_alarm = cloudwatch.Alarm(
2576 self,
2577 f"CompositeEksCpu{region_id}",
2578 metric=cloudwatch.Metric(
2579 namespace="ContainerInsights",
2580 metric_name="node_cpu_utilization",
2581 dimensions_map={"ClusterName": cluster_name},
2582 statistic="Average",
2583 period=Duration.minutes(5),
2584 ),
2585 threshold=90,
2586 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2587 evaluation_periods=2,
2588 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2589 )
2590 regional_alarms[region].append(eks_cpu_alarm)
2592 eks_memory_alarm = cloudwatch.Alarm(
2593 self,
2594 f"CompositeEksMemory{region_id}",
2595 metric=cloudwatch.Metric(
2596 namespace="ContainerInsights",
2597 metric_name="node_memory_utilization",
2598 dimensions_map={"ClusterName": cluster_name},
2599 statistic="Average",
2600 period=Duration.minutes(5),
2601 ),
2602 threshold=90,
2603 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2604 evaluation_periods=2,
2605 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2606 )
2607 regional_alarms[region].append(eks_memory_alarm)
2609 # Create composite alarm for critical regional issues. Every region
2610 # above appends exactly the CPU and memory alarms (two), so this
2611 # always fires; no `len(alarms) >= 2` guard is needed here.
2612 for region, alarms in regional_alarms.items():
2613 region_id = region.replace("-", "").title()
2614 composite_alarm = cloudwatch.CompositeAlarm(
2615 self,
2616 f"RegionalCriticalAlarm{region_id}",
2617 alarm_description=f"Critical: Multiple issues detected in {region}",
2618 alarm_rule=cloudwatch.AlarmRule.all_of(*alarms),
2619 )
2620 composite_alarm.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2622 # API Gateway + Lambda composite alarm (only if api_gateway_stack is available)
2623 if self.api_gateway_stack and self.api_gateway_stack.proxy_lambda is not None:
2624 api_name = self.api_gateway_stack.api.rest_api_name
2625 proxy_function_name = self.api_gateway_stack.proxy_lambda.function_name
2627 api_error_alarm = cloudwatch.Alarm(
2628 self,
2629 "CompositeApiErrors",
2630 metric=cloudwatch.Metric(
2631 namespace="AWS/ApiGateway",
2632 metric_name="5XXError",
2633 dimensions_map={"ApiName": api_name},
2634 statistic="Sum",
2635 period=Duration.minutes(5),
2636 ),
2637 threshold=5,
2638 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2639 evaluation_periods=2,
2640 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2641 )
2643 lambda_error_alarm = cloudwatch.Alarm(
2644 self,
2645 "CompositeLambdaErrors",
2646 metric=cloudwatch.Metric(
2647 namespace="AWS/Lambda",
2648 metric_name="Errors",
2649 dimensions_map={"FunctionName": proxy_function_name},
2650 statistic="Sum",
2651 period=Duration.minutes(5),
2652 ),
2653 threshold=3,
2654 comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
2655 evaluation_periods=2,
2656 treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
2657 )
2659 api_lambda_composite = cloudwatch.CompositeAlarm(
2660 self,
2661 "ApiLambdaCompositeAlarm",
2662 alarm_description="Critical: Both API Gateway and Lambda proxy have errors",
2663 alarm_rule=cloudwatch.AlarmRule.all_of(api_error_alarm, lambda_error_alarm),
2664 )
2665 api_lambda_composite.add_alarm_action(cw_actions.SnsAction(self.alert_topic))
2667 def _create_custom_metrics(self) -> None:
2668 """Create custom metric filters and log groups"""
2669 for regional_stack in self.regional_stacks:
2670 region = regional_stack.deployment_region
2671 region_id = region.replace("-", "").title()
2673 # Health monitor log group
2674 # log_group_name intentionally omitted - let CDK generate unique name
2675 logs.LogGroup(
2676 self,
2677 f"HealthMonitorLogGroup{region_id}",
2678 retention=logs.RetentionDays.ONE_MONTH,
2679 removal_policy=RemovalPolicy.DESTROY,
2680 )
2682 # Manifest processor log group
2683 # log_group_name intentionally omitted - let CDK generate unique name
2684 logs.LogGroup(
2685 self,
2686 f"ManifestProcessorLogGroup{region_id}",
2687 retention=logs.RetentionDays.ONE_MONTH,
2688 removal_policy=RemovalPolicy.DESTROY,
2689 )
2691 def _create_outputs(self) -> None:
2692 """Create CloudFormation outputs"""
2693 CfnOutput(
2694 self,
2695 "DashboardUrl",
2696 value=f"https://console.aws.amazon.com/cloudwatch/home?region={self.region}#dashboards:name={self.dashboard.dashboard_name}",
2697 description="CloudWatch Dashboard URL",
2698 )
2700 CfnOutput(
2701 self,
2702 "AlertTopicArn",
2703 value=self.alert_topic.topic_arn,
2704 description="SNS Topic ARN for monitoring alerts",
2705 )
2707 CfnOutput(
2708 self,
2709 "AlarmCount",
2710 value="See CloudWatch Alarms console for full list",
2711 description="Monitoring alarms created",
2712 )