Coverage for diagrams / infra_diagrams / generate.py: 100.00%
143 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-10-28 16:19 +0000
1#!/usr/bin/env python3
2"""Generate infrastructure diagrams for GCO with cdk-dia.
4This script synthesizes the CDK app the same way ``app.py`` does and renders
5one architecture diagram per stack type (global, api-gateway, regional,
6regional-api, monitoring, analytics) plus a combined full-architecture
7diagram, as PNG, using `cdk-dia <https://github.com/pistazie/cdk-dia>`_.
9Why cdk-dia (and not aws-pdk)?
10-----------------------------
11The previous generator used ``aws-pdk``'s ``cdk-graph`` +
12``cdk-graph-plugin-diagram``. Every ``aws-pdk`` release (through the latest,
130.26.15) hard-pins ``cdk-nag<3.0.0``, which is incompatible with the
14cdk-nag 3.x this project now uses — it blocks the ``[cdk]`` extra and the
15lockfile, and aws-pdk is end-of-life (its successor, the Nx Plugin for AWS,
16is a TypeScript monorepo scaffolder that does not carry the diagram plugin
17forward). ``cdk-dia`` is an actively maintained, purpose-built CDK diagram
18tool that depends only on ``aws-cdk-lib``/``constructs`` (no cdk-nag) and
19renders AWS-icon diagrams via the system ``dot`` binary.
21How it works
22------------
23``cdk-dia`` reads a *synthesized* cloud assembly (``cdk.out/tree.json``) — it
24does not run ``cdk synth`` itself. So this script synthesizes each diagram's
25stack set in-process to a temporary ``cdk.out``. Only the Docker image asset is
26stubbed so no container daemon is required; Helm installer Lambda, Step
27Functions, custom-resource provider, IAM, and network constructs remain real
28in the synthesized topology. The script then invokes the locked ``cdk-dia``
29binary from the root npm graph against ``<cdk.out>/tree.json``.
31Per-stack diagrams synthesize just the target stack (passing placeholder
32strings for cross-stack inputs). ``regional-api`` also instantiates the
33regional stack because its constructor consumes the regional VPC construct;
34``--include`` then scopes the diagram to the regional-api stack. The full view
35always includes each regional aggregation bridge; direct caller access remains
36a separate policy toggle.
38Output is PNG only (cdk-dia does not emit SVG). The full-architecture diagram
39is rendered twice: a collapsed overview and a ``--no-collapse`` detailed view.
41Usage:
42 python diagrams/infra_diagrams/generate.py # all diagrams
43 python diagrams/infra_diagrams/generate.py --stack all # all diagrams
44 python diagrams/infra_diagrams/generate.py --stack global
45 python diagrams/infra_diagrams/generate.py --stack api-gateway
46 python diagrams/infra_diagrams/generate.py --stack regional
47 python diagrams/infra_diagrams/generate.py --stack regional-api
48 python diagrams/infra_diagrams/generate.py --stack monitoring
49 python diagrams/infra_diagrams/generate.py --stack analytics
51Prerequisites:
52 * Graphviz ``dot`` on PATH (``brew install graphviz`` / ``apt-get install
53 graphviz``).
54 * Node.js + npm. Run the root ``npm ci`` command first so ``cdk-dia``
55 executes from the committed lockfile rather than an on-demand graph.
56"""
58from __future__ import annotations
60import argparse
61import contextlib
62import subprocess
63import sys
64import tempfile
65from collections.abc import Callable, Iterator
66from pathlib import Path
67from typing import Any
68from unittest.mock import patch
70# Add project root to path. This script lives at
71# ``diagrams/infra_diagrams/generate.py`` so the project root is two parents
72# up. The ``sys.path.insert`` lets the script be invoked standalone
73# (``python diagrams/infra_diagrams/generate.py``) without a prior
74# ``pip install -e .``.
75_PROJECT_ROOT = Path(__file__).parent.parent.parent.resolve()
76sys.path.insert(0, str(_PROJECT_ROOT))
78import aws_cdk as cdk # noqa: E402
80from cli.stacks import cdk_asset_consumer # noqa: E402
81from diagrams.infra_diagrams._catalog import INFRA_DIAGRAM_NAMES # noqa: E402
82from gco.config.config_loader import ConfigLoader # noqa: E402
83from gco.stacks.analytics_stack import GCOAnalyticsStack # noqa: E402
84from gco.stacks.api_gateway_global_stack import ( # noqa: E402
85 AnalyticsApiConfig,
86 GCOApiGatewayGlobalStack,
87)
88from gco.stacks.global_stack import GCOGlobalStack # noqa: E402
89from gco.stacks.monitoring_stack import GCOMonitoringStack # noqa: E402
90from gco.stacks.regional_api_gateway_stack import GCORegionalApiGatewayStack # noqa: E402
91from gco.stacks.regional_stack import GCORegionalStack # noqa: E402
94def _locked_node_tool(package_name: str) -> Path:
95 """Return a root node_modules binary or fail with install guidance."""
96 binary = _PROJECT_ROOT / "node_modules" / ".bin" / package_name
97 if not binary.is_file():
98 raise RuntimeError(
99 f"{package_name} is not installed from package-lock.json; run "
100 "'npm ci --ignore-scripts --no-audit --no-fund' at the project root"
101 )
102 return binary
105# A builder instantiates the stacks for one diagram and returns the stack ids
106# to pass to ``cdk-dia --include`` (or ``None`` to diagram every stack in the
107# assembly, used for the full-architecture views).
108Builder = Callable[[cdk.App, ConfigLoader], "list[str] | None"]
111@contextlib.contextmanager
112def _mocked_regional_assets() -> Iterator[None]:
113 """Stub only Docker image assets while retaining the CDK topology.
115 Lambda, Step Functions, providers, IAM, and network constructs remain real
116 so regional and aggregate diagrams do not silently omit Helm convergence.
117 Docker image publication is the only boundary the offline renderer does
118 not need to model.
119 """
120 with patch("gco.stacks.regional_stack.ecr_assets.DockerImageAsset") as mock_docker:
121 mock_docker.return_value.image_uri = (
122 "123456789012.dkr.ecr.us-east-1.amazonaws.com/test:latest"
123 )
124 yield
127def _run_cdk_dia(
128 tree_path: Path, target: Path, *, include: list[str] | None, collapse: bool
129) -> None:
130 """Invoke the locked ``cdk-dia`` against a synthesized ``tree.json``."""
131 cmd = [
132 str(_locked_node_tool("cdk-dia")),
133 "--tree",
134 str(tree_path),
135 "--target",
136 str(target),
137 ]
138 if not collapse:
139 cmd.append("--no-collapse")
140 if include:
141 cmd += ["--include", *include]
142 subprocess.run(cmd, check=True) # fixed argv, no shell, paths we control
143 # cdk-dia writes a Graphviz ``.dot`` sidecar next to the target. It's a
144 # transient intermediate (and its AWS-icon ``image=`` refs are absolute
145 # local npx-cache paths, so it isn't portable) — drop it. The PNG has the
146 # icons rasterized in and is the committed artifact.
147 target.with_suffix(".dot").unlink(missing_ok=True)
150def _generate(
151 name: str,
152 title: str,
153 build: Builder,
154 *,
155 collapse: bool = True,
156 context: dict[str, Any] | None = None,
157) -> None:
158 """Synthesize the app built by ``build`` and render it to ``<name>.png``."""
159 print(f"\n📊 Generating {title}...")
160 output_dir = Path(__file__).parent
161 with tempfile.TemporaryDirectory() as tmp:
162 assembly_dir = Path(tmp) / "cdk.out"
163 with cdk_asset_consumer(_PROJECT_ROOT):
164 app = cdk.App(outdir=str(assembly_dir), context=context)
165 config = ConfigLoader(app)
166 with _mocked_regional_assets():
167 include = build(app, config)
168 app.synth()
169 _run_cdk_dia(
170 assembly_dir / "tree.json",
171 output_dir / f"{name}.png",
172 include=include,
173 collapse=collapse,
174 )
175 print(f" ✓ Created {name}.png")
178# ---------------------------------------------------------------------------
179# Per-diagram builders (mirror app.py's construction, with placeholder inputs
180# for cross-stack values so each diagram stays scoped to its own stack).
181# ---------------------------------------------------------------------------
184def _build_global(app: cdk.App, config: ConfigLoader) -> list[str]:
185 project = config.get_project_name()
186 region = config.get_deployment_regions()["global"]
187 GCOGlobalStack(
188 app,
189 f"{project}-global",
190 config=config,
191 env=cdk.Environment(region=region),
192 description="Global resources including AWS Global Accelerator",
193 )
194 return [f"{project}-global"]
197def _build_api_gateway(app: cdk.App, config: ConfigLoader) -> list[str]:
198 project = config.get_project_name()
199 region = config.get_deployment_regions()["api_gateway"]
200 GCOApiGatewayGlobalStack(
201 app,
202 f"{project}-api-gateway",
203 global_accelerator_dns="placeholder.awsglobalaccelerator.com",
204 project_name=project,
205 api_gateway_config=config.get_api_gateway_config(),
206 registry_region=config.get_global_region(),
207 certificate_regions=config.get_deployment_regions()["regional"],
208 backend_tls_config=config.get_backend_tls_config(),
209 env=cdk.Environment(region=region),
210 description="Global API Gateway with IAM authentication",
211 )
212 return [f"{project}-api-gateway"]
215def _build_regional(app: cdk.App, config: ConfigLoader) -> list[str]:
216 project = config.get_project_name()
217 region = config.get_deployment_regions()["regional"][0]
218 GCORegionalStack(
219 app,
220 f"{project}-{region}",
221 config=config,
222 region=region,
223 auth_secret_arn=f"arn:aws:secretsmanager:{region}:123456789012:secret:placeholder",
224 env=cdk.Environment(region=region),
225 description=f"Regional resources for {region} - EKS, ALB, Services",
226 )
227 return [f"{project}-{region}"]
230def _build_regional_api(app: cdk.App, config: ConfigLoader) -> list[str]:
231 project = config.get_project_name()
232 region = config.get_deployment_regions()["regional"][0]
233 # The regional API gateway stack needs a real VPC construct, so we
234 # instantiate the regional stack (with placeholder inputs) purely to
235 # supply it; ``--include`` scopes the diagram to the regional-api stack.
236 regional_stack = GCORegionalStack(
237 app,
238 f"{project}-{region}",
239 config=config,
240 region=region,
241 auth_secret_arn=f"arn:aws:secretsmanager:{region}:123456789012:secret:placeholder",
242 env=cdk.Environment(region=region),
243 description=f"Regional resources for {region}",
244 )
245 GCORegionalApiGatewayStack(
246 app,
247 f"{project}-regional-api-{region}",
248 config=config,
249 region=region,
250 vpc=regional_stack.vpc,
251 auth_secret_arn=f"arn:aws:secretsmanager:{region}:123456789012:secret:placeholder",
252 aggregator_role_arn=("arn:aws:iam::123456789012:role/gco-diagram-cross-region-aggregator"),
253 env=cdk.Environment(region=region),
254 description=f"Regional aggregation and workload bridge for {region}",
255 )
256 return [f"{project}-regional-api-{region}"]
259def _build_full(app: cdk.App, config: ConfigLoader) -> list[str] | None:
260 """Build global, API, regional, monitoring, and enabled analytics stacks.
262 Mirrors ``app.py::main`` stack for stack and argument for argument
263 (descriptions aside); ``tests/test_cdk_harness_contract.py`` fails when the
264 two drift. Returns ``None`` so callers diagram every stack. ``monitoring``
265 passes its own ``--include`` to scope the view to the monitoring stack.
266 """
267 project = config.get_project_name()
268 regions = config.get_deployment_regions()
270 global_stack = GCOGlobalStack(
271 app, f"{project}-global", config=config, env=cdk.Environment(region=regions["global"])
272 )
273 api_gateway_stack = GCOApiGatewayGlobalStack(
274 app,
275 f"{project}-api-gateway",
276 global_accelerator_dns=global_stack.get_accelerator_dns_name(),
277 project_name=project,
278 api_gateway_config=config.get_api_gateway_config(),
279 registry_region=config.get_global_region(),
280 certificate_regions=regions["regional"],
281 backend_tls_config=config.get_backend_tls_config(),
282 max_request_body_bytes=config.get_manifest_processor_config().get(
283 "max_request_body_bytes", 1_048_576
284 ),
285 env=cdk.Environment(region=regions["api_gateway"]),
286 )
287 api_gateway_stack.add_stack_dependency(global_stack)
289 regional_stacks = []
290 for region in regions["regional"]:
291 regional_stack = GCORegionalStack(
292 app,
293 f"{project}-{region}",
294 config=config,
295 region=region,
296 auth_secret_arn=api_gateway_stack.secret.secret_arn,
297 env=cdk.Environment(region=region),
298 )
299 regional_stack.add_stack_dependency(global_stack)
300 regional_stack.add_stack_dependency(api_gateway_stack)
301 regional_stacks.append(regional_stack)
303 regional_api_stack = GCORegionalApiGatewayStack(
304 app,
305 f"{project}-regional-api-{region}",
306 config=config,
307 region=region,
308 vpc=regional_stack.vpc,
309 auth_secret_arn=api_gateway_stack.secret.secret_arn,
310 aggregator_role_arn=api_gateway_stack.aggregator_role.role_arn,
311 env=cdk.Environment(region=region),
312 )
313 regional_api_stack.add_stack_dependency(regional_stack)
315 monitoring_stack = GCOMonitoringStack(
316 app,
317 f"{project}-monitoring",
318 config=config,
319 global_stack=global_stack,
320 regional_stacks=regional_stacks,
321 api_gateway_stack=api_gateway_stack,
322 env=cdk.Environment(region=regions["monitoring"]),
323 )
324 for regional_stack in regional_stacks:
325 monitoring_stack.add_stack_dependency(regional_stack)
327 if config.get_analytics_enabled():
328 analytics_stack = GCOAnalyticsStack(
329 app,
330 f"{project}-analytics",
331 config=config,
332 env=cdk.Environment(region=regions["api_gateway"]),
333 description=(
334 "Optional ML and analytics environment (SageMaker Studio, EMR Serverless, Cognito)"
335 ),
336 )
337 analytics_stack.add_stack_dependency(global_stack)
338 api_gateway_stack.set_analytics_config(
339 AnalyticsApiConfig(
340 user_pool_arn=analytics_stack.cognito_pool.user_pool_arn,
341 user_pool_client_id=analytics_stack.cognito_client.user_pool_client_id,
342 presigned_url_lambda=analytics_stack.presigned_url_lambda,
343 studio_domain_name=analytics_stack.studio_domain.domain_name or "",
344 callback_url=(
345 f"https://{api_gateway_stack.api.rest_api_id}.execute-api."
346 f"{regions['api_gateway']}.{api_gateway_stack.url_suffix}/prod/studio/callback"
347 ),
348 )
349 )
350 api_gateway_stack.add_stack_dependency(analytics_stack)
351 return None
354def _build_monitoring(app: cdk.App, config: ConfigLoader) -> list[str]:
355 _build_full(app, config)
356 return [f"{config.get_project_name()}-monitoring"]
359def _build_analytics(app: cdk.App, config: ConfigLoader) -> list[str]:
360 project = config.get_project_name()
361 region = config.get_deployment_regions()["api_gateway"]
362 GCOAnalyticsStack(
363 app,
364 f"{project}-analytics",
365 config=config,
366 env=cdk.Environment(region=region),
367 description=(
368 "Optional ML and analytics environment (SageMaker Studio, EMR Serverless, Cognito)"
369 ),
370 )
371 return [f"{project}-analytics"]
374# Context overlay that force-enables the analytics environment (mirrors the
375# overlay the property tests use), so ``ConfigLoader.get_analytics_enabled()``
376# returns True during the analytics-diagram synth.
377_ANALYTICS_CONTEXT: dict[str, Any] = {
378 "analytics_environment": {
379 "enabled": True,
380 "hyperpod": {"enabled": False},
381 "canvas": {"enabled": False},
382 "cognito": {"domain_prefix": None, "removal_policy": "destroy"},
383 "efs": {"removal_policy": "destroy"},
384 "studio": {"user_profile_name_prefix": None},
385 },
386}
389def main() -> None:
390 parser = argparse.ArgumentParser(description="Generate GCO infrastructure diagrams")
391 parser.add_argument(
392 "--stack",
393 choices=[
394 "all",
395 "global",
396 "api-gateway",
397 "regional",
398 "regional-api",
399 "monitoring",
400 "analytics",
401 ],
402 default="all",
403 help="Which stack diagram to generate (default: all)",
404 )
405 args = parser.parse_args()
407 output_dir = Path(__file__).parent
408 print("🏗️ GCO Infrastructure Diagram Generator (cdk-dia)")
409 print("=" * 50)
411 if args.stack in ("all", "global"):
412 _generate("global-stack", "GCO Global Stack - AWS Global Accelerator", _build_global)
413 if args.stack in ("all", "api-gateway"):
414 _generate("api-gateway-stack", "GCO API Gateway Stack", _build_api_gateway)
415 if args.stack in ("all", "regional"):
416 _generate("regional-stack", "GCO Regional Stack", _build_regional)
417 if args.stack in ("all", "regional-api"):
418 _generate("regional-api-stack", "GCO Regional API Gateway Stack", _build_regional_api)
419 if args.stack in ("all", "monitoring"):
420 _generate("monitoring-stack", "GCO Monitoring Stack", _build_monitoring)
421 if args.stack in ("all", "analytics"):
422 _generate(
423 "analytics-stack",
424 "GCO Analytics Stack - SageMaker Studio + EMR + Cognito",
425 _build_analytics,
426 context=_ANALYTICS_CONTEXT,
427 )
428 if args.stack == "all":
429 _generate(
430 "full-architecture",
431 "GCO Complete Infrastructure Architecture",
432 _build_full,
433 context=_ANALYTICS_CONTEXT,
434 )
435 _generate(
436 "full-architecture-detailed",
437 "GCO Detailed Architecture",
438 _build_full,
439 collapse=False,
440 context=_ANALYTICS_CONTEXT,
441 )
443 if args.stack == "all":
444 expected = {f"{name}.png" for name in INFRA_DIAGRAM_NAMES}
445 for artifact in output_dir.glob("*.png"):
446 if artifact.name not in expected:
447 artifact.unlink()
448 print(f" 🧹 Removed obsolete {artifact.name}")
449 for sidecar in output_dir.glob("*.dot"):
450 sidecar.unlink()
451 print(f" 🧹 Removed transient {sidecar.name}")
453 print("\n" + "=" * 50)
454 print("✅ Diagram generation complete!")
455 print(f" Output directory: {output_dir.absolute()}")
456 for f in sorted(output_dir.glob("*.png")):
457 print(f" - {f.name} ({f.stat().st_size / 1024 / 1024:.1f} MB)")
460if __name__ == "__main__":
461 main()