Coverage for diagrams / infra_diagrams / generate.py: 100.00%
143 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-09-14 22:07 +0000
1#!/usr/bin/env python3
2"""Generate infrastructure diagrams for GCO with cdk-dia.
4This script synthesizes the CDK app the same way ``app.py`` does and renders
5one architecture diagram per stack type (global, api-gateway, regional,
6regional-api, monitoring, analytics) plus a combined full-architecture
7diagram, as PNG, using `cdk-dia <https://github.com/pistazie/cdk-dia>`_.
9Why cdk-dia (and not aws-pdk)?
10-----------------------------
11The previous generator used ``aws-pdk``'s ``cdk-graph`` +
12``cdk-graph-plugin-diagram``. Every ``aws-pdk`` release (through the latest,
130.26.15) hard-pins ``cdk-nag<3.0.0``, which is incompatible with the
14cdk-nag 3.x this project now uses — it blocks the ``[cdk]`` extra and the
15lockfile, and aws-pdk is end-of-life (its successor, the Nx Plugin for AWS,
16is a TypeScript monorepo scaffolder that does not carry the diagram plugin
17forward). ``cdk-dia`` is an actively maintained, purpose-built CDK diagram
18tool that depends only on ``aws-cdk-lib``/``constructs`` (no cdk-nag) and
19renders AWS-icon diagrams via the system ``dot`` binary.
21How it works
22------------
23``cdk-dia`` reads a *synthesized* cloud assembly (``cdk.out/tree.json``) — it
24does not run ``cdk synth`` itself. So this script synthesizes each diagram's
25stack set in-process to a temporary ``cdk.out``. Only the Docker image asset is
26stubbed so no container daemon is required; Helm installer Lambda, Step
27Functions, custom-resource provider, IAM, and network constructs remain real
28in the synthesized topology. The script then invokes the locked ``cdk-dia``
29binary from the root npm graph against ``<cdk.out>/tree.json``.
31Per-stack diagrams synthesize just the target stack (passing placeholder
32strings for cross-stack inputs). ``regional-api`` also instantiates the
33regional stack because its constructor consumes the regional VPC construct;
34``--include`` then scopes the diagram to the regional-api stack. The full view
35always includes each regional aggregation bridge; direct caller access remains
36a separate policy toggle.
38Output is PNG only (cdk-dia does not emit SVG). The full-architecture diagram
39is rendered twice: a collapsed overview and a ``--no-collapse`` detailed view.
41Usage:
42 python diagrams/infra_diagrams/generate.py # all diagrams
43 python diagrams/infra_diagrams/generate.py --stack all # all diagrams
44 python diagrams/infra_diagrams/generate.py --stack global
45 python diagrams/infra_diagrams/generate.py --stack api-gateway
46 python diagrams/infra_diagrams/generate.py --stack regional
47 python diagrams/infra_diagrams/generate.py --stack regional-api
48 python diagrams/infra_diagrams/generate.py --stack monitoring
49 python diagrams/infra_diagrams/generate.py --stack analytics
51Prerequisites:
52 * Graphviz ``dot`` on PATH (``brew install graphviz`` / ``apt-get install
53 graphviz``).
54 * Node.js + npm. Run the root ``npm ci`` command first so ``cdk-dia``
55 executes from the committed lockfile rather than an on-demand graph.
56"""
58from __future__ import annotations
60import argparse
61import contextlib
62import subprocess
63import sys
64import tempfile
65from collections.abc import Callable, Iterator
66from pathlib import Path
67from typing import Any
68from unittest.mock import patch
70# Add project root to path. This script lives at
71# ``diagrams/infra_diagrams/generate.py`` so the project root is two parents
72# up. The ``sys.path.insert`` lets the script be invoked standalone
73# (``python diagrams/infra_diagrams/generate.py``) without a prior
74# ``pip install -e .``.
75_PROJECT_ROOT = Path(__file__).parent.parent.parent.resolve()
76sys.path.insert(0, str(_PROJECT_ROOT))
78import aws_cdk as cdk # noqa: E402
80from cli.stacks import cdk_asset_consumer # noqa: E402
81from diagrams.infra_diagrams._catalog import INFRA_DIAGRAM_NAMES # noqa: E402
82from gco.config.config_loader import ConfigLoader # noqa: E402
83from gco.stacks.analytics_stack import GCOAnalyticsStack # noqa: E402
84from gco.stacks.api_gateway_global_stack import ( # noqa: E402
85 AnalyticsApiConfig,
86 GCOApiGatewayGlobalStack,
87)
88from gco.stacks.global_stack import GCOGlobalStack # noqa: E402
89from gco.stacks.monitoring_stack import GCOMonitoringStack # noqa: E402
90from gco.stacks.regional_api_gateway_stack import GCORegionalApiGatewayStack # noqa: E402
91from gco.stacks.regional_stack import GCORegionalStack # noqa: E402
94def _locked_node_tool(package_name: str) -> Path:
95 """Return a root node_modules binary or fail with install guidance."""
96 binary = _PROJECT_ROOT / "node_modules" / ".bin" / package_name
97 if not binary.is_file():
98 raise RuntimeError(
99 f"{package_name} is not installed from package-lock.json; run "
100 "'npm ci --ignore-scripts --no-audit --no-fund' at the project root"
101 )
102 return binary
105# A builder instantiates the stacks for one diagram and returns the stack ids
106# to pass to ``cdk-dia --include`` (or ``None`` to diagram every stack in the
107# assembly, used for the full-architecture views).
108Builder = Callable[[cdk.App, ConfigLoader], "list[str] | None"]
111@contextlib.contextmanager
112def _mocked_regional_assets() -> Iterator[None]:
113 """Stub only Docker image assets while retaining the CDK topology.
115 Lambda, Step Functions, providers, IAM, and network constructs remain real
116 so regional and aggregate diagrams do not silently omit Helm convergence.
117 Docker image publication is the only boundary the offline renderer does
118 not need to model.
119 """
120 with patch("gco.stacks.regional_stack.ecr_assets.DockerImageAsset") as mock_docker:
121 mock_docker.return_value.image_uri = (
122 "123456789012.dkr.ecr.us-east-1.amazonaws.com/test:latest"
123 )
124 yield
127def _run_cdk_dia(
128 tree_path: Path, target: Path, *, include: list[str] | None, collapse: bool
129) -> None:
130 """Invoke the locked ``cdk-dia`` against a synthesized ``tree.json``."""
131 cmd = [
132 str(_locked_node_tool("cdk-dia")),
133 "--tree",
134 str(tree_path),
135 "--target",
136 str(target),
137 ]
138 if not collapse:
139 cmd.append("--no-collapse")
140 if include:
141 cmd += ["--include", *include]
142 subprocess.run(cmd, check=True) # noqa: S603 — fixed argv, no shell, paths we control
143 # cdk-dia writes a Graphviz ``.dot`` sidecar next to the target. It's a
144 # transient intermediate (and its AWS-icon ``image=`` refs are absolute
145 # local npx-cache paths, so it isn't portable) — drop it. The PNG has the
146 # icons rasterized in and is the committed artifact.
147 target.with_suffix(".dot").unlink(missing_ok=True)
150def _generate(
151 name: str,
152 title: str,
153 build: Builder,
154 *,
155 collapse: bool = True,
156 context: dict[str, Any] | None = None,
157) -> None:
158 """Synthesize the app built by ``build`` and render it to ``<name>.png``."""
159 print(f"\n📊 Generating {title}...")
160 output_dir = Path(__file__).parent
161 with tempfile.TemporaryDirectory() as tmp:
162 assembly_dir = Path(tmp) / "cdk.out"
163 with cdk_asset_consumer(_PROJECT_ROOT):
164 app = cdk.App(outdir=str(assembly_dir), context=context)
165 config = ConfigLoader(app)
166 with _mocked_regional_assets():
167 include = build(app, config)
168 app.synth()
169 _run_cdk_dia(
170 assembly_dir / "tree.json",
171 output_dir / f"{name}.png",
172 include=include,
173 collapse=collapse,
174 )
175 print(f" ✓ Created {name}.png")
178# ---------------------------------------------------------------------------
179# Per-diagram builders (mirror app.py's construction, with placeholder inputs
180# for cross-stack values so each diagram stays scoped to its own stack).
181# ---------------------------------------------------------------------------
184def _build_global(app: cdk.App, config: ConfigLoader) -> list[str]:
185 project = config.get_project_name()
186 region = config.get_deployment_regions()["global"]
187 GCOGlobalStack(
188 app,
189 f"{project}-global",
190 config=config,
191 env=cdk.Environment(region=region),
192 description="Global resources including AWS Global Accelerator",
193 )
194 return [f"{project}-global"]
197def _build_api_gateway(app: cdk.App, config: ConfigLoader) -> list[str]:
198 project = config.get_project_name()
199 region = config.get_deployment_regions()["api_gateway"]
200 GCOApiGatewayGlobalStack(
201 app,
202 f"{project}-api-gateway",
203 global_accelerator_dns="placeholder.awsglobalaccelerator.com",
204 project_name=project,
205 api_gateway_config=config.get_api_gateway_config(),
206 registry_region=config.get_global_region(),
207 certificate_regions=config.get_deployment_regions()["regional"],
208 backend_tls_config=config.get_backend_tls_config(),
209 env=cdk.Environment(region=region),
210 description="Global API Gateway with IAM authentication",
211 )
212 return [f"{project}-api-gateway"]
215def _build_regional(app: cdk.App, config: ConfigLoader) -> list[str]:
216 project = config.get_project_name()
217 region = config.get_deployment_regions()["regional"][0]
218 GCORegionalStack(
219 app,
220 f"{project}-{region}",
221 config=config,
222 region=region,
223 auth_secret_arn=f"arn:aws:secretsmanager:{region}:123456789012:secret:placeholder",
224 env=cdk.Environment(region=region),
225 description=f"Regional resources for {region} - EKS, ALB, Services",
226 )
227 return [f"{project}-{region}"]
230def _build_regional_api(app: cdk.App, config: ConfigLoader) -> list[str]:
231 project = config.get_project_name()
232 region = config.get_deployment_regions()["regional"][0]
233 # The regional API gateway stack needs a real VPC construct, so we
234 # instantiate the regional stack (with placeholder inputs) purely to
235 # supply it; ``--include`` scopes the diagram to the regional-api stack.
236 regional_stack = GCORegionalStack(
237 app,
238 f"{project}-{region}",
239 config=config,
240 region=region,
241 auth_secret_arn=f"arn:aws:secretsmanager:{region}:123456789012:secret:placeholder",
242 env=cdk.Environment(region=region),
243 description=f"Regional resources for {region}",
244 )
245 GCORegionalApiGatewayStack(
246 app,
247 f"{project}-regional-api-{region}",
248 config=config,
249 region=region,
250 vpc=regional_stack.vpc,
251 auth_secret_arn=f"arn:aws:secretsmanager:{region}:123456789012:secret:placeholder",
252 aggregator_role_arn=("arn:aws:iam::123456789012:role/gco-diagram-cross-region-aggregator"),
253 env=cdk.Environment(region=region),
254 description=f"Regional aggregation and workload bridge for {region}",
255 )
256 return [f"{project}-regional-api-{region}"]
259def _build_full(app: cdk.App, config: ConfigLoader) -> list[str] | None:
260 """Build global, API, regional, monitoring, and enabled analytics stacks.
262 Returns ``None`` so callers diagram every stack. ``monitoring`` passes its
263 own ``--include`` to scope the view to the monitoring stack.
264 """
265 project = config.get_project_name()
266 regions = config.get_deployment_regions()
268 global_stack = GCOGlobalStack(
269 app, f"{project}-global", config=config, env=cdk.Environment(region=regions["global"])
270 )
271 api_gateway_stack = GCOApiGatewayGlobalStack(
272 app,
273 f"{project}-api-gateway",
274 global_accelerator_dns=global_stack.get_accelerator_dns_name(),
275 project_name=project,
276 api_gateway_config=config.get_api_gateway_config(),
277 registry_region=config.get_global_region(),
278 certificate_regions=regions["regional"],
279 backend_tls_config=config.get_backend_tls_config(),
280 env=cdk.Environment(region=regions["api_gateway"]),
281 )
282 api_gateway_stack.add_dependency(global_stack)
284 regional_stacks = []
285 for region in regions["regional"]:
286 regional_stack = GCORegionalStack(
287 app,
288 f"{project}-{region}",
289 config=config,
290 region=region,
291 auth_secret_arn=api_gateway_stack.secret.secret_arn,
292 env=cdk.Environment(region=region),
293 )
294 regional_stack.add_dependency(global_stack)
295 regional_stack.add_dependency(api_gateway_stack)
296 regional_stacks.append(regional_stack)
298 regional_api_stack = GCORegionalApiGatewayStack(
299 app,
300 f"{project}-regional-api-{region}",
301 config=config,
302 region=region,
303 vpc=regional_stack.vpc,
304 auth_secret_arn=api_gateway_stack.secret.secret_arn,
305 aggregator_role_arn=api_gateway_stack.aggregator_role.role_arn,
306 env=cdk.Environment(region=region),
307 )
308 regional_api_stack.add_dependency(regional_stack)
310 monitoring_stack = GCOMonitoringStack(
311 app,
312 f"{project}-monitoring",
313 config=config,
314 global_stack=global_stack,
315 regional_stacks=regional_stacks,
316 api_gateway_stack=api_gateway_stack,
317 env=cdk.Environment(region=regions["monitoring"]),
318 )
319 for regional_stack in regional_stacks:
320 monitoring_stack.add_dependency(regional_stack)
322 if config.get_analytics_enabled():
323 analytics_stack = GCOAnalyticsStack(
324 app,
325 f"{project}-analytics",
326 config=config,
327 env=cdk.Environment(region=regions["api_gateway"]),
328 description=(
329 "Optional ML and analytics environment (SageMaker Studio, EMR Serverless, Cognito)"
330 ),
331 )
332 analytics_stack.add_dependency(global_stack)
333 api_gateway_stack.set_analytics_config(
334 AnalyticsApiConfig(
335 user_pool_arn=analytics_stack.cognito_pool.user_pool_arn,
336 user_pool_client_id=analytics_stack.cognito_client.user_pool_client_id,
337 presigned_url_lambda=analytics_stack.presigned_url_lambda,
338 studio_domain_name=analytics_stack.studio_domain.domain_name or "",
339 callback_url=(
340 f"https://{api_gateway_stack.api.rest_api_id}.execute-api."
341 f"{regions['api_gateway']}.amazonaws.com/prod/studio/callback"
342 ),
343 )
344 )
345 api_gateway_stack.add_dependency(analytics_stack)
346 return None
349def _build_monitoring(app: cdk.App, config: ConfigLoader) -> list[str]:
350 _build_full(app, config)
351 return [f"{config.get_project_name()}-monitoring"]
354def _build_analytics(app: cdk.App, config: ConfigLoader) -> list[str]:
355 project = config.get_project_name()
356 region = config.get_deployment_regions()["api_gateway"]
357 GCOAnalyticsStack(
358 app,
359 f"{project}-analytics",
360 config=config,
361 env=cdk.Environment(region=region),
362 description=(
363 "Optional ML and analytics environment (SageMaker Studio, EMR Serverless, Cognito)"
364 ),
365 )
366 return [f"{project}-analytics"]
369# Context overlay that force-enables the analytics environment (mirrors the
370# overlay the property tests use), so ``ConfigLoader.get_analytics_enabled()``
371# returns True during the analytics-diagram synth.
372_ANALYTICS_CONTEXT: dict[str, Any] = {
373 "analytics_environment": {
374 "enabled": True,
375 "hyperpod": {"enabled": False},
376 "canvas": {"enabled": False},
377 "cognito": {"domain_prefix": None, "removal_policy": "destroy"},
378 "efs": {"removal_policy": "destroy"},
379 "studio": {"user_profile_name_prefix": None},
380 },
381}
384def main() -> None:
385 parser = argparse.ArgumentParser(description="Generate GCO infrastructure diagrams")
386 parser.add_argument(
387 "--stack",
388 choices=[
389 "all",
390 "global",
391 "api-gateway",
392 "regional",
393 "regional-api",
394 "monitoring",
395 "analytics",
396 ],
397 default="all",
398 help="Which stack diagram to generate (default: all)",
399 )
400 args = parser.parse_args()
402 output_dir = Path(__file__).parent
403 print("🏗️ GCO Infrastructure Diagram Generator (cdk-dia)")
404 print("=" * 50)
406 if args.stack in ("all", "global"):
407 _generate("global-stack", "GCO Global Stack - AWS Global Accelerator", _build_global)
408 if args.stack in ("all", "api-gateway"):
409 _generate("api-gateway-stack", "GCO API Gateway Stack", _build_api_gateway)
410 if args.stack in ("all", "regional"):
411 _generate("regional-stack", "GCO Regional Stack", _build_regional)
412 if args.stack in ("all", "regional-api"):
413 _generate("regional-api-stack", "GCO Regional API Gateway Stack", _build_regional_api)
414 if args.stack in ("all", "monitoring"):
415 _generate("monitoring-stack", "GCO Monitoring Stack", _build_monitoring)
416 if args.stack in ("all", "analytics"):
417 _generate(
418 "analytics-stack",
419 "GCO Analytics Stack - SageMaker Studio + EMR + Cognito",
420 _build_analytics,
421 context=_ANALYTICS_CONTEXT,
422 )
423 if args.stack == "all":
424 _generate(
425 "full-architecture",
426 "GCO Complete Infrastructure Architecture",
427 _build_full,
428 context=_ANALYTICS_CONTEXT,
429 )
430 _generate(
431 "full-architecture-detailed",
432 "GCO Detailed Architecture",
433 _build_full,
434 collapse=False,
435 context=_ANALYTICS_CONTEXT,
436 )
438 if args.stack == "all":
439 expected = {f"{name}.png" for name in INFRA_DIAGRAM_NAMES}
440 for artifact in output_dir.glob("*.png"):
441 if artifact.name not in expected:
442 artifact.unlink()
443 print(f" 🧹 Removed obsolete {artifact.name}")
444 for sidecar in output_dir.glob("*.dot"):
445 sidecar.unlink()
446 print(f" 🧹 Removed transient {sidecar.name}")
448 print("\n" + "=" * 50)
449 print("✅ Diagram generation complete!")
450 print(f" Output directory: {output_dir.absolute()}")
451 for f in sorted(output_dir.glob("*.png")):
452 print(f" - {f.name} ({f.stat().st_size / 1024 / 1024:.1f} MB)")
455if __name__ == "__main__":
456 main()