Coverage for scripts / dump_nag_findings.py: 100.00%
35 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-10-28 16:19 +0000
1#!/usr/bin/env python3
2"""Dev-only helper: run the cdk-nag compliance test harness and print
3a human-readable inventory of every finding, grouped by pack + rule +
4resource-path, with the config name(s) each finding appears under.
6When to use this
7----------------
8Reach for this script when ``tests/test_nag_compliance.py`` starts
9failing and you need a compact view of what cdk-nag is actually
10objecting to. pytest's default output buries the per-finding detail
11inside an ``AssertionError`` repr — useful for CI gate failures, but
12hard to read when you're trying to scope a new acknowledgment. This
13script:
151. Iterates ``NAG_CONFIGS``, the list the pytest gate parametrizes over
16 (imported from ``tests._cdk_config_matrix`` so the two can never drift).
172. For each config, builds the full CDK app the way ``app.py`` does
18 (``tests._cdk_app_builder``, the builder the gate uses), registers the
19 five rule packs as policy-validation plugins, synthesizes, and reads the
20 resulting ``validation-report.json``.
213. Groups them by ``(pack, rule, resource_path)`` so a wildcard that
22 appears in N configs shows up once with the list of config names
23 rather than N repeated entries.
244. Exits 0 if zero findings, 1 otherwise — which matches the pytest
25 gate and means you can pipe this script's output to
26 ``pre-commit`` or use it as a quick smoke before pushing.
28When NOT to use this
29--------------------
30CI should use the pytest gate, not this script. This prints to stdout
31and doesn't produce a junit.xml. For development use only.
33Relationship to other tooling
34-----------------------------
35* ``tests/test_nag_compliance.py`` — the PR gate. Parameterizes over
36 the same ``NAG_CONFIGS`` and fails if any unacknowledged finding exists.
37 cdk-nag v3 writes findings to ``validation-report.json`` in the cloud
38 assembly; both this script and the gate read that file.
39* ``tests/test_cdk_synthesis_matrix.py`` — runs ``app.synth()``
40 serially in-process for each config. Catches synth-time breakage; does
41 NOT catch cdk-nag findings because ``app.synth()`` exits 0 even when
42 unacknowledged findings exist.
44Typical workflow
45----------------
46 # A finding shows up in CI on your PR. Reproduce locally:
47 python3 scripts/dump_nag_findings.py
49 # Scope the acknowledgment in gco/stacks/nag_suppressions.py or
50 # whichever construct owns the resource, ideally with
51 # ``applies_to`` as tight as possible (prefer an exact literal
52 # ARN/detail over an unscoped blanket).
53 # Re-run:
54 python3 scripts/dump_nag_findings.py
55 # -> exits 0, no findings.
57 # Then run the full pytest gate to confirm (serially):
58 pytest tests/test_nag_compliance.py -q
59"""
61from __future__ import annotations
63import sys
64from pathlib import Path
65from typing import Any
67# Put the repo root on sys.path so ``tests._cdk_app_builder``,
68# ``tests._cdk_config_matrix`` and ``tests.test_nag_compliance`` are importable
69# when this script is
70# invoked as ``python3 scripts/dump_nag_findings.py`` from any CWD.
71REPO_ROOT = Path(__file__).resolve().parent.parent
72if str(REPO_ROOT) not in sys.path:
73 sys.path.insert(0, str(REPO_ROOT))
75from tests._cdk_app_builder import synthesize_app # noqa: E402
76from tests._cdk_config_matrix import NAG_CONFIGS as CONFIGS # noqa: E402
77from tests.test_nag_compliance import _build_app, _collect_nag_violations # noqa: E402
80def run_config(name: str, overrides: dict[str, object]) -> list[dict[str, Any]]:
81 """Build and synthesize the full CDK app under one config overlay.
83 Mirrors what ``tests/test_nag_compliance.py::TestCdkNagCompliance``
84 does, minus the pytest wiring: ``_build_app`` registers the rule packs and
85 ``synthesize_app`` builds every app.py stack and synthesizes offline, with
86 the Docker image assets stood in so no Docker daemon is required.
88 Returns the list of finding dicts read from ``validation-report.json``
89 (keys: ``pack`` / ``rule`` / ``description`` / ``paths``). An empty
90 list means the config is clean.
91 """
92 app = _build_app(context_overrides=overrides)
93 # Synthesis is what runs the validation plugins and writes
94 # validation-report.json, so the findings are read only afterwards.
95 synthesize_app(app)
96 return _collect_nag_violations(app)
99def main() -> int:
100 # Track findings as {(pack, rule, resource_path): [config_names]}
101 # so findings that repeat across configs collapse to one entry
102 # with a list of where they showed up. The keys are deliberately
103 # tuples (not dicts) so the ``sorted()`` call below is
104 # deterministic — same input produces same output, which matters
105 # when this output gets pasted into bug reports or commit
106 # messages.
107 all_findings: dict[tuple[str, str, str], list[str]] = {}
109 for name, overrides in CONFIGS:
110 print(f"\n{'=' * 72}")
111 print(f"CONFIG: {name}")
112 print(f"{'=' * 72}")
113 findings = run_config(name, overrides)
114 print(f" total findings: {len(findings)}")
115 for f in findings:
116 paths = f["paths"] or ["(no construct path)"]
117 for path in paths:
118 key = (str(f["pack"]), str(f["rule"]), str(path))
119 all_findings.setdefault(key, []).append(name)
121 print(f"\n{'=' * 72}")
122 print(f"UNIQUE FINDINGS ACROSS ALL CONFIGS: {len(all_findings)}")
123 print(f"{'=' * 72}")
124 for (pack, rule, path), cfgs in sorted(all_findings.items()):
125 print(f"\n [{pack}] {rule}")
126 print(f" path: {path}")
127 # dedupe + sort configs so order is stable
128 print(f" seen in: {', '.join(sorted(set(cfgs)))}")
130 # Non-zero exit if any finding exists — matches the pytest gate,
131 # so this script can be used as a quick pre-push smoke.
132 return 0 if not all_findings else 1
135if __name__ == "__main__":
136 raise SystemExit(main())