Coverage for cli / commands / monitoring_cmd.py: 100.00%
178 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-09-14 22:07 +0000
1"""GCO cluster observability command group.
3Provides the ``gco monitoring`` sub-commands:
5* ``enable`` / ``disable`` / ``status`` — flip the
6 ``cluster_observability.enabled`` toggle in ``cdk.json`` (on by default).
7* ``open`` — ``kubectl port-forward`` to Grafana / Prometheus / Alertmanager
8 over the PRIVATE EKS API endpoint. Because the endpoint is private by
9 default, ``open`` detects that posture and can tunnel to the API through an
10 SSM-managed instance (``--via-ssm``) so the forward works from a laptop.
11* ``users`` — manage Grafana users via the admin HTTP API (see
12 :mod:`cli.monitoring_user_mgmt`; wired in a follow-up command module).
14The Click wiring mirrors ``analytics_cmd.py``. In-cluster access always goes
15through the private API endpoint — there is no public Grafana ingress.
16"""
18from __future__ import annotations
20import subprocess
21import sys
22from typing import Any
24import click
25import requests
27from ..config import GCOConfig
28from ..output import confirm, emit_structured_document, get_output_formatter
30pass_config = click.make_pass_decorator(GCOConfig, ensure=True)
32# kube-prometheus-stack + OpenCost service names (release name = chart key)
33# and their service ports, plus a sensible default local port for each.
34# OpenCost's UI container listens on 9090 in-cluster; its local default is
35# 9091 so `gco monitoring open --service opencost` can run alongside a
36# Prometheus forward on 9090 without a port clash.
37_SERVICES: dict[str, dict[str, Any]] = {
38 "grafana": {
39 "target": "svc/kube-prometheus-stack-grafana",
40 "remote_port": 80,
41 "default_local_port": 3000,
42 },
43 "prometheus": {
44 "target": "svc/kube-prometheus-stack-prometheus",
45 "remote_port": 9090,
46 "default_local_port": 9090,
47 },
48 "alertmanager": {
49 "target": "svc/kube-prometheus-stack-alertmanager",
50 "remote_port": 9093,
51 "default_local_port": 9093,
52 },
53 "opencost": {
54 "target": "svc/opencost",
55 "remote_port": 9090,
56 "default_local_port": 9091,
57 },
58 "opencost-api": {
59 "target": "svc/opencost",
60 "remote_port": 9003,
61 "default_local_port": 9003,
62 },
63 # MLflow tracking server (fullnameOverride keeps the bare release
64 # name). The official chart exposes the server 1:1 — Service port 5000
65 # to container port 5000 — and 5000 is also MLflow's canonical local
66 # port, so it runs alongside the Grafana (3000) and Prometheus (9090)
67 # forwards.
68 "mlflow": {
69 "target": "svc/mlflow",
70 "remote_port": 5000,
71 "default_local_port": 5000,
72 },
73}
75_MONITORING_NAMESPACE = "monitoring"
76_GRAFANA_SECRET = "kube-prometheus-stack-grafana"
78# Default self-terminate backstop for an `--via-ssm auto` bastion.
79# Mirrors cli.ephemeral_bastion.DEFAULT_TTL_MINUTES (kept literal to avoid an
80# import at module load; the ephemeral_bastion module validates the range).
81_DEFAULT_BASTION_TTL_MINUTES = 120
84@click.group()
85@pass_config
86def monitoring(config: Any) -> None:
87 """Manage in-cluster observability (Prometheus + Grafana + Alertmanager)."""
90# ---------------------------------------------------------------------------
91# Toggle commands — status / enable / disable
92# ---------------------------------------------------------------------------
95@monitoring.command("status")
96@pass_config
97def monitoring_status(config: Any) -> None:
98 """Show the current cluster observability toggle state from cdk.json."""
99 from ..stacks import get_cluster_observability_config
101 formatter = get_output_formatter(config)
102 try:
103 current = get_cluster_observability_config()
104 formatter.print_info("Cluster observability config:")
105 formatter.print(current)
106 except Exception as exc: # noqa: BLE001 — surface every loader error
107 formatter.print_error(f"Failed to read cluster observability config: {exc}")
108 sys.exit(1)
111@monitoring.command("enable")
112@click.option("--yes", "-y", is_flag=True, help="Skip confirmation.")
113@pass_config
114def monitoring_enable(config: Any, yes: bool) -> None:
115 """Enable cluster observability in cdk.json (installs kube-prometheus-stack).
117 Observability is on by default; use this to re-enable after a
118 ``gco monitoring disable``. Prints the follow-up deploy command — does not
119 deploy automatically.
120 """
121 from ..stacks import update_cluster_observability_config
123 formatter = get_output_formatter(config)
124 if not yes:
125 formatter.print_info(
126 "Cluster observability (kube-prometheus-stack) will be enabled on every region."
127 )
128 confirm("\nEnable cluster observability?", abort=True)
130 try:
131 update_cluster_observability_config({"enabled": True})
132 formatter.print_success("Cluster observability enabled in cdk.json")
133 formatter.print_info(
134 f"Run `gco stacks deploy {config.project_name}-<region>` (or deploy-all) to apply"
135 )
136 except Exception as exc: # noqa: BLE001 — user-facing error from file I/O
137 formatter.print_error(f"Failed to enable cluster observability: {exc}")
138 sys.exit(1)
141@monitoring.command("disable")
142@click.option("--yes", "-y", is_flag=True, help="Skip confirmation.")
143@pass_config
144def monitoring_disable(config: Any, yes: bool) -> None:
145 """Disable cluster observability in cdk.json.
147 Flips ``cluster_observability.enabled`` to ``false``; the grafana /
148 prometheus / alertmanager sub-blocks (sizes, retention, rotation schedule)
149 are left untouched so preferences survive a disable/enable cycle. The
150 in-cluster stack is removed on the next deploy.
151 """
152 from ..stacks import update_cluster_observability_config
154 formatter = get_output_formatter(config)
155 if not yes:
156 formatter.print_warning("This will disable cluster observability.")
157 formatter.print_warning(
158 "Prometheus/Grafana/Alertmanager and their EBS volumes are removed on next deploy."
159 )
160 confirm("Are you sure?", abort=True)
162 try:
163 update_cluster_observability_config({"enabled": False})
164 formatter.print_success("Cluster observability disabled in cdk.json")
165 formatter.print_info(
166 f"Run `gco stacks deploy {config.project_name}-<region>` (or deploy-all) to apply"
167 )
168 except Exception as exc: # noqa: BLE001 — user-facing error from file I/O
169 formatter.print_error(f"Failed to disable cluster observability: {exc}")
170 sys.exit(1)
173# ---------------------------------------------------------------------------
174# open — port-forward (private endpoint aware)
175# ---------------------------------------------------------------------------
178@monitoring.command("open")
179@click.option(
180 "--service",
181 type=click.Choice(sorted(_SERVICES)),
182 default="grafana",
183 show_default=True,
184 help="Which component to port-forward.",
185)
186@click.option("--region", help="Cluster region (defaults to the first cdk.json regional entry).")
187@click.option("--local-port", type=int, help="Local port to bind (defaults per-service).")
188@click.option(
189 "--via-ssm",
190 "via_ssm",
191 metavar="INSTANCE_ID|auto",
192 help=(
193 "Tunnel to the private API endpoint through an SSM-managed instance. "
194 "Pass an instance id to use an existing one, or 'auto' to provision a "
195 "self-terminating ephemeral bastion and tear it down when the forward stops."
196 ),
197)
198@click.option(
199 "--bastion-ttl-minutes",
200 type=int,
201 default=_DEFAULT_BASTION_TTL_MINUTES,
202 show_default=True,
203 help="Self-terminate backstop (minutes) for an `--via-ssm auto` bastion.",
204)
205@click.option(
206 "--yes",
207 "-y",
208 "assume_yes",
209 is_flag=True,
210 help="Skip the confirmation prompt when provisioning an `--via-ssm auto` bastion.",
211)
212@pass_config
213def monitoring_open(
214 config: Any,
215 service: str,
216 region: str | None,
217 local_port: int | None,
218 via_ssm: str | None,
219 bastion_ttl_minutes: int,
220 assume_yes: bool,
221) -> None:
222 """Port-forward to a monitoring component over the private EKS endpoint.
224 Runs in the foreground; press Ctrl-C to stop. On a private-endpoint cluster
225 (the default) pass ``--via-ssm <instance-id>`` to tunnel through an existing
226 SSM-managed instance, or ``--via-ssm auto`` to have the CLI provision a
227 minimal, self-terminating ephemeral bastion for the session and tear it down
228 on exit.
229 """
230 from ..cluster_tunnel import open_api_server_tunnel, resolve_region
231 from ..kubectl_helpers import build_port_forward_command, update_kubeconfig
233 formatter = get_output_formatter(config)
234 svc = _SERVICES[service]
235 target_region = resolve_region(config, region)
236 cluster = f"{config.project_name}-{target_region}"
237 bind_port = local_port or svc["default_local_port"]
239 try:
240 update_kubeconfig(cluster, target_region)
241 except (RuntimeError, ValueError) as exc:
242 formatter.print_error(str(exc))
243 sys.exit(1)
245 # The shared context manager detects the endpoint posture, optionally
246 # provisions an ephemeral bastion (`--via-ssm auto`), opens the SSM tunnel,
247 # and guarantees teardown of both the tunnel and any bastion on exit.
248 try:
249 with open_api_server_tunnel(
250 formatter,
251 cluster=cluster,
252 region=target_region,
253 via_ssm=via_ssm,
254 bastion_ttl_minutes=bastion_ttl_minutes,
255 assume_yes=assume_yes,
256 ) as session:
257 cmd = build_port_forward_command(
258 _MONITORING_NAMESPACE,
259 svc["target"],
260 bind_port,
261 svc["remote_port"],
262 server=session.server,
263 tls_server_name=session.tls_server_name,
264 )
265 url = f"http://localhost:{bind_port}"
266 formatter.print_success(f"Forwarding {service} → {url} (Ctrl-C to stop)")
267 if service == "grafana":
268 formatter.print_info(
269 "Log in with the Grafana admin credential from the "
270 f"{_GRAFANA_SECRET} Secret (monitoring namespace)."
271 )
272 try:
273 _exec_port_forward(cmd)
274 except KeyboardInterrupt: # pragma: no cover - interactive Ctrl-C
275 return
276 except (RuntimeError, ValueError) as exc:
277 formatter.print_error(str(exc))
278 sys.exit(1)
281def _exec_port_forward(cmd: list[str]) -> None:
282 """Run the (validated) kubectl port-forward argv in the foreground."""
283 subprocess.run(
284 cmd, check=False
285 ) # nosemgrep: dangerous-subprocess-use-audit - argv built by build_port_forward_command; list form, no shell=True
288# ---------------------------------------------------------------------------
289# users subgroup — Grafana native users over the admin HTTP API
290# ---------------------------------------------------------------------------
293def _grafana_conn_options(func: Any) -> Any:
294 """Shared --grafana-url / --admin-user / --admin-password options."""
295 from ..monitoring_user_mgmt import DEFAULT_GRAFANA_URL
297 func = click.option(
298 "--grafana-url",
299 default=DEFAULT_GRAFANA_URL,
300 show_default=True,
301 help="Grafana base URL (reachable via `gco monitoring open`).",
302 )(func)
303 func = click.option(
304 "--admin-user",
305 help="Grafana admin username (default: read from the Grafana Secret).",
306 )(func)
307 func = click.option(
308 "--admin-password",
309 envvar="GCO_GRAFANA_ADMIN_PASSWORD",
310 help=(
311 "Grafana admin password (also $GCO_GRAFANA_ADMIN_PASSWORD; "
312 "default: read from the Grafana Secret)."
313 ),
314 )(func)
315 return func
318def _resolve_grafana_auth(admin_user: str | None, admin_password: str | None) -> tuple[str, str]:
319 """Return ``(user, password)`` from the flags, else from the Grafana Secret."""
320 if admin_password:
321 return (admin_user or "admin", admin_password)
322 from ..monitoring_user_mgmt import read_grafana_admin_credentials
324 return read_grafana_admin_credentials()
327@monitoring.group("users")
328@pass_config
329def users_cmd(config: Any) -> None:
330 """Manage Grafana users via the admin API (over `gco monitoring open`)."""
333@users_cmd.command("add")
334@click.option("--username", required=True, help="Grafana login for the new user.")
335@click.option("--email", help="Email address for the new user (optional).")
336@click.option("--password", help="Set this password. Mutually exclusive with --generate-password.")
337@click.option(
338 "--generate-password",
339 is_flag=True,
340 help="Generate a strong random password and print it once.",
341)
342@_grafana_conn_options
343@pass_config
344def users_add(
345 config: Any,
346 username: str,
347 email: str | None,
348 password: str | None,
349 generate_password: bool,
350 grafana_url: str,
351 admin_user: str | None,
352 admin_password: str | None,
353) -> None:
354 """Create a Grafana user via the admin HTTP API."""
355 from ..monitoring_user_mgmt import create_user
356 from ..monitoring_user_mgmt import generate_password as _gen
358 formatter = get_output_formatter(config)
359 if password and generate_password:
360 formatter.print_error("--password and --generate-password are mutually exclusive")
361 sys.exit(1)
362 if not password and not generate_password:
363 formatter.print_error("Pass --password or --generate-password")
364 sys.exit(1)
366 final_password = password or _gen()
367 try:
368 auth = _resolve_grafana_auth(admin_user, admin_password)
369 user_id = create_user(
370 grafana_url, auth, login=username, password=final_password, email=email
371 )
372 except (requests.RequestException, RuntimeError, ValueError) as exc:
373 formatter.print_error(f"Failed to create Grafana user {username!r}: {exc}")
374 sys.exit(1)
376 if config.output_format == "table":
377 formatter.print_success(f"Created Grafana user {username!r} (id={user_id})")
378 if generate_password:
379 formatter.print_info(f"Generated password (printed exactly once): {final_password}")
380 else:
381 result: dict[str, Any] = {
382 "created": True,
383 "username": username,
384 "user_id": user_id,
385 "email": email,
386 "password_state": "set",
387 "password_generated": generate_password,
388 "password_source": "generated" if generate_password else "provided",
389 }
390 if generate_password:
391 result["password"] = final_password
392 formatter.print(result)
395@users_cmd.command("list")
396@click.option("--as-json", "as_json", is_flag=True, help="Emit JSON instead of a table.")
397@_grafana_conn_options
398@pass_config
399def users_list(
400 config: Any,
401 as_json: bool,
402 grafana_url: str,
403 admin_user: str | None,
404 admin_password: str | None,
405) -> None:
406 """List Grafana organisation users."""
407 from ..monitoring_user_mgmt import list_users
409 formatter = get_output_formatter(config)
410 try:
411 auth = _resolve_grafana_auth(admin_user, admin_password)
412 users = list_users(grafana_url, auth)
413 except (requests.RequestException, RuntimeError, ValueError) as exc:
414 formatter.print_error(f"Failed to list Grafana users: {exc}")
415 sys.exit(1)
417 if as_json:
418 import json
420 emit_structured_document(
421 users,
422 output_format="json",
423 rendered=json.dumps(users, indent=2),
424 )
425 return
426 formatter.print(users)
429@users_cmd.command("remove")
430@click.option("--username", required=True, help="Grafana login/email to remove.")
431@click.option("--yes", is_flag=True, help="Skip the confirmation prompt.")
432@_grafana_conn_options
433@pass_config
434def users_remove(
435 config: Any,
436 username: str,
437 yes: bool,
438 grafana_url: str,
439 admin_user: str | None,
440 admin_password: str | None,
441) -> None:
442 """Delete a Grafana user by login or email."""
443 from ..monitoring_user_mgmt import delete_user, lookup_user_id
445 formatter = get_output_formatter(config)
446 if not yes:
447 confirm(f"Delete Grafana user '{username}'?", abort=True)
449 try:
450 auth = _resolve_grafana_auth(admin_user, admin_password)
451 user_id = lookup_user_id(grafana_url, auth, username)
452 delete_user(grafana_url, auth, user_id)
453 except (requests.RequestException, RuntimeError, ValueError) as exc:
454 formatter.print_error(f"Failed to remove Grafana user {username!r}: {exc}")
455 sys.exit(1)
457 formatter.print_success(f"Deleted Grafana user {username!r}")