Coverage for cli / commands / monitoring_cmd.py: 100.00%
178 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-10-28 16:19 +0000
1"""GCO cluster observability command group.
3Provides the ``gco monitoring`` sub-commands:
5* ``enable`` / ``disable`` / ``status`` — flip the
6 ``cluster_observability.enabled`` toggle in ``cdk.json`` (on by default).
7* ``open`` — ``kubectl port-forward`` to Grafana / Prometheus / Alertmanager
8 over the PRIVATE EKS API endpoint. Because the endpoint is private by
9 default, ``open`` detects that posture and can tunnel to the API through an
10 SSM-managed instance (``--via-ssm``) so the forward works from a laptop.
11* ``users`` — manage Grafana users via the admin HTTP API (see
12 :mod:`cli.monitoring_user_mgmt`; wired in a follow-up command module).
14The Click wiring mirrors ``analytics_cmd.py``. In-cluster access always goes
15through the private API endpoint — there is no public Grafana ingress.
16"""
18from __future__ import annotations
20import sys
21from typing import Any
23import click
24import requests
26from .. import cluster_ui
27from ..config import GCOConfig
28from ..output import confirm, emit_structured_document, get_output_formatter
30pass_config = click.make_pass_decorator(GCOConfig, ensure=True)
32# kube-prometheus-stack + OpenCost service names (release name = chart key)
33# and their service ports, plus a sensible default local port for each.
34# OpenCost's UI container listens on 9090 in-cluster; its local default is
35# 9091 so `gco monitoring open --service opencost` can run alongside a
36# Prometheus forward on 9090 without a port clash.
37_SERVICES: dict[str, dict[str, Any]] = {
38 "grafana": {
39 "target": "svc/kube-prometheus-stack-grafana",
40 "remote_port": 80,
41 "default_local_port": 3000,
42 },
43 "prometheus": {
44 "target": "svc/kube-prometheus-stack-prometheus",
45 "remote_port": 9090,
46 "default_local_port": 9090,
47 },
48 "alertmanager": {
49 "target": "svc/kube-prometheus-stack-alertmanager",
50 "remote_port": 9093,
51 "default_local_port": 9093,
52 },
53 "opencost": {
54 "target": "svc/opencost",
55 "remote_port": 9090,
56 "default_local_port": 9091,
57 },
58 "opencost-api": {
59 "target": "svc/opencost",
60 "remote_port": 9003,
61 "default_local_port": 9003,
62 },
63 # MLflow tracking server (fullnameOverride keeps the bare release
64 # name). The official chart exposes the server 1:1 — Service port 5000
65 # to container port 5000 — and 5000 is also MLflow's canonical local
66 # port, so it runs alongside the Grafana (3000) and Prometheus (9090)
67 # forwards.
68 "mlflow": {
69 "target": "svc/mlflow",
70 "remote_port": 5000,
71 "default_local_port": 5000,
72 },
73}
75_MONITORING_NAMESPACE = "monitoring"
76_GRAFANA_SECRET = "kube-prometheus-stack-grafana" # nosec B105 # Kubernetes Secret name, not its contents
78# Default self-terminate backstop for an `--via-ssm auto` bastion.
79# Mirrors cli.ephemeral_bastion.DEFAULT_TTL_MINUTES (kept literal to avoid an
80# import at module load; the ephemeral_bastion module validates the range).
81_DEFAULT_BASTION_TTL_MINUTES = 120
84@click.group()
85@pass_config
86def monitoring(config: Any) -> None:
87 """Manage in-cluster observability (Prometheus + Grafana + Alertmanager)."""
90# ---------------------------------------------------------------------------
91# Toggle commands — status / enable / disable
92# ---------------------------------------------------------------------------
95@monitoring.command("status")
96@pass_config
97def monitoring_status(config: Any) -> None:
98 """Show the current cluster observability toggle state from cdk.json."""
99 from ..stacks import get_cluster_observability_config
101 formatter = get_output_formatter(config)
102 try:
103 current = get_cluster_observability_config()
104 formatter.print_info("Cluster observability config:")
105 formatter.print(current)
106 except Exception as exc: # surface every loader error
107 formatter.print_error(f"Failed to read cluster observability config: {exc}")
108 sys.exit(1)
111@monitoring.command("enable")
112@click.option("--yes", "-y", is_flag=True, help="Skip confirmation.")
113@pass_config
114def monitoring_enable(config: Any, yes: bool) -> None:
115 """Enable cluster observability in cdk.json (installs kube-prometheus-stack).
117 Observability is on by default; use this to re-enable after a
118 ``gco monitoring disable``. Prints the follow-up deploy command — does not
119 deploy automatically.
120 """
121 from ..stacks import update_cluster_observability_config
123 formatter = get_output_formatter(config)
124 if not yes:
125 formatter.print_info(
126 "Cluster observability (kube-prometheus-stack) will be enabled on every region."
127 )
128 confirm("\nEnable cluster observability?", abort=True)
130 try:
131 update_cluster_observability_config({"enabled": True})
132 formatter.print_success("Cluster observability enabled in cdk.json")
133 formatter.print_info(
134 f"Run `gco stacks deploy {config.project_name}-<region>` (or deploy-all) to apply"
135 )
136 except Exception as exc: # user-facing error from file I/O
137 formatter.print_error(f"Failed to enable cluster observability: {exc}")
138 sys.exit(1)
141@monitoring.command("disable")
142@click.option("--yes", "-y", is_flag=True, help="Skip confirmation.")
143@pass_config
144def monitoring_disable(config: Any, yes: bool) -> None:
145 """Disable cluster observability in cdk.json.
147 Flips ``cluster_observability.enabled`` to ``false``; the grafana /
148 prometheus / alertmanager sub-blocks (sizes, retention, rotation schedule)
149 are left untouched so preferences survive a disable/enable cycle. The
150 in-cluster stack is removed on the next deploy.
151 """
152 from ..stacks import update_cluster_observability_config
154 formatter = get_output_formatter(config)
155 if not yes:
156 formatter.print_warning("This will disable cluster observability.")
157 formatter.print_warning(
158 "Prometheus/Grafana/Alertmanager and their EBS volumes are removed on next deploy."
159 )
160 confirm("Are you sure?", abort=True)
162 try:
163 update_cluster_observability_config({"enabled": False})
164 formatter.print_success("Cluster observability disabled in cdk.json")
165 formatter.print_info(
166 f"Run `gco stacks deploy {config.project_name}-<region>` (or deploy-all) to apply"
167 )
168 except Exception as exc: # user-facing error from file I/O
169 formatter.print_error(f"Failed to disable cluster observability: {exc}")
170 sys.exit(1)
173# ---------------------------------------------------------------------------
174# open — port-forward (private endpoint aware)
175# ---------------------------------------------------------------------------
178@monitoring.command("open")
179@click.option(
180 "--service",
181 type=click.Choice(sorted(_SERVICES)),
182 default="grafana",
183 show_default=True,
184 help="Which component to port-forward.",
185)
186@click.option("--region", help="Cluster region (defaults to the first cdk.json regional entry).")
187@click.option("--local-port", type=int, help="Local port to bind (defaults per-service).")
188@click.option(
189 "--via-ssm",
190 "via_ssm",
191 metavar="INSTANCE_ID|auto",
192 help=(
193 "Tunnel to the private API endpoint through an SSM-managed instance. "
194 "Pass an instance id to use an existing one, or 'auto' to provision a "
195 "self-terminating ephemeral bastion and tear it down when the forward stops. "
196 "Used whenever private endpoint access is on, PUBLIC_AND_PRIVATE included."
197 ),
198)
199@click.option(
200 "--bastion-ttl-minutes",
201 type=int,
202 default=_DEFAULT_BASTION_TTL_MINUTES,
203 show_default=True,
204 help="Self-terminate backstop (minutes) for an `--via-ssm auto` bastion.",
205)
206@click.option(
207 "--yes",
208 "-y",
209 "assume_yes",
210 is_flag=True,
211 help="Skip the confirmation prompt when provisioning an `--via-ssm auto` bastion.",
212)
213@pass_config
214def monitoring_open(
215 config: Any,
216 service: str,
217 region: str | None,
218 local_port: int | None,
219 via_ssm: str | None,
220 bastion_ttl_minutes: int,
221 assume_yes: bool,
222) -> None:
223 """Port-forward to a monitoring component over the private EKS endpoint.
225 Runs in the foreground; press Ctrl-C to stop. On a private-endpoint cluster
226 (the default) pass ``--via-ssm <instance-id>`` to tunnel through an existing
227 SSM-managed instance, or ``--via-ssm auto`` to have the CLI provision a
228 minimal, self-terminating ephemeral bastion for the session and tear it down
229 on exit. ``--via-ssm`` is honored on a PUBLIC_AND_PRIVATE cluster too. The
230 local URL is printed once kubectl's listener is up, not before.
231 """
232 from ..cluster_tunnel import open_api_server_tunnel, resolve_region
233 from ..kubectl_helpers import build_port_forward_command, update_kubeconfig
235 formatter = get_output_formatter(config)
236 svc = _SERVICES[service]
237 target_region = resolve_region(config, region)
238 cluster = f"{config.project_name}-{target_region}"
239 bind_port = local_port or svc["default_local_port"]
241 try:
242 update_kubeconfig(cluster, target_region)
243 except (RuntimeError, ValueError) as exc:
244 formatter.print_error(str(exc))
245 sys.exit(1)
247 # The shared context manager detects the endpoint posture, optionally
248 # provisions an ephemeral bastion (`--via-ssm auto`), opens the SSM tunnel,
249 # and guarantees teardown of both the tunnel and any bastion on exit.
250 try:
251 with open_api_server_tunnel(
252 formatter,
253 cluster=cluster,
254 region=target_region,
255 via_ssm=via_ssm,
256 bastion_ttl_minutes=bastion_ttl_minutes,
257 assume_yes=assume_yes,
258 ) as session:
259 cmd = build_port_forward_command(
260 _MONITORING_NAMESPACE,
261 svc["target"],
262 bind_port,
263 svc["remote_port"],
264 server=session.server,
265 tls_server_name=session.tls_server_name,
266 )
267 url = f"http://localhost:{bind_port}"
268 formatter.print_info(
269 f"Starting kubectl port-forward for {service} on localhost:{bind_port}..."
270 )
272 def _announce() -> None:
273 formatter.print_success(f"Forwarding {service} → {url} (Ctrl-C to stop)")
274 if service == "grafana":
275 formatter.print_info(
276 "Log in with the Grafana admin credential from the "
277 f"{_GRAFANA_SECRET} Secret (monitoring namespace)."
278 )
280 try:
281 cluster_ui.exec_port_forward(cmd, bind_port, on_ready=_announce)
282 except KeyboardInterrupt: # pragma: no cover - interactive Ctrl-C
283 return
284 except (RuntimeError, ValueError) as exc:
285 formatter.print_error(str(exc))
286 sys.exit(1)
289# ---------------------------------------------------------------------------
290# users subgroup — Grafana native users over the admin HTTP API
291# ---------------------------------------------------------------------------
294def _grafana_conn_options(func: Any) -> Any:
295 """Shared --grafana-url / --admin-user / --admin-password options."""
296 from ..monitoring_user_mgmt import DEFAULT_GRAFANA_URL
298 func = click.option(
299 "--grafana-url",
300 default=DEFAULT_GRAFANA_URL,
301 show_default=True,
302 help="Grafana base URL (reachable via `gco monitoring open`).",
303 )(func)
304 func = click.option(
305 "--admin-user",
306 help="Grafana admin username (default: read from the Grafana Secret).",
307 )(func)
308 func = click.option(
309 "--admin-password",
310 envvar="GCO_GRAFANA_ADMIN_PASSWORD",
311 help=(
312 "Grafana admin password (also $GCO_GRAFANA_ADMIN_PASSWORD; "
313 "default: read from the Grafana Secret)."
314 ),
315 )(func)
316 return func
319def _resolve_grafana_auth(admin_user: str | None, admin_password: str | None) -> tuple[str, str]:
320 """Return ``(user, password)`` from the flags, else from the Grafana Secret."""
321 if admin_password:
322 return (admin_user or "admin", admin_password)
323 from ..monitoring_user_mgmt import read_grafana_admin_credentials
325 return read_grafana_admin_credentials()
328@monitoring.group("users")
329@pass_config
330def users_cmd(config: Any) -> None:
331 """Manage Grafana users via the admin API (over `gco monitoring open`)."""
334@users_cmd.command("add")
335@click.option("--username", required=True, help="Grafana login for the new user.")
336@click.option("--email", help="Email address for the new user (optional).")
337@click.option("--password", help="Set this password. Mutually exclusive with --generate-password.")
338@click.option(
339 "--generate-password",
340 is_flag=True,
341 help="Generate a strong random password and print it once.",
342)
343@_grafana_conn_options
344@pass_config
345def users_add(
346 config: Any,
347 username: str,
348 email: str | None,
349 password: str | None,
350 generate_password: bool,
351 grafana_url: str,
352 admin_user: str | None,
353 admin_password: str | None,
354) -> None:
355 """Create a Grafana user via the admin HTTP API."""
356 from ..monitoring_user_mgmt import create_user
357 from ..monitoring_user_mgmt import generate_password as _gen
359 formatter = get_output_formatter(config)
360 if password and generate_password:
361 formatter.print_error("--password and --generate-password are mutually exclusive")
362 sys.exit(1)
363 if not password and not generate_password:
364 formatter.print_error("Pass --password or --generate-password")
365 sys.exit(1)
367 final_password = password or _gen()
368 try:
369 auth = _resolve_grafana_auth(admin_user, admin_password)
370 user_id = create_user(
371 grafana_url, auth, login=username, password=final_password, email=email
372 )
373 except (requests.RequestException, RuntimeError, ValueError) as exc:
374 formatter.print_error(f"Failed to create Grafana user {username!r}: {exc}")
375 sys.exit(1)
377 if config.output_format == "table":
378 formatter.print_success(f"Created Grafana user {username!r} (id={user_id})")
379 if generate_password:
380 formatter.print_info(f"Generated password (printed exactly once): {final_password}")
381 else:
382 result: dict[str, Any] = {
383 "created": True,
384 "username": username,
385 "user_id": user_id,
386 "email": email,
387 "password_state": "set", # nosec B105 # state label, not a credential
388 "password_generated": generate_password,
389 "password_source": "generated" if generate_password else "provided",
390 }
391 if generate_password:
392 result["password"] = final_password
393 formatter.print(result)
396@users_cmd.command("list")
397@click.option("--as-json", "as_json", is_flag=True, help="Emit JSON instead of a table.")
398@_grafana_conn_options
399@pass_config
400def users_list(
401 config: Any,
402 as_json: bool,
403 grafana_url: str,
404 admin_user: str | None,
405 admin_password: str | None,
406) -> None:
407 """List Grafana organisation users."""
408 from ..monitoring_user_mgmt import list_users
410 formatter = get_output_formatter(config)
411 try:
412 auth = _resolve_grafana_auth(admin_user, admin_password)
413 users = list_users(grafana_url, auth)
414 except (requests.RequestException, RuntimeError, ValueError) as exc:
415 formatter.print_error(f"Failed to list Grafana users: {exc}")
416 sys.exit(1)
418 if as_json:
419 import json
421 emit_structured_document(
422 users,
423 output_format="json",
424 rendered=json.dumps(users, indent=2),
425 )
426 return
427 formatter.print(users)
430@users_cmd.command("remove")
431@click.option("--username", required=True, help="Grafana login/email to remove.")
432@click.option("--yes", is_flag=True, help="Skip the confirmation prompt.")
433@_grafana_conn_options
434@pass_config
435def users_remove(
436 config: Any,
437 username: str,
438 yes: bool,
439 grafana_url: str,
440 admin_user: str | None,
441 admin_password: str | None,
442) -> None:
443 """Delete a Grafana user by login or email."""
444 from ..monitoring_user_mgmt import delete_user, lookup_user_id
446 formatter = get_output_formatter(config)
447 if not yes:
448 confirm(f"Delete Grafana user '{username}'?", abort=True)
450 try:
451 auth = _resolve_grafana_auth(admin_user, admin_password)
452 user_id = lookup_user_id(grafana_url, auth, username)
453 delete_user(grafana_url, auth, user_id)
454 except (requests.RequestException, RuntimeError, ValueError) as exc:
455 formatter.print_error(f"Failed to remove Grafana user {username!r}: {exc}")
456 sys.exit(1)
458 formatter.print_success(f"Deleted Grafana user {username!r}")