Coverage for scripts / upgrade_validation / sentinel.py: 100.00%
69 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-10-28 16:19 +0000
1"""A job template written through the API before the upgrade and read back after it.
3``gco upgrade`` promises that the control plane's shared state (the DynamoDB
4tables among it) survives: only the workload tier is destroyed and recreated.
5The sentinel tests that promise the way an operator would notice it breaking.
6It is created through the public API while the base release serves it, and
7after the upgrade the recreated regional stack must return it unchanged, with
8its original creation time, so the item was neither lost nor rewritten. It is
9deleted afterwards; the table itself goes with the global stack at teardown.
10"""
12from __future__ import annotations
14import hashlib
15import time
16from collections.abc import Callable
17from typing import Any
19import requests
21from cli.aws_client import APIRequestError
22from scripts.live_release_validation.models import RunContext
24TEMPLATES_PATH = "/api/v1/templates"
25#: Statuses that mean the API is not serving yet (or throttled), not an answer.
26RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504})
27RETRY_INTERVAL_SECONDS = 20.0
29Sleeper = Callable[[float], None]
30Clock = Callable[[], float]
33def sentinel_name(run_id: str) -> str:
34 """A template name unique to the run that fits the API's 63-character limit."""
35 return "gco-upgrade-sentinel-" + hashlib.sha256(run_id.encode("utf-8")).hexdigest()[:16]
38def sentinel_body(run_id: str) -> dict[str, Any]:
39 """The template the run writes; never submitted as a Job."""
40 return {
41 "name": sentinel_name(run_id),
42 "description": f"Upgrade-validation sentinel for run {run_id}",
43 "manifest": {
44 "apiVersion": "batch/v1",
45 "kind": "Job",
46 "metadata": {"name": "{{name}}", "namespace": "gco-jobs"},
47 "spec": {
48 "template": {
49 "spec": {
50 "restartPolicy": "Never",
51 "containers": [
52 {"name": "sentinel", "image": "{{image}}", "command": ["true"]}
53 ],
54 }
55 }
56 },
57 },
58 "parameters": {"image": "public.ecr.aws/docker/library/busybox:1.38.0"},
59 }
62def _call(
63 ctx: RunContext,
64 method: str,
65 path: str,
66 *,
67 deadline: float,
68 body: dict[str, Any] | None = None,
69 sleep: Sleeper = time.sleep,
70 clock: Clock = time.monotonic,
71) -> dict[str, Any]:
72 """One API call, retried while the API is not answering yet, until ``deadline``."""
73 attempts = 0
74 while True:
75 attempts += 1
76 try:
77 response: dict[str, Any] = ctx.aws_client.call_api(
78 method=method,
79 path=path,
80 body=body,
81 max_attempts=1 if method == "GET" else None,
82 )
83 return response
84 except APIRequestError as exc:
85 if exc.status_code not in RETRYABLE_STATUS_CODES:
86 raise
87 error: Exception = exc
88 except requests.RequestException as exc:
89 error = exc
90 if clock() >= deadline:
91 raise RuntimeError(
92 f"{method} {path} did not succeed in {attempts} attempt(s): "
93 f"{type(error).__name__}: {error}"
94 ) from error
95 sleep(RETRY_INTERVAL_SECONDS)
98def _require_written_template(template: Any, body: dict[str, Any]) -> dict[str, Any]:
99 if not isinstance(template, dict) or not template.get("created_at"):
100 raise RuntimeError(f"The API returned no template record for {body['name']}")
101 drift = sorted(
102 key
103 for key in ("name", "description", "manifest", "parameters")
104 if template.get(key) != body[key]
105 )
106 if drift:
107 raise RuntimeError(f"Template {body['name']} differs from what was written: {drift}")
108 return template
111def create_sentinel(
112 ctx: RunContext,
113 *,
114 timeout_seconds: float,
115 sleep: Sleeper = time.sleep,
116 clock: Clock = time.monotonic,
117) -> dict[str, Any]:
118 """Write the run's sentinel template, or confirm an earlier attempt wrote it."""
119 body = sentinel_body(ctx.settings.run_id)
120 deadline = clock() + timeout_seconds
121 try:
122 response = _call(
123 ctx, "POST", TEMPLATES_PATH, body=body, deadline=deadline, sleep=sleep, clock=clock
124 )
125 template = response.get("template")
126 except APIRequestError as exc:
127 if exc.status_code != 409:
128 raise
129 # A POST whose response was lost may still have written the item.
130 template = read_sentinel(ctx, body["name"], deadline=deadline, sleep=sleep, clock=clock)
131 return _require_written_template(template, body)
134def read_sentinel(
135 ctx: RunContext,
136 name: str,
137 *,
138 deadline: float,
139 sleep: Sleeper = time.sleep,
140 clock: Clock = time.monotonic,
141) -> Any:
142 """The sentinel template as the API returns it now."""
143 response = _call(
144 ctx, "GET", f"{TEMPLATES_PATH}/{name}", deadline=deadline, sleep=sleep, clock=clock
145 )
146 return response.get("template")
149def verify_sentinel(
150 ctx: RunContext,
151 written: dict[str, Any],
152 *,
153 timeout_seconds: float,
154 sleep: Sleeper = time.sleep,
155 clock: Clock = time.monotonic,
156) -> dict[str, Any]:
157 """Require the sentinel unchanged, including the creation time it was written with."""
158 body = sentinel_body(ctx.settings.run_id)
159 template = _require_written_template(
160 read_sentinel(
161 ctx, body["name"], deadline=clock() + timeout_seconds, sleep=sleep, clock=clock
162 ),
163 body,
164 )
165 if template.get("created_at") != written.get("created_at"):
166 raise RuntimeError(
167 f"Template {body['name']} was recreated: created {template.get('created_at')}, "
168 f"written {written.get('created_at')}"
169 )
170 return template
173def delete_sentinel(
174 ctx: RunContext,
175 *,
176 timeout_seconds: float,
177 sleep: Sleeper = time.sleep,
178 clock: Clock = time.monotonic,
179) -> dict[str, Any]:
180 """Delete the sentinel; a 404 means it is already gone."""
181 name = sentinel_name(ctx.settings.run_id)
182 try:
183 _call(
184 ctx,
185 "DELETE",
186 f"{TEMPLATES_PATH}/{name}",
187 deadline=clock() + timeout_seconds,
188 sleep=sleep,
189 clock=clock,
190 )
191 except APIRequestError as exc:
192 if exc.status_code != 404:
193 raise
194 return {"name": name, "deleted": False, "already_absent": True}
195 return {"name": name, "deleted": True}