Coverage for scripts / live_release_validation / inference_contract.py: 100.00%
95 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-09-14 22:07 +0000
1"""Strict immutable runtime contracts for the live inference matrix."""
3from __future__ import annotations
5import re
6from dataclasses import dataclass
7from typing import Any, Literal
9from cli._image_reference import immutable_sha256_digest
11Framework = Literal["vllm", "tgi"]
13INFERENCE_OWNER_LABEL = "gco-managed-inference-validation-owner"
14INFERENCE_CONTRACT_VERSION = 2
15_FRAMEWORK_ORDER: tuple[Framework, ...] = ("vllm", "tgi")
16_DEFAULT_PORTS: dict[Framework, int] = {"vllm": 8000, "tgi": 8080}
17_DEFAULT_REQUEST_PATHS: dict[Framework, str] = {
18 "vllm": "/v1/completions",
19 "tgi": "/generate",
20}
21_RESPONSE_CONTRACTS: dict[Framework, str] = {
22 "vllm": "choices[0].text:non-empty-string",
23 "tgi": "generated_text:non-empty-string",
24}
25_MODEL_INFO_PATHS: dict[Framework, str] = {
26 "vllm": "/v1/models",
27 "tgi": "/info",
28}
31@dataclass(frozen=True)
32class InferenceRuntimeSpec:
33 """One digest-pinned server and immutable model revision."""
35 framework: Framework
36 image: str
37 model_id: str
38 model_revision: str
39 port: int
41 @property
42 def request_path(self) -> str:
43 return _DEFAULT_REQUEST_PATHS[self.framework]
45 @property
46 def model_info_path(self) -> str:
47 return _MODEL_INFO_PATHS[self.framework]
50def _plain_positive_int(value: object) -> bool:
51 return isinstance(value, int) and not isinstance(value, bool) and value > 0
54def _validate_runtime(runtime: InferenceRuntimeSpec) -> None:
55 if runtime.framework not in _FRAMEWORK_ORDER:
56 raise ValueError("inference runtime framework must be 'vllm' or 'tgi'")
57 if immutable_sha256_digest(runtime.image) is None:
58 raise ValueError(
59 f"{runtime.framework} image must be an immutable lowercase @sha256: reference"
60 )
61 if not runtime.model_id.strip() or runtime.model_id != runtime.model_id.strip():
62 raise ValueError(f"{runtime.framework} model_id must be a non-empty trimmed value")
63 if not re.fullmatch(r"[0-9a-f]{40}", runtime.model_revision):
64 raise ValueError(
65 f"{runtime.framework} model_revision must be a full lowercase 40-hex commit"
66 )
67 if runtime.port != _DEFAULT_PORTS[runtime.framework]:
68 raise ValueError(
69 f"{runtime.framework} live validation port must be {_DEFAULT_PORTS[runtime.framework]}"
70 )
73def validate_inference_settings(settings: Any) -> None:
74 """Validate every run-level and per-runtime inference input."""
75 if not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)+", settings.selected_region):
76 raise ValueError("selected_region must be a lowercase AWS Region name")
77 runtimes = settings.inference_runtimes
78 if not isinstance(runtimes, tuple) or tuple(runtime.framework for runtime in runtimes) != (
79 "vllm",
80 "tgi",
81 ):
82 raise ValueError("managed inference validation requires vLLM then TGI runtime specs")
83 for runtime in runtimes:
84 _validate_runtime(runtime)
85 image_digests = [runtime.image.rsplit("@sha256:", 1)[1] for runtime in runtimes]
86 if len(set(image_digests)) != len(image_digests):
87 raise ValueError("vLLM and TGI runtime images must have distinct immutable digests")
88 if not settings.request_prompt.strip():
89 raise ValueError("request_prompt must be non-empty")
90 if not re.fullmatch(r"[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?", settings.namespace):
91 raise ValueError("namespace must be a DNS-safe Kubernetes name")
92 if (
93 not isinstance(settings.gpu_count, int)
94 or isinstance(settings.gpu_count, bool)
95 or settings.gpu_count < 0
96 ):
97 raise ValueError("gpu_count must be a non-negative integer")
98 for field_name in (
99 "request_max_tokens",
100 "baseline_replicas",
101 "autoscale_initial_replicas",
102 "hpa_min_replicas",
103 "hpa_max_replicas",
104 "endpoint_count",
105 "command_timeout_seconds",
106 "readiness_timeout_seconds",
107 "hpa_timeout_seconds",
108 "deletion_timeout_seconds",
109 "monitor_interval_seconds",
110 "hpa_stability_intervals",
111 "job_timeout_seconds",
112 "queue_timeout_seconds",
113 "poll_interval_seconds",
114 "destroy_attempts",
115 "destroy_retry_delay_seconds",
116 ):
117 if not _plain_positive_int(getattr(settings, field_name)):
118 raise ValueError(f"{field_name} must be a positive integer")
119 if settings.endpoint_count != 4:
120 raise ValueError("managed inference validation requires exactly four endpoints")
121 if settings.baseline_replicas != 1 or settings.autoscale_initial_replicas != 1:
122 raise ValueError("all managed inference endpoints must start with one replica")
123 if settings.hpa_min_replicas != 2 or settings.hpa_max_replicas != 2:
124 raise ValueError("the CPU HPA validation requires min_replicas=max_replicas=2")
125 if not isinstance(settings.hpa_cpu_target, int) or isinstance(settings.hpa_cpu_target, bool):
126 raise ValueError("hpa_cpu_target must be an integer")
127 if not 1 <= settings.hpa_cpu_target <= 100:
128 raise ValueError("hpa_cpu_target must be from 1 through 100")
129 if settings.hpa_stability_intervals < 2:
130 raise ValueError("hpa_stability_intervals must be at least two")
131 if settings.health_path != "/health":
132 raise ValueError("managed inference validation requires the official /health path")
133 if not re.fullmatch(r"[1-9][0-9]*m", settings.proxy_tls_cpu_request):
134 raise ValueError("proxy_tls_cpu_request must be a positive millicore quantity")
135 if (
136 not isinstance(settings.proxy_tls_cpu_target, int)
137 or isinstance(settings.proxy_tls_cpu_target, bool)
138 or not 1 <= settings.proxy_tls_cpu_target <= 100
139 ):
140 raise ValueError("proxy_tls_cpu_target must be an integer from 1 through 100")
141 if not settings.consent:
142 raise ValueError("explicit managed inference deployment/destruction consent is required")
145def inference_request_body(settings: Any, runtime: InferenceRuntimeSpec) -> dict[str, Any]:
146 """Return the deterministic request body for one exact framework adapter."""
147 if runtime.framework == "tgi":
148 return {
149 "inputs": settings.request_prompt,
150 "parameters": {
151 "do_sample": False,
152 "max_new_tokens": settings.request_max_tokens,
153 },
154 }
155 return {
156 "max_tokens": settings.request_max_tokens,
157 "model": runtime.model_id,
158 "prompt": settings.request_prompt,
159 "stream": False,
160 "temperature": 0,
161 }
164def inference_framework_env(runtime: InferenceRuntimeSpec) -> dict[str, str]:
165 """Return only official launcher environment for the selected runtime."""
166 if runtime.framework == "tgi":
167 return {
168 "MODEL_ID": runtime.model_id,
169 "PORT": str(runtime.port),
170 "REVISION": runtime.model_revision,
171 }
172 return {"MODEL": runtime.model_id}
175def inference_deploy_extra_args(runtime: InferenceRuntimeSpec) -> tuple[str, ...]:
176 """Return official immutable-model arguments for the selected runtime."""
177 if runtime.framework == "vllm":
178 return (
179 "--model",
180 runtime.model_id,
181 "--revision",
182 runtime.model_revision,
183 )
184 return ()
187def _runtime_identity(settings: Any, runtime: InferenceRuntimeSpec) -> dict[str, Any]:
188 return {
189 "framework": runtime.framework,
190 "image": runtime.image,
191 "model": {"id": runtime.model_id, "revision": runtime.model_revision},
192 "server": {
193 "port": runtime.port,
194 "health_path": settings.health_path,
195 },
196 "request_contract": {
197 "path": runtime.request_path,
198 "body": inference_request_body(settings, runtime),
199 "response": _RESPONSE_CONTRACTS[runtime.framework],
200 },
201 "probe_contract": {
202 "health_path": settings.health_path,
203 "model_info_path": runtime.model_info_path,
204 "expected_model_id": runtime.model_id,
205 "expected_model_revision": runtime.model_revision,
206 },
207 "deploy_contract": {
208 "framework_env": inference_framework_env(runtime),
209 "extra_args": list(inference_deploy_extra_args(runtime)),
210 },
211 }
214def inference_identity_fields(settings: Any) -> dict[str, Any]:
215 """Return every inference input that must match on resume."""
216 return {
217 "contract_version": INFERENCE_CONTRACT_VERSION,
218 "selected_region": settings.selected_region,
219 "runtimes": [
220 _runtime_identity(settings, runtime) for runtime in settings.inference_runtimes
221 ],
222 "endpoint_contract": {
223 "count": settings.endpoint_count,
224 "namespace": settings.namespace,
225 "gpu_count": settings.gpu_count,
226 "accelerator": "nvidia",
227 "rewrite_image": False,
228 "roles_per_runtime": ["baseline", "hpa"],
229 "baseline_replicas": settings.baseline_replicas,
230 "autoscale_initial_replicas": settings.autoscale_initial_replicas,
231 "hpa": {
232 "metric": "cpu",
233 "target": settings.hpa_cpu_target,
234 "min_replicas": settings.hpa_min_replicas,
235 "max_replicas": settings.hpa_max_replicas,
236 },
237 },
238 "shared_proxy_contract": {
239 "namespace": "gco-system",
240 "deployment": "inference-proxy",
241 "hpa": "inference-proxy-hpa",
242 "tls_container": "api-tls-proxy",
243 "tls_cpu_request": settings.proxy_tls_cpu_request,
244 "tls_cpu_target": settings.proxy_tls_cpu_target,
245 "metric_type": "ContainerResource",
246 },
247 "timeouts": {
248 "command_seconds": settings.command_timeout_seconds,
249 "readiness_seconds": settings.readiness_timeout_seconds,
250 "hpa_seconds": settings.hpa_timeout_seconds,
251 "deletion_seconds": settings.deletion_timeout_seconds,
252 "poll_seconds": settings.poll_interval_seconds,
253 "monitor_interval_seconds": settings.monitor_interval_seconds,
254 "hpa_stability_intervals": settings.hpa_stability_intervals,
255 "job_seconds": settings.job_timeout_seconds,
256 "queue_seconds": settings.queue_timeout_seconds,
257 "destroy_attempts": settings.destroy_attempts,
258 "destroy_retry_delay_seconds": settings.destroy_retry_delay_seconds,
259 },
260 "consent": settings.consent,
261 }