Coverage for scripts / live_release_validation / inference_contract.py: 100.00%
164 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-10-28 16:19 +0000
1"""Strict immutable runtime contracts for the live inference matrix.
3The matrix proves three serving runtimes end to end: vLLM through its
4OpenAI-compatible surface, SGLang through its native surface, and Triton
5Inference Server through the KServe v2 protocol with weights served from the
6model registry (``gco models upload`` and ``--model-source``). vLLM is also
7proven as a Mooncake shared KV-cache store (``--mooncake-mode store`` with the
8S3 cold tier), which is the path that exercises the inference monitor's
9role-keyed reconciliation, the per-region ``mooncake-master``, the regional
10bucket discovery grant, and the rollout that follows ``configure-store``.
11Hugging Face Text Generation Inference used to be the second runtime; it
12entered maintenance mode on 2025-12-11 and its repository was archived
13read-only on 2026-03-21, so it is no longer something a release should be
14proven against.
15"""
17from __future__ import annotations
19import copy
20import hashlib
21import re
22from dataclasses import dataclass
23from pathlib import Path
24from typing import Any, Literal
26from cli._image_reference import immutable_sha256_digest
28Framework = Literal["vllm", "sglang", "triton"]
29#: One scenario of the matrix. ``baseline`` is a static single replica,
30#: ``hpa`` a CPU-autoscaled pair, ``store`` a Mooncake shared KV-cache store
31#: with the cold tier (vLLM only).
32EndpointRole = Literal["baseline", "hpa", "store"]
34INFERENCE_OWNER_LABEL = "gco-managed-inference-validation-owner"
35# Bump whenever the matrix below changes shape: the version is part of the
36# resume identity, so a checkpoint written under an older contract refuses to
37# resume against a newer one instead of silently mixing adapters.
38INFERENCE_CONTRACT_VERSION = 5
39_FRAMEWORK_ORDER: tuple[Framework, ...] = ("vllm", "sglang", "triton")
40#: The scenarios each runtime runs, in order. The classic baseline/HPA pair
41#: covers the single-Deployment path; vLLM adds the Mooncake store scenario.
42#: Triton runs the baseline only: its HPA path is the same Deployment HPA the
43#: vLLM scenario already proves, and a second GPU replica would buy nothing.
44_ROLES_BY_FRAMEWORK: dict[Framework, tuple[EndpointRole, ...]] = {
45 "vllm": ("baseline", "hpa", "store"),
46 "sglang": ("baseline", "hpa"),
47 "triton": ("baseline",),
48}
49# Each runtime's documented default listener: vLLM's OpenAI server,
50# ``sglang.launch_server`` (``--port`` default 30000) and Triton's HTTP
51# endpoint.
52_DEFAULT_PORTS: dict[Framework, int] = {"vllm": 8000, "sglang": 30000, "triton": 8000}
53_DEFAULT_REQUEST_PATHS: dict[Framework, str] = {
54 "vllm": "/v1/completions",
55 "sglang": "/generate",
56}
57_RESPONSE_CONTRACTS: dict[Framework, str] = {
58 "vllm": "choices[0].text:non-empty-string",
59 "sglang": "text:non-empty-string",
60 "triton": "outputs[0].data:echoes-input",
61}
62# vLLM lists the served model through the OpenAI inventory; SGLang's
63# ``/server_info`` reports the resolved launcher arguments, so it carries
64# both ``model_path`` and the ``revision`` the server was started with; Triton
65# has no model list, so its statistics extension names every loaded model.
66_MODEL_INFO_PATHS: dict[Framework, str] = {
67 "vllm": "/v1/models",
68 "sglang": "/server_info",
69 "triton": "/v2/models/stats",
70}
71#: Triton reports readiness on its own KServe path; the other runtimes use the
72#: run-level ``/health``.
73_HEALTH_PATHS: dict[Framework, str] = {"triton": "/v2/health/ready"}
75#: vLLM's structured decisions API (v0.32.0+). The vLLM baseline scenario
76#: calls it after its completion through the same authenticated path, which
77#: proves the inference proxy forwards it and the pinned image serves it.
78STRUCTURED_DECISION_PATH = "/v1/systemone"
79#: The scenarios that also run the structured decision request. One is enough:
80#: the HPA and store scenarios run the same server with the same arguments.
81_STRUCTURED_DECISION_ROLES: dict[Framework, tuple[str, ...]] = {"vllm": ("baseline",)}
82#: vLLM serves ``/v1/systemone`` for a generation model only when its log
83#: probabilities are real probabilities (``label_mass`` sums them), so every
84#: vLLM scenario starts the server this way.
85STRUCTURED_DECISION_LOGPROBS_MODE = "raw_logprobs"
86#: Model ids vLLM v0.32.0 registers a structured decision read strategy for:
87#: the Qwen3 / Qwen3.5 families, LiquidAI's d1 and Cloudflare's clef-omni
88#: (``READ_STRATEGIES`` in vllm/entrypoints/generate/structured_decisions/
89#: strategies.py, keyed by architecture). An id outside them would start
90#: normally and then answer ``/v1/systemone`` with "does not support", hours
91#: into a run, so the matrix refuses it up front.
92_STRUCTURED_DECISION_MODEL_RE = re.compile(r"(?i)(?:^|/)(?:qwen3|d1-|clef-omni)")
93#: One question of each type, over a state whose answers are unambiguous, so a
94#: served answer is checked for its shape and for the obvious label.
95STRUCTURED_DECISION_STATE = (
96 "Order 1182 was promised for Monday and shipped nine days late. "
97 "The customer has asked for a refund twice."
98)
99STRUCTURED_DECISION_QUESTIONS: dict[str, dict[str, Any]] = {
100 "topic": {
101 "type": "choice",
102 "instructions": "What is this message mainly about?",
103 "criteria": {
104 "shipping": "Delivery dates or delays",
105 "weather": "The weather forecast",
106 },
107 },
108 "late": {"type": "noul", "instructions": "Did the order ship late?"},
109 "urgency": {
110 "type": "score",
111 "instructions": "How urgent is a reply?",
112 "criteria": ["low", "medium", "high"],
113 },
114}
115_STRUCTURED_DECISION_RESPONSE_CONTRACT = (
116 "object==structured_decision; answers[topic]=choice in criteria, "
117 "answers[late]=noul in [0,1], answers[urgency]=score in [0,2]; "
118 "probabilities sum to 1; diagnostics.label_mass in [0,1] per question"
119)
121#: The Triton model the harness serves. It is a Python-backend identity model
122#: generated by the harness itself (see :data:`TRITON_MODEL_REPOSITORY_FILES`),
123#: uploaded to the model registry before the scenario and removed after it, so
124#: the run needs no third-party weights and the response can be checked
125#: exactly: the output tensor must echo the input tensor.
126TRITON_MODEL_NAME = "identity"
127TRITON_MODEL_VERSION = "1"
128TRITON_INPUT_DATA: tuple[float, ...] = (1.0, 2.0, 3.0)
129TRITON_MODEL_REPOSITORY_FILES: dict[str, str] = {
130 f"{TRITON_MODEL_NAME}/config.pbtxt": (
131 f'name: "{TRITON_MODEL_NAME}"\n'
132 'backend: "python"\n'
133 "max_batch_size: 0\n"
134 'input [ { name: "INPUT0" data_type: TYPE_FP32 dims: [ -1 ] } ]\n'
135 'output [ { name: "OUTPUT0" data_type: TYPE_FP32 dims: [ -1 ] } ]\n'
136 "instance_group [ { kind: KIND_CPU count: 1 } ]\n"
137 ),
138 f"{TRITON_MODEL_NAME}/{TRITON_MODEL_VERSION}/model.py": (
139 "import triton_python_backend_utils as pb_utils\n"
140 "\n"
141 "\n"
142 "class TritonPythonModel:\n"
143 " def execute(self, requests):\n"
144 " responses = []\n"
145 " for request in requests:\n"
146 ' tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0")\n'
147 ' output = pb_utils.Tensor("OUTPUT0", tensor.as_numpy())\n'
148 " responses.append(pb_utils.InferenceResponse(output_tensors=[output]))\n"
149 " return responses\n"
150 ),
151}
152#: Where the deploy's init container syncs ``--model-source`` to, and
153#: therefore the ``--model-repository`` Triton is started with. The last path
154#: element is the endpoint name.
155TRITON_MODEL_REPOSITORY_MOUNT = "/models"
156#: The pod-template annotation the inference monitor stamps with the digest of
157#: the rendered ``mooncake.json`` (mirrors
158#: ``gco.services.inference_monitor.MOONCAKE_CONFIG_DIGEST_ANNOTATION``; the
159#: unit tests pin the two together). A changed digest is how the harness
160#: proves a ``configure-store`` rolled the role pods.
161MOONCAKE_CONFIG_DIGEST_ANNOTATION = "gco.io/mooncake-config-digest"
162#: The byte-size ``configure-store`` writes during the store scenario so the
163#: re-rendered ``mooncake.json`` differs from the deploy-time file (the default
164#: local buffer is 2 GiB) and the role pods must roll.
165MOONCAKE_STORE_RECONFIGURED_LOCAL_BUFFER_BYTES = 1073741824
168def triton_model_repository_revision() -> str:
169 """Return the content digest of the generated Triton repository, as a 40-hex pin.
171 The Triton runtime has no upstream model commit to pin; its model is the
172 repository this module generates. Hashing every file path and content gives
173 the same immutable-revision shape the Hugging Face runtimes carry, so the
174 identity fields detect a changed repository exactly as they detect a
175 changed model commit.
176 """
177 digest = hashlib.sha256()
178 for path, content in sorted(TRITON_MODEL_REPOSITORY_FILES.items()):
179 digest.update(path.encode("utf-8"))
180 digest.update(b"\0")
181 digest.update(content.encode("utf-8"))
182 digest.update(b"\0")
183 return digest.hexdigest()[:40]
186def write_triton_model_repository(root: Path) -> Path:
187 """Materialize the generated Triton model repository under ``root``.
189 Returns the repository root, laid out as Triton expects
190 (``<model>/config.pbtxt`` and ``<model>/<version>/model.py``) so
191 ``gco models upload <root> --name <model>`` publishes a repository the
192 endpoint's ``--model-repository`` can serve directly.
193 """
194 for relative, content in TRITON_MODEL_REPOSITORY_FILES.items():
195 target = root / relative
196 target.parent.mkdir(parents=True, exist_ok=True)
197 target.write_text(content, encoding="utf-8")
198 return root
201@dataclass(frozen=True)
202class InferenceRuntimeSpec:
203 """One digest-pinned server and immutable model revision.
205 For Triton ``model_id`` is the generated repository's model name and
206 ``model_revision`` its content digest (:func:`triton_model_repository_revision`).
207 """
209 framework: Framework
210 image: str
211 model_id: str
212 model_revision: str
213 port: int
215 @property
216 def request_path(self) -> str:
217 if self.framework == "triton":
218 return f"/v2/models/{self.model_id}/infer"
219 return _DEFAULT_REQUEST_PATHS[self.framework]
221 @property
222 def model_info_path(self) -> str:
223 return _MODEL_INFO_PATHS[self.framework]
225 @property
226 def roles(self) -> tuple[EndpointRole, ...]:
227 """The scenarios this runtime runs, in matrix order."""
228 return _ROLES_BY_FRAMEWORK[self.framework]
230 @property
231 def structured_decision_roles(self) -> tuple[str, ...]:
232 """The scenarios that also call ``/v1/systemone`` (vLLM baseline only)."""
233 return _STRUCTURED_DECISION_ROLES.get(self.framework, ())
235 @property
236 def uses_model_registry(self) -> bool:
237 """Whether the deploy serves weights from the model registry (``--model-source``)."""
238 return self.framework == "triton"
241def expected_endpoint_count(runtimes: tuple[InferenceRuntimeSpec, ...]) -> int:
242 """The number of scenarios the matrix runs for ``runtimes``."""
243 return sum(len(runtime.roles) for runtime in runtimes)
246def _plain_positive_int(value: object) -> bool:
247 return isinstance(value, int) and not isinstance(value, bool) and value > 0
250def _validate_runtime(runtime: InferenceRuntimeSpec) -> None:
251 if runtime.framework not in _FRAMEWORK_ORDER:
252 raise ValueError("inference runtime framework must be 'vllm', 'sglang' or 'triton'")
253 if immutable_sha256_digest(runtime.image) is None:
254 raise ValueError(
255 f"{runtime.framework} image must be an immutable lowercase @sha256: reference"
256 )
257 if not runtime.model_id.strip() or runtime.model_id != runtime.model_id.strip():
258 raise ValueError(f"{runtime.framework} model_id must be a non-empty trimmed value")
259 if not re.fullmatch(r"[0-9a-f]{40}", runtime.model_revision):
260 raise ValueError(
261 f"{runtime.framework} model_revision must be a full lowercase 40-hex commit"
262 )
263 if runtime.framework == "triton" and (
264 runtime.model_id != TRITON_MODEL_NAME
265 or runtime.model_revision != triton_model_repository_revision()
266 ):
267 raise ValueError(
268 "triton model identity must be the harness-generated repository "
269 f"({TRITON_MODEL_NAME} at revision {triton_model_repository_revision()})"
270 )
271 if runtime.port != _DEFAULT_PORTS[runtime.framework]:
272 raise ValueError(
273 f"{runtime.framework} live validation port must be {_DEFAULT_PORTS[runtime.framework]}"
274 )
275 if runtime.structured_decision_roles and not _STRUCTURED_DECISION_MODEL_RE.search(
276 runtime.model_id
277 ):
278 raise ValueError(
279 f"{runtime.framework} model_id must be a model vLLM serves /v1/systemone for "
280 "(a Qwen3 or Qwen3.5 model, LiquidAI/d1-*, or Cloudflare/clef-omni)"
281 )
284def validate_inference_settings(settings: Any) -> None:
285 """Validate every run-level and per-runtime inference input."""
286 if not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)+", settings.selected_region):
287 raise ValueError("selected_region must be a lowercase AWS Region name")
288 runtimes = settings.inference_runtimes
289 if (
290 not isinstance(runtimes, tuple)
291 or tuple(runtime.framework for runtime in runtimes) != _FRAMEWORK_ORDER
292 ):
293 raise ValueError(
294 "managed inference validation requires vLLM, SGLang then Triton runtime specs"
295 )
296 for runtime in runtimes:
297 _validate_runtime(runtime)
298 image_digests = [runtime.image.rsplit("@sha256:", 1)[1] for runtime in runtimes]
299 if len(set(image_digests)) != len(image_digests):
300 raise ValueError("every runtime image must have a distinct immutable digest")
301 if not settings.request_prompt.strip():
302 raise ValueError("request_prompt must be non-empty")
303 if not re.fullmatch(r"[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?", settings.namespace):
304 raise ValueError("namespace must be a DNS-safe Kubernetes name")
305 if (
306 not isinstance(settings.gpu_count, int)
307 or isinstance(settings.gpu_count, bool)
308 or settings.gpu_count < 0
309 ):
310 raise ValueError("gpu_count must be a non-negative integer")
311 for field_name in (
312 "request_max_tokens",
313 "baseline_replicas",
314 "autoscale_initial_replicas",
315 "hpa_min_replicas",
316 "hpa_max_replicas",
317 "command_timeout_seconds",
318 "readiness_timeout_seconds",
319 "hpa_timeout_seconds",
320 "deletion_timeout_seconds",
321 "monitor_interval_seconds",
322 "hpa_stability_intervals",
323 "job_timeout_seconds",
324 "queue_timeout_seconds",
325 "poll_interval_seconds",
326 "destroy_attempts",
327 "destroy_retry_delay_seconds",
328 ):
329 if not _plain_positive_int(getattr(settings, field_name)):
330 raise ValueError(f"{field_name} must be a positive integer")
331 expected_count = expected_endpoint_count(runtimes)
332 if settings.endpoint_count != expected_count:
333 raise ValueError(
334 f"managed inference validation requires exactly {expected_count} endpoints "
335 "(every scenario of every runtime)"
336 )
337 if settings.baseline_replicas != 1 or settings.autoscale_initial_replicas != 1:
338 raise ValueError("all managed inference endpoints must start with one replica")
339 if settings.hpa_min_replicas != 2 or settings.hpa_max_replicas != 2:
340 raise ValueError("the CPU HPA validation requires min_replicas=max_replicas=2")
341 if not isinstance(settings.hpa_cpu_target, int) or isinstance(settings.hpa_cpu_target, bool):
342 raise ValueError("hpa_cpu_target must be an integer")
343 if not 1 <= settings.hpa_cpu_target <= 100:
344 raise ValueError("hpa_cpu_target must be from 1 through 100")
345 if settings.hpa_stability_intervals < 2:
346 raise ValueError("hpa_stability_intervals must be at least two")
347 if settings.health_path != "/health":
348 raise ValueError("managed inference validation requires the official /health path")
349 if not re.fullmatch(r"[1-9][0-9]*m", settings.proxy_tls_cpu_request):
350 raise ValueError("proxy_tls_cpu_request must be a positive millicore quantity")
351 if (
352 not isinstance(settings.proxy_tls_cpu_target, int)
353 or isinstance(settings.proxy_tls_cpu_target, bool)
354 or not 1 <= settings.proxy_tls_cpu_target <= 100
355 ):
356 raise ValueError("proxy_tls_cpu_target must be an integer from 1 through 100")
357 if not settings.consent:
358 raise ValueError("explicit managed inference deployment/destruction consent is required")
361def inference_request_body(settings: Any, runtime: InferenceRuntimeSpec) -> dict[str, Any]:
362 """Return the deterministic request body for one exact framework adapter.
364 SGLang is driven through its native ``/generate`` API rather than its
365 OpenAI-compatible one on purpose: the two runtimes must exercise
366 different wire contracts, otherwise a response that only satisfies the
367 OpenAI shape would pass for both and the matrix would prove one adapter
368 twice.
369 """
370 if runtime.framework == "sglang":
371 return {
372 "sampling_params": {
373 "max_new_tokens": settings.request_max_tokens,
374 "temperature": 0,
375 },
376 "text": settings.request_prompt,
377 }
378 if runtime.framework == "triton":
379 # KServe v2: one FP32 input tensor the identity model echoes back, so
380 # the response is checked for equality rather than for non-emptiness.
381 return {
382 "inputs": [
383 {
384 "name": "INPUT0",
385 "shape": [len(TRITON_INPUT_DATA)],
386 "datatype": "FP32",
387 "data": list(TRITON_INPUT_DATA),
388 }
389 ]
390 }
391 return {
392 "max_tokens": settings.request_max_tokens,
393 "model": runtime.model_id,
394 "prompt": settings.request_prompt,
395 "stream": False,
396 "temperature": 0,
397 }
400def inference_framework_env(runtime: InferenceRuntimeSpec) -> dict[str, str]:
401 """Return the environment the deploy carries for the selected runtime.
403 ``MODEL`` is the GCO convention every renderer and ``gco inference invoke``
404 understand; both Hugging Face launchers receive the model itself on argv
405 (see :func:`inference_deploy_extra_args`). Triton serves whatever the
406 synced model repository holds and takes no environment.
407 """
408 if runtime.framework == "triton":
409 return {}
410 return {"MODEL": runtime.model_id}
413def inference_health_path(settings: Any, runtime: InferenceRuntimeSpec) -> str:
414 """The readiness path the deploy probes and ``gco inference health`` reads."""
415 return _HEALTH_PATHS.get(runtime.framework, str(settings.health_path))
418def inference_model_repository_path(endpoint_name: str) -> str:
419 """Where the init container lands ``--model-source`` for ``endpoint_name``."""
420 return f"{TRITON_MODEL_REPOSITORY_MOUNT}/{endpoint_name}"
423def inference_deploy_extra_args(
424 runtime: InferenceRuntimeSpec, *, endpoint_name: str = "<endpoint>"
425) -> tuple[str, ...]:
426 """Return the official launcher arguments for the selected runtime.
428 The Hugging Face runtimes receive their immutable model on argv. Triton's
429 image has no default command, so the deploy passes ``tritonserver`` with
430 the model repository the init container synced for ``endpoint_name``
431 (the placeholder stands in when only the shape is being recorded).
432 """
433 if runtime.framework == "sglang":
434 return (
435 "--model-path",
436 runtime.model_id,
437 "--revision",
438 runtime.model_revision,
439 )
440 if runtime.framework == "triton":
441 return (
442 "tritonserver",
443 f"--model-repository={inference_model_repository_path(endpoint_name)}",
444 )
445 return (
446 "--model",
447 runtime.model_id,
448 "--revision",
449 runtime.model_revision,
450 "--logprobs-mode",
451 STRUCTURED_DECISION_LOGPROBS_MODE,
452 )
455def inference_structured_decision_body(runtime: InferenceRuntimeSpec) -> dict[str, Any]:
456 """The ``/v1/systemone`` request the vLLM baseline scenario sends after its completion."""
457 return {
458 "model": runtime.model_id,
459 "state": STRUCTURED_DECISION_STATE,
460 "questions": copy.deepcopy(STRUCTURED_DECISION_QUESTIONS),
461 }
464def inference_mooncake_block(
465 role: EndpointRole, *, reconfigured: bool = False
466) -> dict[str, Any] | None:
467 """The ``spec.mooncake`` block the store scenario persists, or ``None`` for others.
469 ``reconfigured`` adds the ``local_buffer_size`` the scenario writes with
470 ``configure-store`` after its first invocation.
471 """
472 if role != "store":
473 return None
474 store: dict[str, Any] = {"enabled": True, "cold_tier_enabled": True}
475 if reconfigured:
476 store["local_buffer_size"] = str(MOONCAKE_STORE_RECONFIGURED_LOCAL_BUFFER_BYTES)
477 return {"mode": "store", "store": store, "transfer": {"protocol": "tcp"}}
480def _runtime_identity(settings: Any, runtime: InferenceRuntimeSpec) -> dict[str, Any]:
481 return {
482 "framework": runtime.framework,
483 "image": runtime.image,
484 "model": {"id": runtime.model_id, "revision": runtime.model_revision},
485 "roles": list(runtime.roles),
486 "server": {
487 "port": runtime.port,
488 "health_path": inference_health_path(settings, runtime),
489 "model_source": "model-registry" if runtime.uses_model_registry else "launcher",
490 },
491 "request_contract": {
492 "path": runtime.request_path,
493 "body": inference_request_body(settings, runtime),
494 "response": _RESPONSE_CONTRACTS[runtime.framework],
495 },
496 "probe_contract": {
497 "health_path": inference_health_path(settings, runtime),
498 "model_info_path": runtime.model_info_path,
499 "expected_model_id": runtime.model_id,
500 "expected_model_revision": runtime.model_revision,
501 },
502 "deploy_contract": {
503 "framework_env": inference_framework_env(runtime),
504 "extra_args": list(inference_deploy_extra_args(runtime)),
505 },
506 "structured_decision_contract": (
507 {
508 "path": STRUCTURED_DECISION_PATH,
509 "roles": list(runtime.structured_decision_roles),
510 "body": inference_structured_decision_body(runtime),
511 "response": _STRUCTURED_DECISION_RESPONSE_CONTRACT,
512 }
513 if runtime.structured_decision_roles
514 else None
515 ),
516 }
519def inference_identity_fields(settings: Any) -> dict[str, Any]:
520 """Return every inference input that must match on resume."""
521 return {
522 "contract_version": INFERENCE_CONTRACT_VERSION,
523 "selected_region": settings.selected_region,
524 "runtimes": [
525 _runtime_identity(settings, runtime) for runtime in settings.inference_runtimes
526 ],
527 "endpoint_contract": {
528 "count": settings.endpoint_count,
529 "namespace": settings.namespace,
530 "gpu_count": settings.gpu_count,
531 "accelerator": "nvidia",
532 "rewrite_image": False,
533 "roles_per_runtime": {
534 runtime.framework: list(runtime.roles) for runtime in settings.inference_runtimes
535 },
536 "mooncake_store": {
537 "mode": "store",
538 "protocol": "tcp",
539 "cold_tier": True,
540 "reconfigured_local_buffer_size": MOONCAKE_STORE_RECONFIGURED_LOCAL_BUFFER_BYTES,
541 },
542 "baseline_replicas": settings.baseline_replicas,
543 "autoscale_initial_replicas": settings.autoscale_initial_replicas,
544 "hpa": {
545 "metric": "cpu",
546 "target": settings.hpa_cpu_target,
547 "min_replicas": settings.hpa_min_replicas,
548 "max_replicas": settings.hpa_max_replicas,
549 },
550 },
551 "shared_proxy_contract": {
552 "namespace": "gco-system",
553 "deployment": "inference-proxy",
554 "hpa": "inference-proxy-hpa",
555 "tls_container": "api-tls-proxy",
556 "tls_cpu_request": settings.proxy_tls_cpu_request,
557 "tls_cpu_target": settings.proxy_tls_cpu_target,
558 "metric_type": "ContainerResource",
559 },
560 "timeouts": {
561 "command_seconds": settings.command_timeout_seconds,
562 "readiness_seconds": settings.readiness_timeout_seconds,
563 "hpa_seconds": settings.hpa_timeout_seconds,
564 "deletion_seconds": settings.deletion_timeout_seconds,
565 "poll_seconds": settings.poll_interval_seconds,
566 "monitor_interval_seconds": settings.monitor_interval_seconds,
567 "hpa_stability_intervals": settings.hpa_stability_intervals,
568 "job_seconds": settings.job_timeout_seconds,
569 "queue_seconds": settings.queue_timeout_seconds,
570 "destroy_attempts": settings.destroy_attempts,
571 "destroy_retry_delay_seconds": settings.destroy_retry_delay_seconds,
572 },
573 "consent": settings.consent,
574 }