Coverage for cli / commands / inference_cmd.py: 100.00%
882 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-10-28 16:19 +0000
1"""Inference endpoint commands."""
3import codecs
4import json
5import sys
6from email.message import Message
7from typing import Any
9import click
11from ..config import GCOConfig
12from ..output import (
13 confirm,
14 emit_structured_document,
15 get_output_formatter,
16 interactive_echo,
17)
19pass_config = click.make_pass_decorator(GCOConfig, ensure=True)
22@click.group()
23@pass_config
24def inference(config: Any) -> None:
25 """Manage multi-region inference endpoints."""
28@inference.command("deploy")
29@click.argument("endpoint_name")
30@click.option(
31 "--image",
32 "-i",
33 default=None,
34 help="Container image (e.g. vllm/vllm-openai:v0.32.0). Optional with "
35 "--mooncake-mode: falls back to the default upstream Mooncake-enabled vLLM image.",
36)
37@click.option(
38 "--framework",
39 type=click.Choice(["vllm", "sglang"]),
40 default=None,
41 help="Explicit serving runtime contract; persisted for renderer/probe behavior. "
42 "'sglang' renders the official launcher (python3 -m sglang.launch_server) "
43 "listening on --port, with --model-path taken from -e MODEL=... unless passed "
44 "via --extra-args, and needs an NVIDIA GPU of compute capability 8.0+ "
45 "(A10G/L4/L40S/A100/H100): its pods are kept off T4 (g4dn) and older GPUs.",
46)
47@click.option(
48 "--region",
49 "-r",
50 multiple=True,
51 help="Target region(s). Repeatable. Default: all deployed regions",
52)
53@click.option("--replicas", default=1, help="Replicas per region (default: 1)")
54@click.option("--gpu-count", default=1, help="GPUs per replica (default: 1)")
55@click.option(
56 "--gpu-type",
57 help="Preferred instance type (e.g. g5.xlarge). A hint, not a requirement: "
58 "the pods prefer it, and Karpenter launches another type when it can't get this one.",
59)
60@click.option(
61 "--port",
62 default=8000,
63 help="Container port (default: 8000; SGLang's documented default is 30000)",
64)
65@click.option(
66 "--model-path",
67 help="Mount the inference namespace's EFS volume at /models without syncing "
68 "anything (--model-source mounts it too); point -e MODEL at a path under it.",
69)
70@click.option(
71 "--model-source",
72 help="S3 URI for model weights (e.g. $(gco models uri llama3)), synced to "
73 "/models/<endpoint name> in each region by an init container.",
74)
75@click.option("--health-path", default="/health", help="Health check path (default: /health)")
76@click.option("--env", "-e", multiple=True, help="Environment variable (KEY=VALUE). Repeatable")
77@click.option("--namespace", "-n", default="gco-inference", help="Kubernetes namespace")
78@click.option("--label", "-l", multiple=True, help="Label (key=value). Repeatable")
79@click.option("--min-replicas", type=int, default=None, help="Autoscaling: minimum replicas")
80@click.option("--max-replicas", type=int, default=None, help="Autoscaling: maximum replicas")
81@click.option(
82 "--autoscale-metric",
83 multiple=True,
84 help="Autoscaling metric (cpu:70, memory:80, gpu:60). Repeatable. Enables "
85 "autoscaling. CPU/memory scale via the native HPA; gpu (and gpu_memory) "
86 "scale on CloudWatch GPU utilization via KEDA.",
87)
88@click.option(
89 "--capacity-type",
90 type=click.Choice(["on-demand", "spot"]),
91 default=None,
92 help="Node capacity type. 'spot' uses cheaper preemptible instances.",
93)
94@click.option(
95 "--extra-args",
96 multiple=True,
97 help="Extra arguments passed to the container (e.g. '--kv-transfer-config {...}'). Repeatable.",
98)
99@click.option(
100 "--accelerator",
101 type=click.Choice(["nvidia", "neuron"]),
102 default="nvidia",
103 help="Accelerator type: 'nvidia' for GPU instances (default), 'neuron' for Trainium/Inferentia.",
104)
105@click.option(
106 "--node-selector",
107 multiple=True,
108 help="Node selector (key=value). Repeatable. E.g. --node-selector eks.amazonaws.com/instance-family=inf2",
109)
110@click.option(
111 "--no-rewrite-image",
112 is_flag=True,
113 default=False,
114 help="Skip the per-region ECR URI rewrite. The image URI is sent verbatim "
115 "to every target region (operator owns cross-region pulls).",
116)
117@click.option(
118 "--mooncake-mode",
119 type=click.Choice(["disaggregated", "store", "both"]),
120 default=None,
121 help="Enable Mooncake serving: 'disaggregated' splits prefill/decode, "
122 "'store' runs a shared KV-cache store, 'both' composes the two. When set "
123 "and -i is omitted, the default upstream Mooncake-enabled vLLM image is used.",
124)
125@click.option(
126 "--prefill-replicas",
127 type=int,
128 default=1,
129 help="Prefill instance count (X in an XpYd topology) for split modes.",
130)
131@click.option(
132 "--decode-replicas",
133 type=int,
134 default=1,
135 help="Decode instance count (Y in an XpYd topology) for split modes.",
136)
137@click.option(
138 "--mooncake-protocol",
139 type=click.Choice(["rdma", "tcp"]),
140 default=None,
141 help="Mooncake transfer intent. 'rdma' (the default) schedules role pods "
142 "on EFA and configures vLLM's connector protocol as 'efa'; 'tcp' is the "
143 "non-EFA fallback. Requires --mooncake-mode.",
144)
145@click.option(
146 "--mooncake-device-name",
147 default=None,
148 help="Network device passed to Mooncake (for example efa_0 or eth0). "
149 "Omit or pass an empty value for auto-detection. Requires --mooncake-mode.",
150)
151@click.option(
152 "--mooncake-autoscale",
153 multiple=True,
154 help="Per-role Mooncake autoscaling as ROLE:MIN:MAX[:METRIC:TARGET ...], "
155 "e.g. 'prefill:1:8' or 'decode:2:16:cpu:70:gpu:60'. Repeatable (one per "
156 "role); append additional METRIC:TARGET pairs to scale a role on multiple "
157 "metrics (cpu/memory via HPA, gpu/gpu_memory via KEDA CloudWatch). Requires "
158 "--mooncake-mode disaggregated|both; populates spec.mooncake.autoscaling "
159 "(distinct from the legacy --autoscale-metric/--min-replicas flags).",
160)
161@click.option(
162 "--mooncake-cold-tier",
163 is_flag=True,
164 default=False,
165 help="Enable the asynchronous per-region S3 cold tier for the shared "
166 "KV-cache store (the cold tier extends the store). Pre-warm it with "
167 "'gco inference populate-kv'. Requires --mooncake-mode store or both.",
168)
169@click.option(
170 "--mooncake-proxy-image",
171 default=None,
172 help="Container image for the prefill-decode proxy (disaggregated/both). "
173 "Defaults to the endpoint image, which bundles the reference proxy.",
174)
175@click.option(
176 "--mooncake-admin-key-secret",
177 default=None,
178 help="Name of an existing Kubernetes Secret holding the prefill-decode "
179 "proxy ADMIN_API_KEY. Optional: when omitted, each region's monitor "
180 "auto-provisions a {name}-admin Secret with a generated key.",
181)
182@pass_config
183def inference_deploy(
184 config: Any,
185 endpoint_name: Any,
186 image: Any,
187 framework: Any,
188 region: Any,
189 replicas: Any,
190 gpu_count: Any,
191 gpu_type: Any,
192 port: Any,
193 model_path: Any,
194 model_source: Any,
195 health_path: Any,
196 env: Any,
197 namespace: Any,
198 label: Any,
199 min_replicas: Any,
200 max_replicas: Any,
201 autoscale_metric: Any,
202 capacity_type: Any,
203 extra_args: Any,
204 accelerator: Any,
205 node_selector: Any,
206 no_rewrite_image: Any,
207 mooncake_mode: Any,
208 prefill_replicas: Any,
209 decode_replicas: Any,
210 mooncake_protocol: Any,
211 mooncake_device_name: Any,
212 mooncake_autoscale: Any,
213 mooncake_cold_tier: Any,
214 mooncake_proxy_image: Any,
215 mooncake_admin_key_secret: Any,
216) -> None:
217 """Deploy an inference endpoint to one or more regions.
219 The endpoint is registered in DynamoDB and the inference_monitor
220 in each target region creates the Kubernetes resources automatically.
222 Examples:
223 gco inference deploy my-llm -i vllm/vllm-openai:v0.32.0
225 gco inference deploy qwen \\
226 -i vllm/vllm-openai:v0.32.0 \\
227 -e MODEL=Qwen/Qwen2.5-1.5B-Instruct \\
228 --extra-args=--max-model-len --extra-args 4096
230 gco inference deploy llama3-70b \\
231 -i vllm/vllm-openai:v0.32.0 \\
232 -r us-east-1 -r eu-west-1 \\
233 --replicas 2 --gpu-count 4 \\
234 --model-source $(gco models uri llama3-70b) \\
235 -e MODEL=/models/llama3-70b
237 vLLM is started with --model $(MODEL) unless --extra-args already name
238 the model; each --extra-args value is one server argument.
239 """
240 from ..inference import get_inference_manager
242 formatter = get_output_formatter(config)
244 # Parse env vars and labels
245 env_dict = {}
246 for e_var in env:
247 if "=" in e_var:
248 k, v = e_var.split("=", 1)
249 env_dict[k] = v
251 labels_dict = {}
252 for lbl in label:
253 if "=" in lbl:
254 k, v = lbl.split("=", 1)
255 labels_dict[k] = v
257 node_selector_dict = {}
258 for ns in node_selector:
259 if "=" in ns:
260 k, v = ns.split("=", 1)
261 node_selector_dict[k] = v
263 # Build autoscaling config
264 autoscaling_config = None
265 if autoscale_metric:
266 metrics = []
267 for m in autoscale_metric:
268 if ":" in m:
269 mtype, mtarget = m.split(":", 1)
270 metrics.append({"type": mtype, "target": int(mtarget)})
271 else:
272 metrics.append({"type": m, "target": 70})
273 autoscaling_config = {
274 "enabled": True,
275 "min_replicas": min_replicas or 1,
276 "max_replicas": max_replicas or 10,
277 "metrics": metrics,
278 }
280 # Transfer overrides are meaningful only when a Mooncake block is being
281 # authored. With no override, the monitor resolves the default RDMA intent
282 # to vLLM's explicit EFA connector protocol and auto-detects the device.
283 if (mooncake_protocol is not None or mooncake_device_name is not None) and not mooncake_mode:
284 formatter.print_error(
285 "--mooncake-protocol and --mooncake-device-name require --mooncake-mode."
286 )
287 sys.exit(1)
289 mooncake_transfer_config: dict[str, Any] | None = None
290 if mooncake_protocol is not None or mooncake_device_name is not None:
291 mooncake_transfer_config = {}
292 if mooncake_protocol is not None:
293 mooncake_transfer_config["protocol"] = mooncake_protocol
294 if mooncake_device_name is not None:
295 mooncake_transfer_config["device_name"] = mooncake_device_name
297 # Build per-role Mooncake autoscaling config (spec.mooncake.autoscaling).
298 # This is distinct from the legacy single-Deployment autoscaling above:
299 # each ROLE:MIN:MAX token sets a role's bounds, and any number of trailing
300 # METRIC:TARGET pairs add scaling signals for that role. Bounds and metrics
301 # are validated fail-fast in the deploy path before anything is persisted.
302 mooncake_autoscaling_config: dict[str, Any] | None = None
303 if mooncake_autoscale:
304 if not mooncake_mode:
305 formatter.print_error(
306 "--mooncake-autoscale requires --mooncake-mode (disaggregated or both)."
307 )
308 sys.exit(1)
309 mooncake_autoscaling_config = {"enabled": True}
310 for entry in mooncake_autoscale:
311 parts = entry.split(":")
312 # ROLE:MIN:MAX, then zero or more METRIC:TARGET pairs.
313 if len(parts) < 3 or (len(parts) - 3) % 2 != 0:
314 formatter.print_error(
315 f"Invalid --mooncake-autoscale value '{entry}'. Expected "
316 "ROLE:MIN:MAX optionally followed by METRIC:TARGET pairs."
317 )
318 sys.exit(1)
319 role = parts[0]
320 if role not in ("prefill", "decode"):
321 formatter.print_error(
322 f"Invalid --mooncake-autoscale role '{role}'. Expected 'prefill' or 'decode'."
323 )
324 sys.exit(1)
325 try:
326 role_block: dict[str, Any] = {
327 "min_replicas": int(parts[1]),
328 "max_replicas": int(parts[2]),
329 }
330 metric_tokens = parts[3:]
331 metrics = [
332 {"type": metric_tokens[i], "target": int(metric_tokens[i + 1])}
333 for i in range(0, len(metric_tokens), 2)
334 ]
335 if metrics:
336 role_block["metrics"] = metrics
337 except ValueError:
338 formatter.print_error(
339 f"Invalid --mooncake-autoscale numbers in '{entry}'. MIN, MAX, "
340 "and each TARGET must be integers."
341 )
342 sys.exit(1)
343 mooncake_autoscaling_config[role] = role_block
345 # --mooncake-cold-tier opts into the async per-region S3 cold tier, which
346 # extends the shared store, so it only applies to store/both modes.
347 if mooncake_cold_tier and mooncake_mode not in ("store", "both"):
348 formatter.print_error(
349 "--mooncake-cold-tier requires --mooncake-mode store or both "
350 "(the cold tier extends the shared KV-cache store)."
351 )
352 sys.exit(1)
354 mooncake_store_config: dict[str, Any] | None = None
355 if mooncake_cold_tier:
356 mooncake_store_config = {"enabled": True, "cold_tier_enabled": True}
358 # Configure the prefill-decode proxy that fronts split modes: an explicit
359 # image (otherwise it defaults to the endpoint image) and the name of the
360 # Kubernetes Secret holding its ADMIN_API_KEY.
361 mooncake_proxy_config: dict[str, Any] | None = None
362 if mooncake_proxy_image or mooncake_admin_key_secret:
363 mooncake_proxy_config = {}
364 if mooncake_proxy_image:
365 mooncake_proxy_config["image"] = mooncake_proxy_image
366 if mooncake_admin_key_secret:
367 mooncake_proxy_config["admin_api_key_secret"] = mooncake_admin_key_secret
369 # When no admin-key Secret is named, each region's monitor auto-provisions a
370 # {name}-admin Secret with a generated key, so no manual step is needed.
371 if mooncake_mode in ("disaggregated", "both") and not mooncake_admin_key_secret:
372 formatter.print_info(
373 "No --mooncake-admin-key-secret given; each region's inference "
374 "monitor will auto-provision a '{name}-admin' Secret with a "
375 "generated ADMIN_API_KEY. Pass --mooncake-admin-key-secret to use "
376 "your own Secret instead."
377 )
379 try:
380 manager = get_inference_manager(config)
381 result = manager.deploy(
382 endpoint_name=endpoint_name,
383 image=image,
384 framework=framework,
385 target_regions=list(region) if region else None,
386 replicas=replicas,
387 gpu_count=gpu_count,
388 gpu_type=gpu_type,
389 port=port,
390 model_path=model_path,
391 model_source=model_source,
392 health_check_path=health_path,
393 env=env_dict if env_dict else None,
394 namespace=namespace,
395 labels=labels_dict if labels_dict else None,
396 autoscaling=autoscaling_config,
397 capacity_type=capacity_type,
398 extra_args=list(extra_args) if extra_args else None,
399 accelerator=accelerator,
400 node_selector=node_selector_dict if node_selector_dict else None,
401 rewrite_image=not no_rewrite_image,
402 mooncake_mode=mooncake_mode,
403 prefill_replicas=prefill_replicas,
404 decode_replicas=decode_replicas,
405 mooncake_store=mooncake_store_config,
406 mooncake_transfer=mooncake_transfer_config,
407 mooncake_proxy=mooncake_proxy_config,
408 mooncake_autoscaling=mooncake_autoscaling_config,
409 )
411 formatter.print_success(f"Endpoint '{endpoint_name}' registered for deployment")
412 regions_str = ", ".join(result.get("target_regions", []))
413 formatter.print_info(f"Target regions: {regions_str}")
414 formatter.print_info(f"Ingress path: {result.get('ingress_path', '')}")
415 formatter.print_info(
416 "The inference_monitor in each region will create the resources. "
417 "Use 'gco inference status' to track progress."
418 )
420 # Warn if deploying to a subset of regions. The endpoint is already
421 # registered by now, so a failed lookup here is only a missing
422 # warning: reporting "Failed to deploy endpoint" and exiting 1 would
423 # misstate a deploy that succeeded.
424 if region:
425 from ..aws_client import get_aws_client as _get_client
427 try:
428 all_stacks = _get_client(config).discover_regional_stacks()
429 except Exception as lookup_error:
430 formatter.print_warning(
431 "Could not list the other deployed Regions to compare with -r "
432 f"({lookup_error}); the endpoint is registered for: {regions_str}."
433 )
434 else:
435 all_regions = set(all_stacks.keys())
436 target_set = set(result.get("target_regions", []))
437 missing = all_regions - target_set
438 if missing:
439 formatter.print_warning(
440 f"Endpoint is NOT deployed to: {', '.join(sorted(missing))}. "
441 "Global Accelerator may route users to those regions where "
442 "the endpoint won't exist. Consider deploying to all regions "
443 "(omit -r) for consistent global routing."
444 )
446 if config.output_format != "table":
447 formatter.print(result)
449 except ValueError as e:
450 formatter.print_error(str(e))
451 sys.exit(1)
452 except Exception as e:
453 formatter.print_error(f"Failed to deploy endpoint: {e}")
454 sys.exit(1)
457@inference.command("list")
458@click.option("--state", "-s", help="Filter by state (deploying, running, stopped, deleted)")
459@click.option("--region", "-r", help="Filter by target region")
460@pass_config
461def inference_list(config: Any, state: Any, region: Any) -> None:
462 """List inference endpoints.
464 Examples:
465 gco inference list
466 gco inference list --state running
467 gco inference list -r us-east-1
468 """
469 from ..inference import get_inference_manager
471 formatter = get_output_formatter(config)
473 try:
474 manager = get_inference_manager(config)
475 endpoints = manager.list_endpoints(desired_state=state, region=region)
477 if config.output_format != "table":
478 formatter.print(endpoints)
479 return
481 if not endpoints:
482 formatter.print_info("No inference endpoints found")
483 return
485 print(f"\n Inference Endpoints ({len(endpoints)} found)")
486 print(" " + "-" * 85)
487 print(f" {'NAME':<25} {'STATE':<12} {'REGIONS':<25} {'REPLICAS':>8} {'IMAGE'}")
488 print(" " + "-" * 85)
489 for ep in endpoints:
490 name = ep.get("endpoint_name", "")[:24]
491 ep_state = ep.get("desired_state", "unknown")
492 regions = ", ".join(ep.get("target_regions", []))[:24]
493 spec = ep.get("spec", {})
494 replicas = spec.get("replicas", 1) if isinstance(spec, dict) else 1
495 image = spec.get("image", "")[:40] if isinstance(spec, dict) else ""
496 print(f" {name:<25} {ep_state:<12} {regions:<25} {replicas:>8} {image}")
498 print()
500 except Exception as e:
501 formatter.print_error(f"Failed to list endpoints: {e}")
502 sys.exit(1)
505@inference.command("status")
506@click.argument("endpoint_name")
507@pass_config
508def inference_status(config: Any, endpoint_name: Any) -> None:
509 """Show detailed status of an inference endpoint.
511 Examples:
512 gco inference status my-llm
513 """
514 from ..inference import get_inference_manager
516 formatter = get_output_formatter(config)
518 try:
519 manager = get_inference_manager(config)
520 endpoint = manager.get_endpoint(endpoint_name)
522 if not endpoint:
523 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
524 sys.exit(1)
526 if config.output_format != "table":
527 formatter.print(endpoint)
528 return
530 spec = endpoint.get("spec", {})
531 print(f"\n Endpoint: {endpoint_name}")
532 print(" " + "-" * 60)
533 print(f" State: {endpoint.get('desired_state', 'unknown')}")
534 print(f" Image: {spec.get('image', 'N/A')}")
535 print(f" Replicas: {spec.get('replicas', 1)}")
536 print(f" GPUs: {spec.get('gpu_count', 0)}")
537 print(f" Port: {spec.get('port', 8000)}")
538 print(f" Path: {endpoint.get('ingress_path', 'N/A')}")
539 print(f" Namespace: {endpoint.get('namespace', 'N/A')}")
540 print(f" Created: {endpoint.get('created_at', 'N/A')}")
542 # Region status
543 region_status = endpoint.get("region_status", {})
544 if region_status:
545 print("\n Region Status:")
546 print(f" {'REGION':<18} {'STATE':<12} {'READY':>5} {'DESIRED':>7} {'LAST SYNC'}")
547 print(" " + "-" * 65)
548 for r, status in region_status.items():
549 if isinstance(status, dict):
550 r_state = status.get("state", "unknown")
551 ready = status.get("replicas_ready", 0)
552 desired = status.get("replicas_desired", 0)
553 last_sync = status.get("last_sync", "N/A")
554 if last_sync and len(last_sync) > 19:
555 last_sync = last_sync[:19]
556 print(f" {r:<18} {r_state:<12} {ready:>5} {desired:>7} {last_sync}")
557 else:
558 target_regions = endpoint.get("target_regions", [])
559 print(f"\n Target regions: {', '.join(target_regions)}")
560 print(" (Waiting for inference_monitor to sync)")
562 print()
564 except Exception as e:
565 formatter.print_error(f"Failed to get endpoint status: {e}")
566 sys.exit(1)
569@inference.command("scale")
570@click.argument("endpoint_name")
571@click.option("--replicas", "-r", required=True, type=int, help="New replica count")
572@pass_config
573def inference_scale(config: Any, endpoint_name: Any, replicas: Any) -> None:
574 """Scale an inference endpoint.
576 Examples:
577 gco inference scale my-llm --replicas 4
578 """
579 from ..inference import get_inference_manager
581 formatter = get_output_formatter(config)
583 try:
584 manager = get_inference_manager(config)
585 result = manager.scale(endpoint_name, replicas)
587 if result:
588 formatter.print_success(f"Endpoint '{endpoint_name}' scaled to {replicas} replicas")
589 else:
590 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
591 sys.exit(1)
593 except Exception as e:
594 formatter.print_error(f"Failed to scale endpoint: {e}")
595 sys.exit(1)
598@inference.command("stop")
599@click.argument("endpoint_name")
600@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
601@pass_config
602def inference_stop(config: Any, endpoint_name: Any, yes: Any) -> None:
603 """Stop an inference endpoint (scale to zero, keep config).
605 Examples:
606 gco inference stop my-llm -y
607 """
608 from ..inference import get_inference_manager
610 formatter = get_output_formatter(config)
612 if not yes:
613 confirm(f"Stop endpoint '{endpoint_name}'?", abort=True)
615 try:
616 manager = get_inference_manager(config)
617 result = manager.stop(endpoint_name)
619 if result:
620 formatter.print_success(f"Endpoint '{endpoint_name}' marked for stop")
621 else:
622 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
623 sys.exit(1)
625 except Exception as e:
626 formatter.print_error(f"Failed to stop endpoint: {e}")
627 sys.exit(1)
630@inference.command("start")
631@click.argument("endpoint_name")
632@pass_config
633def inference_start(config: Any, endpoint_name: Any) -> None:
634 """Start a stopped inference endpoint.
636 Examples:
637 gco inference start my-llm
638 """
639 from ..inference import get_inference_manager
641 formatter = get_output_formatter(config)
643 try:
644 manager = get_inference_manager(config)
645 result = manager.start(endpoint_name)
647 if result:
648 formatter.print_success(f"Endpoint '{endpoint_name}' marked for start")
649 else:
650 formatter.print_error(
651 f"Endpoint '{endpoint_name}' is not a stopped endpoint. If it was deleted "
652 "or purged, redeploy it with 'gco inference deploy'."
653 )
654 sys.exit(1)
656 except Exception as e:
657 formatter.print_error(f"Failed to start endpoint: {e}")
658 sys.exit(1)
661def _warn_missing_regions(
662 formatter: Any,
663 endpoint_name: str,
664 target_regions: list[str],
665 deployed_regions: set[str],
666) -> None:
667 """Warn when deployed Regions are left out of an endpoint's Region list.
669 Global Accelerator routes each request to the closest healthy Region, so a
670 request for this endpoint can land in a Region that does not serve it and
671 get a 404 there.
672 """
673 missing = sorted(deployed_regions - set(target_regions))
674 if missing:
675 formatter.print_warning(
676 f"Endpoint '{endpoint_name}' is NOT deployed to: {', '.join(missing)}. "
677 "Global Accelerator may route users to those regions where the endpoint "
678 "won't exist. Add them with 'gco inference add-region'."
679 )
682@inference.command("add-region")
683@click.argument("endpoint_name")
684@click.argument("region")
685@pass_config
686def inference_add_region(config: Any, endpoint_name: Any, region: Any) -> None:
687 """Add a Region to an existing inference endpoint.
689 An endpoint keeps the Region list it was deployed with, so a Region added to
690 the deployment later does not run it. This command adds REGION to that list;
691 the inference monitor in REGION then creates the endpoint from its stored
692 spec on its next pass. REGION must already run a GCO regional stack.
694 Examples:
695 gco inference add-region my-llm us-west-2
696 """
697 from ..aws_client import get_aws_client
698 from ..inference import get_inference_manager
700 formatter = get_output_formatter(config)
702 try:
703 manager = get_inference_manager(config)
704 endpoint = manager.get_endpoint(endpoint_name)
705 if endpoint is None:
706 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
707 sys.exit(1)
708 current = list(endpoint.get("target_regions") or [])
709 if region in current:
710 formatter.print_info(
711 f"Endpoint '{endpoint_name}' already targets {region}: {', '.join(current)}"
712 )
713 return
714 deployed = set(get_aws_client(config).discover_regional_stacks())
715 if region not in deployed:
716 formatter.print_error(
717 f"No GCO regional stack is deployed in {region}. Deploy it first with "
718 f"'gco stacks regions add {region} -y' and 'gco stacks deploy-all -y'."
719 )
720 sys.exit(1)
721 result = manager.add_region(endpoint_name, region)
722 if result is None:
723 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
724 sys.exit(1)
725 targets = list(result.get("target_regions") or [])
726 formatter.print_success(f"Endpoint '{endpoint_name}' now targets: {', '.join(targets)}")
727 formatter.print_info(
728 f"The inference monitor in {region} creates the endpoint's resources. "
729 f"Use 'gco inference status {endpoint_name}' to track progress."
730 )
731 _warn_missing_regions(formatter, endpoint_name, targets, deployed)
732 if config.output_format != "table":
733 formatter.print(result)
734 except ValueError as e:
735 formatter.print_error(str(e))
736 sys.exit(1)
737 except Exception as e:
738 formatter.print_error(f"Failed to add {region} to endpoint '{endpoint_name}': {e}")
739 sys.exit(1)
742@inference.command("remove-region")
743@click.argument("endpoint_name")
744@click.argument("region")
745@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
746@pass_config
747def inference_remove_region(config: Any, endpoint_name: Any, region: Any, yes: Any) -> None:
748 """Remove a Region from an inference endpoint.
750 The inference monitor in REGION deletes the endpoint's resources there, and
751 the endpoint keeps serving from its other Regions. Removing the endpoint's
752 last Region is refused; use 'gco inference stop' or 'gco inference delete'
753 instead.
755 Examples:
756 gco inference remove-region my-llm us-west-2 -y
757 """
758 from ..aws_client import get_aws_client
759 from ..inference import get_inference_manager
761 formatter = get_output_formatter(config)
763 try:
764 manager = get_inference_manager(config)
765 endpoint = manager.get_endpoint(endpoint_name)
766 if endpoint is None:
767 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
768 sys.exit(1)
769 current = list(endpoint.get("target_regions") or [])
770 if region not in current:
771 formatter.print_info(
772 f"Endpoint '{endpoint_name}' does not target {region}: {', '.join(current)}"
773 )
774 return
775 if current == [region]:
776 formatter.print_error(
777 f"{region} is the only Region endpoint '{endpoint_name}' runs in. Use "
778 f"'gco inference stop {endpoint_name}' or 'gco inference delete "
779 f"{endpoint_name}' instead."
780 )
781 sys.exit(1)
782 if not yes:
783 confirm(
784 f"Remove {region} from endpoint '{endpoint_name}'? Its resources in "
785 f"{region} are deleted.",
786 abort=True,
787 )
788 result = manager.remove_region(endpoint_name, region)
789 if result is None:
790 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
791 sys.exit(1)
792 targets = list(result.get("target_regions") or [])
793 formatter.print_success(f"Endpoint '{endpoint_name}' now targets: {', '.join(targets)}")
794 formatter.print_info(
795 f"The inference monitor in {region} deletes the endpoint's resources there."
796 )
797 try:
798 deployed = set(get_aws_client(config).discover_regional_stacks())
799 except Exception as lookup_error:
800 formatter.print_warning(
801 f"Could not list the deployed Regions ({lookup_error}); requests that "
802 f"Global Accelerator sends to {region} for this endpoint get a 404."
803 )
804 else:
805 _warn_missing_regions(formatter, endpoint_name, targets, deployed)
806 if config.output_format != "table":
807 formatter.print(result)
808 except click.Abort:
809 raise
810 except ValueError as e:
811 formatter.print_error(str(e))
812 sys.exit(1)
813 except Exception as e:
814 formatter.print_error(f"Failed to remove {region} from endpoint '{endpoint_name}': {e}")
815 sys.exit(1)
818@inference.command("delete")
819@click.argument("endpoint_name")
820@click.option(
821 "--expected-owner-label",
822 default=None,
823 metavar="KEY=VALUE",
824 hidden=True,
825 help="Require an exact stored owner label in the same atomic delete-state update",
826)
827@click.option(
828 "--expected-lifecycle-id",
829 default=None,
830 hidden=True,
831 help="Require the immutable endpoint lifecycle observed by an automation client",
832)
833@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
834@pass_config
835def inference_delete(
836 config: Any,
837 endpoint_name: Any,
838 expected_owner_label: Any,
839 expected_lifecycle_id: Any,
840 yes: Any,
841) -> None:
842 """Delete an inference endpoint from all regions.
844 The inference_monitor in each region will clean up the K8s resources.
846 Examples:
847 gco inference delete my-llm -y
848 """
849 from ..inference import get_inference_manager
851 formatter = get_output_formatter(config)
853 owner_condition: tuple[str, str] | None = None
854 if expected_owner_label is not None:
855 label_name, separator, label_value = str(expected_owner_label).partition("=")
856 if not separator or not label_name or not label_value:
857 formatter.print_error("--expected-owner-label must be KEY=VALUE")
858 sys.exit(1)
859 owner_condition = (label_name, label_value)
860 lifecycle_condition = (
861 str(expected_lifecycle_id).strip() if expected_lifecycle_id is not None else None
862 )
863 if lifecycle_condition == "":
864 formatter.print_error("--expected-lifecycle-id must be non-empty")
865 sys.exit(1)
866 if (owner_condition is None) != (lifecycle_condition is None):
867 formatter.print_error(
868 "--expected-owner-label and --expected-lifecycle-id must be supplied together"
869 )
870 sys.exit(1)
872 if not yes:
873 confirm(f"Delete endpoint '{endpoint_name}' from all regions?", abort=True)
875 try:
876 manager = get_inference_manager(config)
877 result = manager.delete(
878 endpoint_name,
879 expected_owner_label=owner_condition,
880 expected_lifecycle_id=lifecycle_condition,
881 )
883 if result:
884 formatter.print_success(
885 f"Endpoint '{endpoint_name}' marked for deletion. "
886 "The inference_monitor will clean up resources in each region."
887 )
888 else:
889 suffix = " or ownership/lifecycle condition failed" if owner_condition else ""
890 formatter.print_error(f"Endpoint '{endpoint_name}' not found{suffix}")
891 sys.exit(1)
893 except Exception as e:
894 formatter.print_error(f"Failed to delete endpoint: {e}")
895 sys.exit(1)
898@inference.command("update-image")
899@click.argument("endpoint_name")
900@click.option("--image", "-i", required=True, help="New container image")
901@pass_config
902def inference_update_image(config: Any, endpoint_name: Any, image: Any) -> None:
903 """Update the container image for an inference endpoint.
905 Triggers a rolling update across all target regions.
907 Examples:
908 gco inference update-image my-llm -i vllm/vllm-openai:v0.32.0
909 """
910 from ..inference import get_inference_manager
912 formatter = get_output_formatter(config)
914 try:
915 manager = get_inference_manager(config)
916 result = manager.update_image(endpoint_name, image)
918 if result:
919 formatter.print_success(f"Endpoint '{endpoint_name}' image updated to {image}")
920 formatter.print_info("Rolling update will be applied by inference_monitor")
921 else:
922 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
923 sys.exit(1)
925 except Exception as e:
926 formatter.print_error(f"Failed to update image: {e}")
927 sys.exit(1)
930#: Why ``invoke`` can't build a Triton request from a prompt, and what to pass.
931_TRITON_INVOKE_HINT = (
932 "Triton serves models by name and takes input tensors, not a prompt: pass "
933 "--path /v2/models/<model>/infer and --data with a KServe v2 inference request, "
934 'e.g. --data \'{"inputs": [{"name": "INPUT0", "shape": [3], "datatype": "FP32", '
935 '"data": [1, 2, 3]}]}\'. `gco inference models <name>` lists the loaded models.'
936)
939def _resolve_invoke_framework(persisted_framework: Any, image_lower: str) -> str:
940 """Return the serving contract ``invoke``/``models`` should speak to an endpoint.
942 The persisted ``framework`` wins; image heuristics only cover records
943 created before the field existed. Anything unrecognised is treated as an
944 OpenAI-compatible server.
945 """
946 if persisted_framework in ("vllm", "sglang"):
947 return str(persisted_framework)
948 if "vllm" in image_lower:
949 return "vllm"
950 if "sglang" in image_lower:
951 return "sglang"
952 if "tritonserver" in image_lower or "triton" in image_lower:
953 return "triton"
954 return "openai"
957def _flag_value(args: list[str], flag: str) -> str | None:
958 """The value of ``flag`` in ``args``, given as ``flag value`` or ``flag=value``."""
959 for index, arg in enumerate(args):
960 if arg == flag and index + 1 < len(args):
961 return args[index + 1]
962 if arg.startswith(f"{flag}="):
963 return arg.split("=", 1)[1]
964 return None
967def _openai_request_model(spec: dict[str, Any], *, vllm: bool) -> str | None:
968 """The ``model`` an OpenAI-compatible request to this endpoint must name.
970 Mirrors how the renderer starts vLLM: ``--served-model-name`` is the name
971 the API answers to; otherwise the model the arguments name (``--model``,
972 or the positional tag ``vllm serve`` takes first) wins over the ``MODEL``
973 env convention. ``None`` means the spec doesn't say, including a model
974 named inside a ``--config`` file, so the caller asks ``/v1/models``.
975 """
976 args = [str(arg) for arg in spec.get("args") or []]
977 for flag in ("--served-model-name", "--model"):
978 value = _flag_value(args, flag)
979 if value:
980 return value
981 if vllm and args and not args[0].startswith("-"):
982 return args[0]
983 if vllm and any(arg == "--config" or arg.startswith("--config=") for arg in args):
984 return None
985 model = (spec.get("env") or {}).get("MODEL")
986 return str(model) if model else None
989def _model_named_only_by_env(spec: Any, model: str) -> bool:
990 """True when ``model`` comes from the spec's ``MODEL`` alone, not its args."""
991 if not isinstance(spec, dict) or (spec.get("env") or {}).get("MODEL") != model:
992 return False
993 args = [str(arg) for arg in spec.get("args") or []]
994 if args and not args[0].startswith("-"):
995 return False
996 return all(_flag_value(args, flag) is None for flag in ("--served-model-name", "--model"))
999def _served_model_hint(
1000 client: Any,
1001 *,
1002 endpoint_name: str,
1003 endpoint_path: str,
1004 requested_model: str,
1005 region: str | None,
1006 spec: Any,
1007) -> str | None:
1008 """Explain a 404 for a model the endpoint does not serve, or return None.
1010 Asks the endpoint's ``/v1/models`` which models it serves. The hint is
1011 given only when that inventory answers and leaves out the requested model,
1012 so a 404 with any other cause keeps the server's own message alone.
1013 """
1014 try:
1015 response = client.make_authenticated_request(
1016 method="GET",
1017 path=f"{endpoint_path}/v1/models",
1018 target_region=region,
1019 )
1020 if not response.ok:
1021 return None
1022 data = response.json().get("data", [])
1023 served = [str(item["id"]) for item in data if isinstance(item, dict) and item.get("id")]
1024 except Exception: # nosec B110 # the hint is best-effort
1025 return None
1026 if not served or requested_model in served:
1027 return None
1029 region_flag = f" -r {region}" if region else ""
1030 example = json.dumps(
1031 {"model": served[0], "prompt": "Hello", "max_tokens": 50}, separators=(", ", ": ")
1032 )
1033 lines = [f"{endpoint_name} serves {', '.join(served)}, not {requested_model}."]
1034 if _model_named_only_by_env(spec, requested_model):
1035 lines.append(
1036 "vLLM does not read MODEL. Endpoints deployed with -e MODEL before v8.12.0 "
1037 "were started without it and serve vLLM's default model, and an upgrade does "
1038 f"not change a running endpoint. Recreate it to serve {requested_model}; see "
1039 "'Invoke Says the Model Does Not Exist' in docs/TROUBLESHOOTING.md."
1040 )
1041 lines.append(
1042 f"To call the served model now: gco inference invoke {endpoint_name}{region_flag} "
1043 f"-d '{example}'"
1044 )
1045 return "\n".join(lines)
1048_SGLANG_SERVER_INFO_IDENTITY_KEYS = (
1049 "model_path",
1050 "served_model_name",
1051 "revision",
1052 "tokenizer_path",
1053 "version",
1054)
1057@inference.command("invoke")
1058@click.argument("endpoint_name")
1059@click.option("--prompt", "-p", help="Text prompt to send")
1060@click.option("--data", "-d", help="Raw JSON body to send")
1061@click.option(
1062 "--path", "api_path", default=None, help="API sub-path (default: auto-detect from framework)"
1063)
1064@click.option("--region", "-r", help="Target region for the request")
1065@click.option(
1066 "--max-tokens", type=int, default=100, help="Maximum tokens to generate (default: 100)"
1067)
1068@click.option(
1069 "--stream/--no-stream",
1070 default=None,
1071 help="Enable or disable incremental response streaming. Raw JSON with "
1072 "'stream': true enables streaming automatically.",
1073)
1074@pass_config
1075def inference_invoke(
1076 config: Any,
1077 endpoint_name: Any,
1078 prompt: Any,
1079 data: Any,
1080 api_path: Any,
1081 region: Any,
1082 max_tokens: Any,
1083 stream: Any,
1084) -> None:
1085 """Send a request to an inference endpoint and print the response.
1087 Automatically discovers the endpoint's stored API path (the legacy
1088 ``ingress_path`` record field) and routes the request through API Gateway
1089 with SigV4 authentication.
1091 Examples:
1092 gco inference invoke my-llm -p "What is GPU orchestration?"
1094 gco inference invoke my-llm -d '{"prompt": "Hello", "max_tokens": 50}'
1096 gco inference invoke my-llm -p "Explain K8s" --path /v1/completions
1098 gco inference invoke d1-decisions --path /v1/systemone \\
1099 -d '{"state": "Shipped 9 days late", "questions": {"late": {"type": "noul"}}}'
1100 """
1101 import json as _json
1103 from ..aws_client import get_aws_client
1104 from ..inference import get_inference_manager
1106 formatter = get_output_formatter(config)
1108 if not prompt and not data:
1109 formatter.print_error("Provide --prompt (-p) or --data (-d)")
1110 sys.exit(1)
1112 try:
1113 # Look up the endpoint's stored API prefix and serving spec. The record
1114 # retains the historical ``ingress_path`` field name for compatibility;
1115 # requests still traverse only the shared authenticated Ingress.
1116 manager = get_inference_manager(config)
1117 endpoint = manager.get_endpoint(endpoint_name)
1118 if not endpoint:
1119 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
1120 sys.exit(1)
1122 endpoint_path = endpoint.get("ingress_path", f"/inference/{endpoint_name}")
1123 spec = endpoint.get("spec", {})
1124 image = spec.get("image", "") if isinstance(spec, dict) else ""
1125 image_lower = image.lower() if isinstance(image, str) else ""
1126 persisted_framework = spec.get("framework") if isinstance(spec, dict) else None
1128 parsed_data: dict[str, Any] | None = None
1129 if data:
1130 parsed_json = _json.loads(data)
1131 if not isinstance(parsed_json, dict):
1132 raise ValueError("--data must contain a JSON object")
1133 parsed_data = parsed_json
1135 # An explicit flag wins over the body. Without a flag, raw OpenAI JSON
1136 # can opt into streamed transport by carrying its normal stream field.
1137 if stream is None:
1138 stream_response = parsed_data is not None and parsed_data.get("stream") is True
1139 else:
1140 stream_response = bool(stream)
1141 if parsed_data is not None and stream is not None:
1142 parsed_data["stream"] = stream_response
1144 # Persisted framework is authoritative for neutral/private image names;
1145 # image heuristics remain only for legacy records created before that
1146 # field existed.
1147 resolved_framework = _resolve_invoke_framework(persisted_framework, image_lower)
1148 if resolved_framework == "triton" and (api_path is None or parsed_data is None):
1149 # Triton has no prompt API: an inference request names a model and
1150 # carries its input tensors, so neither can be guessed here.
1151 formatter.print_error(_TRITON_INVOKE_HINT)
1152 sys.exit(1)
1153 if api_path is None:
1154 # Native SGLang API; streaming is a body flag on the same path.
1155 api_path = "/generate" if resolved_framework == "sglang" else "/v1/completions"
1157 full_path = f"{endpoint_path}{api_path}"
1159 # Build the request body.
1160 body: dict[str, Any]
1161 if parsed_data is not None:
1162 body = parsed_data
1163 else:
1164 assert prompt is not None
1165 if "generate" in api_path:
1166 # SGLang native format: the prompt travels as ``text`` and the
1167 # decoding controls under ``sampling_params``.
1168 body = {
1169 "text": prompt,
1170 "sampling_params": {"max_new_tokens": max_tokens},
1171 "stream": stream_response,
1172 }
1173 elif "/v2/" in api_path:
1174 # A KServe v2 (Triton) path takes tensors; a prompt can't
1175 # become an inference request.
1176 formatter.print_error(_TRITON_INVOKE_HINT)
1177 sys.exit(1)
1178 else:
1179 # OpenAI-compatible (vLLM, etc.)
1180 # Determine model name for OpenAI-compatible request
1181 model_name = endpoint_name
1182 if isinstance(spec, dict):
1183 is_vllm = persisted_framework == "vllm" or "vllm" in image_lower
1184 model_name = _openai_request_model(spec, vllm=is_vllm) or endpoint_name
1185 # Default for vLLM with no explicit model — auto-detect
1186 # by querying /v1/models on the running endpoint
1187 if model_name == endpoint_name and is_vllm:
1188 try:
1189 detect_client = get_aws_client(config)
1190 models_path = f"/inference/{endpoint_name}/v1/models"
1191 models_resp = detect_client.make_authenticated_request(
1192 method="GET",
1193 path=models_path,
1194 target_region=region,
1195 )
1196 if models_resp.ok:
1197 models_data = models_resp.json().get("data", [])
1198 if models_data:
1199 model_name = models_data[0]["id"]
1200 except Exception: # nosec B110 # discovery is best-effort
1201 pass # Fall through to endpoint_name as model
1202 body = {"model": model_name, "max_tokens": max_tokens, "stream": stream_response}
1203 if api_path.rstrip("/").endswith("/chat/completions"):
1204 # The chat API takes messages, not a prompt: the prompt
1205 # becomes a single user turn.
1206 body["messages"] = [{"role": "user", "content": prompt}]
1207 else:
1208 body["prompt"] = prompt
1210 # The OpenAI-compatible model name this request asks for, if any. A
1211 # 404 for it is checked against what the endpoint actually serves.
1212 requested_model = body.get("model") if resolved_framework in ("vllm", "openai") else None
1214 def _report_http_error() -> None:
1215 formatter.print_error(f"HTTP {response.status_code}: {response.text[:500]}")
1216 if response.status_code == 404 and isinstance(requested_model, str):
1217 hint = _served_model_hint(
1218 client,
1219 endpoint_name=endpoint_name,
1220 endpoint_path=endpoint_path,
1221 requested_model=requested_model,
1222 region=region,
1223 spec=spec,
1224 )
1225 if hint:
1226 formatter.print_info(hint)
1227 sys.exit(1)
1229 if stream_response:
1230 # Keep streamed stdout byte-for-byte pipeline-friendly; request
1231 # metadata belongs on stderr when the response itself is streamed.
1232 print(f"ℹ POST {full_path}", file=sys.stderr)
1233 else:
1234 formatter.print_info(f"POST {full_path}")
1236 # Make the authenticated request. ``stream=True`` prevents requests
1237 # from preloading the body so chunks can reach stdout as they arrive.
1238 client = get_aws_client(config)
1239 response = client.make_authenticated_request(
1240 method="POST",
1241 path=full_path,
1242 body=body,
1243 target_region=region,
1244 stream=stream_response,
1245 )
1247 if stream_response:
1248 try:
1249 if not response.ok:
1250 _report_http_error()
1252 # Requests assumes ISO-8859-1 for text/* without a declared
1253 # charset. Model token streams are UTF-8 in practice, so honor
1254 # only an explicit response charset and otherwise use UTF-8.
1255 content_type = response.headers.get("content-type", "")
1256 encoding = "utf-8"
1257 if isinstance(content_type, str):
1258 parsed_content_type = Message()
1259 parsed_content_type["content-type"] = content_type
1260 declared_charset = parsed_content_type.get_content_charset()
1261 if declared_charset is not None:
1262 try:
1263 codecs.lookup(declared_charset)
1264 except LookupError:
1265 pass
1266 else:
1267 encoding = declared_charset
1269 decoder = codecs.getincrementaldecoder(encoding)(errors="replace")
1270 for chunk in response.iter_content(chunk_size=8192, decode_unicode=False):
1271 if not chunk:
1272 continue
1273 output = chunk if isinstance(chunk, str) else decoder.decode(chunk)
1274 if output:
1275 sys.stdout.write(output)
1276 sys.stdout.flush()
1277 remainder = decoder.decode(b"", final=True)
1278 if remainder:
1279 sys.stdout.write(remainder)
1280 sys.stdout.flush()
1281 finally:
1282 response.close()
1283 return
1285 # Buffered responses retain the friendly extraction used by the CLI.
1286 if response.ok:
1287 try:
1288 resp_json = response.json()
1289 # Extract the generated text for common formats
1290 text = None
1291 if "choices" in resp_json:
1292 # OpenAI format
1293 choices = resp_json["choices"]
1294 if choices:
1295 text = choices[0].get("text") or choices[0].get("message", {}).get(
1296 "content"
1297 )
1298 elif isinstance(resp_json, dict) and isinstance(resp_json.get("text"), str):
1299 # SGLang native /generate format
1300 text = resp_json["text"]
1302 if text and config.output_format == "table":
1303 print(f"\n{text.strip()}\n")
1304 else:
1305 emit_structured_document(
1306 resp_json,
1307 output_format="json",
1308 rendered=_json.dumps(resp_json, indent=2),
1309 )
1310 except _json.JSONDecodeError:
1311 print(response.text)
1312 else:
1313 _report_http_error()
1315 except Exception as e:
1316 formatter.print_error(f"Failed to invoke endpoint: {e}")
1317 sys.exit(1)
1320@inference.command("canary")
1321@click.argument("endpoint_name")
1322@click.option("--image", "-i", required=True, help="New container image for canary")
1323@click.option(
1324 "--weight",
1325 "-w",
1326 default=10,
1327 type=int,
1328 help="Percentage of traffic to canary (1-99, default: 10)",
1329)
1330@click.option(
1331 "--replicas", "-r", default=1, type=int, help="Number of canary replicas (default: 1)"
1332)
1333@pass_config
1334def inference_canary(
1335 config: Any, endpoint_name: Any, image: Any, weight: Any, replicas: Any
1336) -> None:
1337 """Start a canary deployment with a new image.
1339 Routes a percentage of traffic to the canary while the primary
1340 continues serving the rest. Use 'promote' to make the canary
1341 the new primary, or 'rollback' to remove it.
1343 Examples:
1344 gco inference canary my-llm -i vllm/vllm-openai:v0.32.0 --weight 10
1345 gco inference canary my-llm -i new-image:latest -w 25 -r 2
1346 """
1347 from ..inference import get_inference_manager
1349 formatter = get_output_formatter(config)
1351 try:
1352 manager = get_inference_manager(config)
1353 result = manager.canary_deploy(endpoint_name, image, weight=weight, replicas=replicas)
1355 if not result:
1356 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
1357 sys.exit(1)
1359 formatter.print_success(
1360 f"Canary started: {weight}% traffic → {image} ({replicas} replica(s))"
1361 )
1362 formatter.print_info(f"Monitor with: gco inference status {endpoint_name}")
1363 formatter.print_info(f"Promote with: gco inference promote {endpoint_name}")
1364 formatter.print_info(f"Rollback with: gco inference rollback {endpoint_name}")
1366 except ValueError as e:
1367 formatter.print_error(str(e))
1368 sys.exit(1)
1369 except Exception as e:
1370 formatter.print_error(f"Failed to start canary: {e}")
1371 sys.exit(1)
1374@inference.command("promote")
1375@click.argument("endpoint_name")
1376@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1377@pass_config
1378def inference_promote(config: Any, endpoint_name: Any, yes: Any) -> None:
1379 """Promote the canary to primary.
1381 Replaces the primary image with the canary image and removes
1382 the canary deployment. All traffic goes to the new image.
1384 Examples:
1385 gco inference promote my-llm -y
1386 """
1387 from ..inference import get_inference_manager
1389 formatter = get_output_formatter(config)
1391 try:
1392 manager = get_inference_manager(config)
1393 endpoint = manager.get_endpoint(endpoint_name)
1395 if not endpoint:
1396 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
1397 sys.exit(1)
1399 canary = endpoint.get("spec", {}).get("canary")
1400 if not canary:
1401 formatter.print_error(f"Endpoint '{endpoint_name}' has no active canary")
1402 sys.exit(1)
1404 if not yes:
1405 current_image = endpoint.get("spec", {}).get("image", "unknown")
1406 interactive_echo(f" Current primary: {current_image}")
1407 interactive_echo(f" Canary image: {canary.get('image', 'unknown')}")
1408 interactive_echo(f" Canary weight: {canary.get('weight', 0)}%")
1409 if not confirm(" Promote canary to primary?"):
1410 formatter.print_info("Cancelled")
1411 return
1413 result = manager.promote_canary(endpoint_name)
1414 if result:
1415 new_image = result.get("spec", {}).get("image", "unknown")
1416 formatter.print_success(f"Promoted: all traffic now serving {new_image}")
1417 else:
1418 formatter.print_error("Promotion failed")
1419 sys.exit(1)
1421 except ValueError as e:
1422 formatter.print_error(str(e))
1423 sys.exit(1)
1424 except Exception as e:
1425 formatter.print_error(f"Failed to promote canary: {e}")
1426 sys.exit(1)
1429@inference.command("rollback")
1430@click.argument("endpoint_name")
1431@click.option("--yes", "-y", is_flag=True, help="Skip confirmation")
1432@pass_config
1433def inference_rollback(config: Any, endpoint_name: Any, yes: Any) -> None:
1434 """Remove the canary deployment, keeping the primary unchanged.
1436 All traffic returns to the primary deployment.
1438 Examples:
1439 gco inference rollback my-llm -y
1440 """
1441 from ..inference import get_inference_manager
1443 formatter = get_output_formatter(config)
1445 try:
1446 manager = get_inference_manager(config)
1447 endpoint = manager.get_endpoint(endpoint_name)
1449 if not endpoint:
1450 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
1451 sys.exit(1)
1453 canary = endpoint.get("spec", {}).get("canary")
1454 if not canary:
1455 formatter.print_error(f"Endpoint '{endpoint_name}' has no active canary")
1456 sys.exit(1)
1458 if not yes:
1459 interactive_echo(f" Canary image: {canary.get('image', 'unknown')}")
1460 interactive_echo(f" Canary weight: {canary.get('weight', 0)}%")
1461 if not confirm(" Remove canary and restore full traffic to primary?"):
1462 formatter.print_info("Cancelled")
1463 return
1465 result = manager.rollback_canary(endpoint_name)
1466 if result:
1467 primary_image = result.get("spec", {}).get("image", "unknown")
1468 formatter.print_success(f"Rolled back: all traffic now serving {primary_image}")
1469 else:
1470 formatter.print_error("Rollback failed")
1471 sys.exit(1)
1473 except ValueError as e:
1474 formatter.print_error(str(e))
1475 sys.exit(1)
1476 except Exception as e:
1477 formatter.print_error(f"Failed to rollback canary: {e}")
1478 sys.exit(1)
1481@inference.command("health")
1482@click.argument("endpoint_name")
1483@click.option("--region", "-r", help="Target region to check")
1484@pass_config
1485def inference_health(config: Any, endpoint_name: Any, region: Any) -> None:
1486 """Check if an inference endpoint is healthy and ready to serve.
1488 Hits the endpoint's health check path and reports status and latency.
1490 Examples:
1491 gco inference health my-llm
1493 gco inference health my-llm -r us-east-1
1494 """
1495 import time as _time
1497 from ..aws_client import get_aws_client
1498 from ..inference import get_inference_manager
1500 formatter = get_output_formatter(config)
1502 try:
1503 manager = get_inference_manager(config)
1504 endpoint = manager.get_endpoint(endpoint_name)
1505 if not endpoint:
1506 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
1507 sys.exit(1)
1509 endpoint_path = endpoint.get("ingress_path", f"/inference/{endpoint_name}")
1510 spec = endpoint.get("spec", {})
1511 health_path = (
1512 spec.get("health_check_path", "/health") if isinstance(spec, dict) else "/health"
1513 )
1514 full_path = f"{endpoint_path}{health_path}"
1516 client = get_aws_client(config)
1517 start = _time.monotonic()
1518 response = client.make_authenticated_request(
1519 method="GET",
1520 path=full_path,
1521 target_region=region,
1522 )
1523 latency_ms = (_time.monotonic() - start) * 1000
1525 result = {
1526 "endpoint": endpoint_name,
1527 "status": "healthy" if response.ok else "unhealthy",
1528 "http_status": response.status_code,
1529 "latency_ms": round(latency_ms, 1),
1530 "path": full_path,
1531 }
1533 try:
1534 result["body"] = response.json()
1535 except Exception:
1536 result["body"] = response.text[:200] if response.text else None
1538 if config.output_format == "table":
1539 status_icon = "✓" if response.ok else "✗"
1540 formatter.print_info(
1541 f"{status_icon} {endpoint_name}: {result['status']} "
1542 f"(HTTP {response.status_code}, {result['latency_ms']}ms)"
1543 )
1544 else:
1545 formatter.print(result)
1547 except Exception as e:
1548 formatter.print_error(f"Health check failed: {e}")
1549 sys.exit(1)
1552@inference.command("models")
1553@click.argument("endpoint_name")
1554@click.option(
1555 "--framework",
1556 type=click.Choice(["vllm", "sglang"]),
1557 default=None,
1558 help="Runtime metadata contract (default: persisted endpoint framework or vLLM).",
1559)
1560@click.option("--region", "-r", help="Target region to query")
1561@pass_config
1562def inference_models(
1563 config: Any,
1564 endpoint_name: Any,
1565 framework: Any,
1566 region: Any,
1567) -> None:
1568 """Read exact model identity from vLLM /v1/models or SGLang /server_info.
1570 vLLM's OpenAI inventory is printed as returned, and so is a Triton
1571 endpoint's /v2/models/stats, which names every loaded model and version
1572 (Triton has no model list of its own). For SGLang the command
1573 reads ``/server_info`` (the launcher arguments the running server
1574 resolved) and prints only the identity fields — ``model_path``,
1575 ``served_model_name``, ``revision``, ``tokenizer_path`` and ``version`` —
1576 so the exact model *and* the exact revision the process was started with
1577 are visible without the multi-kilobyte scheduler state that endpoint also
1578 carries.
1579 """
1580 import json as _json
1582 from ..aws_client import get_aws_client
1583 from ..inference import get_inference_manager
1585 formatter = get_output_formatter(config)
1587 try:
1588 manager = get_inference_manager(config)
1589 endpoint = manager.get_endpoint(endpoint_name)
1590 if not endpoint:
1591 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
1592 sys.exit(1)
1594 ingress_path = endpoint.get("ingress_path", f"/inference/{endpoint_name}")
1595 spec = endpoint.get("spec")
1596 persisted_framework = spec.get("framework") if isinstance(spec, dict) else None
1597 image = spec.get("image") if isinstance(spec, dict) else None
1598 image_lower = image.lower() if isinstance(image, str) else ""
1599 if persisted_framework is not None and persisted_framework not in ("vllm", "sglang"):
1600 raise ValueError("endpoint has an unsupported persisted inference framework")
1601 selected_framework = framework or _resolve_invoke_framework(
1602 persisted_framework, image_lower
1603 )
1604 if selected_framework not in ("vllm", "sglang", "triton"):
1605 # Unknown OpenAI-compatible images publish no model identity
1606 # document this command knows how to read; the OpenAI inventory is
1607 # the closest thing to one.
1608 selected_framework = "vllm"
1609 # Triton has no model list; its statistics extension reports every
1610 # loaded model by name and version, which is the inventory we need.
1611 model_path = {
1612 "vllm": "v1/models",
1613 "sglang": "server_info",
1614 "triton": "v2/models/stats",
1615 }[selected_framework]
1616 full_path = f"{ingress_path}/{model_path}"
1618 client = get_aws_client(config)
1619 response = client.make_authenticated_request(
1620 method="GET",
1621 path=full_path,
1622 target_region=region,
1623 )
1625 if response.ok:
1626 try:
1627 resp_json = response.json()
1628 if selected_framework == "sglang":
1629 if not isinstance(resp_json, dict):
1630 raise ValueError("SGLang /server_info did not return a JSON object")
1631 resp_json = {
1632 key: resp_json[key]
1633 for key in _SGLANG_SERVER_INFO_IDENTITY_KEYS
1634 if key in resp_json
1635 }
1636 emit_structured_document(
1637 resp_json,
1638 output_format="json",
1639 rendered=_json.dumps(resp_json, indent=2),
1640 )
1641 except _json.JSONDecodeError:
1642 print(response.text)
1643 else:
1644 formatter.print_error(f"HTTP {response.status_code}: {response.text[:500]}")
1645 sys.exit(1)
1647 except Exception as e:
1648 formatter.print_error(f"Failed to list models: {e}")
1649 sys.exit(1)
1652@inference.command("set-topology")
1653@click.argument("endpoint_name")
1654@click.option(
1655 "--prefill",
1656 required=True,
1657 type=int,
1658 help="Prefill (X) instance count for the XpYd topology.",
1659)
1660@click.option(
1661 "--decode",
1662 required=True,
1663 type=int,
1664 help="Decode (Y) instance count for the XpYd topology.",
1665)
1666@pass_config
1667def inference_set_topology(config: Any, endpoint_name: Any, prefill: Any, decode: Any) -> None:
1668 """Resize a disaggregated endpoint's prefill/decode topology.
1670 Updates the endpoint's prefill (X) and decode (Y) instance counts and
1671 re-triggers reconciliation so each region's monitor adjusts the role
1672 replica counts. Both counts must be integers in the range 1..1000.
1674 Examples:
1675 gco inference set-topology llama-pd --prefill 3 --decode 2
1676 """
1677 from ..inference import get_inference_manager
1679 formatter = get_output_formatter(config)
1681 try:
1682 manager = get_inference_manager(config)
1683 result = manager.set_topology(endpoint_name, prefill, decode)
1685 if result:
1686 formatter.print_success(
1687 f"Endpoint '{endpoint_name}' topology set to {prefill}p{decode}d"
1688 )
1689 formatter.print_info(
1690 "The inference_monitor will adjust prefill and decode "
1691 "replica counts in each region."
1692 )
1693 if config.output_format != "table":
1694 formatter.print(result)
1695 else:
1696 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
1697 sys.exit(1)
1699 except ValueError as e:
1700 formatter.print_error(str(e))
1701 sys.exit(1)
1702 except Exception as e:
1703 formatter.print_error(f"Failed to set topology: {e}")
1704 sys.exit(1)
1707@inference.command("configure-store")
1708@click.argument("endpoint_name")
1709@click.option(
1710 "--cold-tier/--no-cold-tier",
1711 "cold_tier",
1712 default=None,
1713 help="Opt the endpoint into (or out of) the asynchronous S3 cold tier. "
1714 "Enabling it also enables the shared store it extends.",
1715)
1716@click.option(
1717 "--offload",
1718 type=click.Choice(["cpu", "disk", "none"]),
1719 default=None,
1720 help="KV-store offload tier for spilling cache beyond GPU memory.",
1721)
1722@click.option(
1723 "--global-segment-size",
1724 type=int,
1725 default=None,
1726 help="Global segment size in bytes for the KV-cache store.",
1727)
1728@click.option(
1729 "--local-buffer-size",
1730 type=int,
1731 default=None,
1732 help="Local buffer size in bytes for the KV-cache store.",
1733)
1734@click.option(
1735 "--enable-store/--disable-store",
1736 "enabled",
1737 default=None,
1738 help="Enable or disable the shared KV-cache store.",
1739)
1740@pass_config
1741def inference_configure_store(
1742 config: Any,
1743 endpoint_name: Any,
1744 cold_tier: Any,
1745 offload: Any,
1746 global_segment_size: Any,
1747 local_buffer_size: Any,
1748 enabled: Any,
1749) -> None:
1750 """Update the shared KV-cache store on a Mooncake endpoint.
1752 Merges the given settings into the endpoint's existing KV-cache store
1753 configuration and re-triggers reconciliation so each region's monitor picks
1754 up the change. Enabling the cold tier also enables the shared store it
1755 extends. Use 'gco inference populate-kv' to pre-warm the cold tier.
1757 Examples:
1758 gco inference configure-store my-llm --cold-tier
1759 gco inference configure-store my-llm --offload cpu --local-buffer-size 2147483648
1760 """
1761 from ..inference import get_inference_manager
1763 formatter = get_output_formatter(config)
1765 try:
1766 manager = get_inference_manager(config)
1767 endpoint = manager.get_endpoint(endpoint_name)
1768 if not endpoint:
1769 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
1770 sys.exit(1)
1772 # Merge onto the endpoint's current store block so changing one field
1773 # does not drop the others.
1774 spec = endpoint.get("spec", {}) if isinstance(endpoint, dict) else {}
1775 mooncake = spec.get("mooncake", {}) if isinstance(spec, dict) else {}
1776 store_config = dict(mooncake.get("store") or {})
1778 if enabled is not None:
1779 store_config["enabled"] = enabled
1780 if cold_tier is not None:
1781 store_config["cold_tier_enabled"] = cold_tier
1782 if cold_tier:
1783 # The cold tier extends the shared store, so enabling it enables
1784 # the store too.
1785 store_config["enabled"] = True
1786 if offload is not None:
1787 store_config["offload"] = offload
1788 if global_segment_size is not None:
1789 store_config["global_segment_size"] = global_segment_size
1790 if local_buffer_size is not None:
1791 store_config["local_buffer_size"] = local_buffer_size
1793 if not store_config:
1794 formatter.print_error(
1795 "No store settings given. Pass --cold-tier, --offload, "
1796 "--global-segment-size, --local-buffer-size, or --enable-store."
1797 )
1798 sys.exit(1)
1800 result = manager.configure_store(endpoint_name, store_config)
1801 if result:
1802 formatter.print_success(f"Endpoint '{endpoint_name}' store configuration updated")
1803 formatter.print_info(
1804 "The inference_monitor will re-render the KV-cache store "
1805 "configuration in each region and roll the role pods onto it."
1806 )
1807 if config.output_format != "table":
1808 formatter.print(result)
1809 else:
1810 formatter.print_error(f"Endpoint '{endpoint_name}' not found")
1811 sys.exit(1)
1813 except ValueError as e:
1814 formatter.print_error(str(e))
1815 sys.exit(1)
1816 except Exception as e:
1817 formatter.print_error(f"Failed to configure store: {e}")
1818 sys.exit(1)
1821@inference.command("populate-kv")
1822@click.argument("endpoint_name")
1823@click.argument("local_path")
1824@click.option(
1825 "--region",
1826 "-r",
1827 required=True,
1828 help="Region whose general-purpose bucket backs the endpoint's KV-cache cold tier.",
1829)
1830@pass_config
1831def inference_populate_kv(config: Any, endpoint_name: Any, local_path: Any, region: Any) -> None:
1832 """Upload data into an endpoint's Mooncake KV-cache cold tier.
1834 Uploads a local file or directory to the region's general-purpose bucket
1835 under the cold-tier key prefix the endpoint reads from
1836 (mooncake-kv/<endpoint>/). The endpoint must be deployed with the cold tier
1837 enabled (deploy with --mooncake-cold-tier, or run
1838 'gco inference configure-store <name> --cold-tier') for its pods to read the
1839 uploaded data.
1841 Examples:
1842 gco inference populate-kv my-llm ./kv-warm-set/ --region us-east-1
1843 """
1844 from ..models import get_regional_bucket_manager
1846 formatter = get_output_formatter(config)
1848 try:
1849 manager = get_regional_bucket_manager(config)
1850 formatter.print_info(
1851 f"Uploading {local_path} into the KV-cache cold tier for "
1852 f"'{endpoint_name}' in '{region}'..."
1853 )
1854 result = manager.populate_kv_cache(local_path, region, endpoint_name)
1856 formatter.print_success(
1857 f"Uploaded {result['files_uploaded']} file(s) to {result['s3_uri']}"
1858 )
1859 formatter.print_info(
1860 "Pods for this endpoint read the cold tier when it is enabled "
1861 "(deploy with --mooncake-cold-tier or 'gco inference configure-store')."
1862 )
1864 if config.output_format != "table":
1865 formatter.print(result)
1867 except Exception as e:
1868 formatter.print_error(f"Failed to populate KV cache: {e}")
1869 sys.exit(1)