Coverage for cli / ephemeral_bastion.py: 100.00%
260 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-10-28 16:19 +0000
1"""Ephemeral SSM bastion lifecycle for reaching a private EKS API endpoint.
3``gco monitoring open --via-ssm <instance-id>`` tunnels to a private EKS
4endpoint through an *existing* SSM-managed instance. This module lets the CLI
5**create that instance on demand** — a minimal, self-terminating ``t3.micro``
6(falling back through equivalent burstable types when a Region or AZ can't
7launch one) in the cluster VPC — and tear it down when the port-forward
8session ends, so an operator doesn't have to keep a standing bastion around
9just to view Grafana.
11Orphan safeguards (defence in depth — a crash, ``Ctrl-C``, or a forgotten
12teardown must never leave a paid instance running):
14* launched with ``--instance-initiated-shutdown-behavior terminate``;
15* user-data schedules ``shutdown -h +<ttl>`` as an unconditional backstop;
16* IMDSv2 required (``HttpTokens=required``);
17* tagged ``gco:ephemeral=true`` + ``gco:purpose`` so any orphan is greppable;
18* the CLI tears it down in a ``finally:`` block.
20Network posture: the instance reuses the cluster's own security group (which is
21self-referencing, so it can reach the private API endpoint) and is placed in one
22of the cluster's private subnets **without a public IP**. GCO private subnets
23have NAT egress, which lets the SSM agent reach Systems Manager over HTTPS while
24keeping the instance unreachable from the internet. No inbound ports are opened;
25SSM is agent-initiated outbound only.
27Style matches :mod:`cli.ssm_tunnel`: pure, validated argv builders (list form,
28never a shell string) that are fully unit-testable, plus thin runtime wrappers
29that shell out to the AWS CLI (the ``cli`` package does not depend on boto3).
30"""
32from __future__ import annotations
34import json
35import logging
36import re
37import subprocess
38import time
39from collections.abc import Iterator
40from contextlib import contextmanager
41from dataclasses import dataclass
43from ._image_uri import aws_partition, aws_url_suffix
45logger = logging.getLogger(__name__)
47# --------------------------------------------------------------------------
48# Constants — the fixed identity of the ephemeral bastion.
49# --------------------------------------------------------------------------
51# The bastion's IAM role, instance profile, and Name tag are scoped to the
52# deployment's project key (cdk.json context.project_name, default "gco" — the
53# same key cli/config.py derives cluster and stack names from), so a non-default
54# deployment addresses its own resources and two differently named deployments in
55# one account don't collide on a shared name. See bastion_role_name() below.
56DEFAULT_PROJECT_NAME = "gco"
57_ROLE_NAME_SUFFIX = "-ephemeral-bastion-role"
58_PROFILE_NAME_SUFFIX = "-ephemeral-bastion-profile"
59_INSTANCE_NAME_SUFFIX = "-ephemeral-ssm-bastion"
62# AmazonSSMManagedInstanceCore is AWS-managed and therefore not project-scoped,
63# but its ARN partition must follow the deployment Region.
64def ssm_managed_policy_arn(region: str) -> str:
65 return f"arn:{aws_partition(region)}:iam::aws:policy/AmazonSSMManagedInstanceCore"
68def bastion_trust_policy(region: str) -> dict[str, object]:
69 """Return an EC2 trust policy using the partition's service DNS suffix."""
70 return {
71 "Version": "2012-10-17",
72 "Statement": [
73 {
74 "Effect": "Allow",
75 "Principal": {"Service": f"ec2.{aws_url_suffix(region)}"},
76 "Action": "sts:AssumeRole",
77 }
78 ],
79 }
82# Back-compatible commercial defaults for callers that imported the constants.
83SSM_MANAGED_POLICY_ARN = ssm_managed_policy_arn("us-east-1")
85BASTION_INSTANCE_TYPE = "t3.micro"
87# Fallback chain when the preferred type cannot be launched in the target
88# subnet — a Region/AZ that lacks the type entirely, or transient
89# InsufficientInstanceCapacity. Ordered cheapest-first and all x86_64 (the
90# resolved AL2023 AMI is x86_64, so an arm64 type can never be substituted
91# without also changing the AMI). The bastion only forwards an SSM tunnel, so
92# any burstable nano/micro/small class is dimensionally identical for the job.
93BASTION_INSTANCE_TYPE_FALLBACKS: tuple[str, ...] = (
94 "t3a.micro",
95 "t3.small",
96 "t2.micro",
97)
99# EC2 error markers that mean "this instance type won't launch here" — the
100# cue to try the next type rather than fail the tunnel. Anything else
101# (auth, quota on vCPUs, malformed request) still raises immediately.
102_INSTANCE_TYPE_UNAVAILABLE_MARKERS = (
103 "InsufficientInstanceCapacity",
104 "Unsupported",
105 "InstanceTypeNotSupported",
106 "unsupported instance type",
107 "not supported in your requested Availability Zone",
108)
110# Public SSM parameter that always resolves to the latest Amazon Linux 2023
111# x86_64 AMI in the target region (t3.micro is x86_64).
112AL2023_AMI_SSM_PARAMETER = "/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64"
114# Tags stamped on every ephemeral instance so orphans are trivially discoverable:
115# aws ec2 describe-instances \
116# --filters Name=tag:gco:ephemeral,Values=true \
117# Name=instance-state-name,Values=running,pending
118TAG_EPHEMERAL_KEY = "gco:ephemeral"
119TAG_PURPOSE_KEY = "gco:purpose"
120TAG_PROJECT_KEY = "gco:project"
121TAG_TTL_KEY = "gco:ttl-minutes"
122BASTION_PURPOSE = "cluster-observability"
124DEFAULT_TTL_MINUTES = 120
125# The self-termination backstop a caller may ask for (five minutes to one day).
126MIN_TTL_MINUTES = 5
127MAX_TTL_MINUTES = 1440
129# EC2 instance-profile propagation to the RunInstances API is eventually
130# consistent; retry the launch for a short window on the "not found" error.
131_PROFILE_PROPAGATION_RETRIES = 6
132_PROFILE_PROPAGATION_WAIT_SECONDS = 5.0
134# EC2 trust policy default; builders derive the principal from their target Region.
135BASTION_TRUST_POLICY: dict[str, object] = bastion_trust_policy("us-east-1")
137# --------------------------------------------------------------------------
138# Validators — every AWS-supplied id is re-validated before it enters an argv.
139# --------------------------------------------------------------------------
141_REGION_RE = re.compile(r"^[a-z]{2,4}(?:-[a-z0-9]+)+-[0-9]+$")
142_CLUSTER_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9\-]{0,99}$")
143_INSTANCE_RE = re.compile(r"^i-[0-9a-f]{8}([0-9a-f]{9})?$")
144_VPC_RE = re.compile(r"^vpc-[0-9a-f]{8}([0-9a-f]{9})?$")
145_SUBNET_RE = re.compile(r"^subnet-[0-9a-f]{8}([0-9a-f]{9})?$")
146_SG_RE = re.compile(r"^sg-[0-9a-f]{8}([0-9a-f]{9})?$")
147_AMI_RE = re.compile(r"^ami-[0-9a-f]{8}([0-9a-f]{9})?$")
148_INSTANCE_TYPE_RE = re.compile(r"^[a-z0-9]+\.[a-z0-9]+$")
151def _validate(value: str, pattern: re.Pattern[str], what: str) -> str:
152 if not isinstance(value, str) or not pattern.match(value):
153 raise ValueError(f"Invalid {what}: {value!r}")
154 return value
157def _validate_ttl(ttl_minutes: int) -> int:
158 try:
159 value = int(ttl_minutes)
160 except (TypeError, ValueError) as exc:
161 raise ValueError(f"Invalid ttl-minutes {ttl_minutes!r}: must be an integer") from exc
162 if not MIN_TTL_MINUTES <= value <= MAX_TTL_MINUTES:
163 raise ValueError(
164 f"Invalid ttl-minutes {value}: must be between {MIN_TTL_MINUTES} and {MAX_TTL_MINUTES}"
165 )
166 return value
169# Project keys follow the cdk.json context.project_name shape (alnum + hyphen).
170_PROJECT_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9\-]{0,40}$")
173def _validate_project(project_name: str) -> str:
174 return _validate(project_name, _PROJECT_RE, "project name")
177def bastion_role_name(project_name: str = DEFAULT_PROJECT_NAME) -> str:
178 """IAM role name for ``project_name``'s ephemeral bastion."""
179 return f"{_validate_project(project_name)}{_ROLE_NAME_SUFFIX}"
182def bastion_profile_name(project_name: str = DEFAULT_PROJECT_NAME) -> str:
183 """Instance-profile name for ``project_name``'s ephemeral bastion."""
184 return f"{_validate_project(project_name)}{_PROFILE_NAME_SUFFIX}"
187def bastion_instance_name(project_name: str = DEFAULT_PROJECT_NAME) -> str:
188 """EC2 ``Name`` tag for ``project_name``'s ephemeral bastion."""
189 return f"{_validate_project(project_name)}{_INSTANCE_NAME_SUFFIX}"
192# Back-compat convenience constants for the default project. The helpers above
193# are the source of truth; these are the default-project values that the builder
194# defaults (and tests) reference.
195BASTION_ROLE_NAME = bastion_role_name()
196BASTION_PROFILE_NAME = bastion_profile_name()
197BASTION_NAME = bastion_instance_name()
200@dataclass(frozen=True)
201class BastionNetwork:
202 """Private VPC placement for an ephemeral bastion."""
204 vpc_id: str
205 subnet_id: str
206 security_group_id: str
209# --------------------------------------------------------------------------
210# Pure argv / payload builders (unit-tested; list form, no shell).
211# --------------------------------------------------------------------------
214def render_user_data(ttl_minutes: int = DEFAULT_TTL_MINUTES) -> str:
215 """Return the bastion boot script: an unconditional self-terminate backstop."""
216 ttl = _validate_ttl(ttl_minutes)
217 return (
218 "#!/bin/bash\n"
219 "# GCO ephemeral SSM bastion — self-terminate backstop.\n"
220 "# The instance is launched with --instance-initiated-shutdown-behavior\n"
221 "# terminate, so this scheduled halt terminates it even if no explicit\n"
222 "# teardown ever runs.\n"
223 f'shutdown -h +{ttl} "gco ephemeral bastion self-terminate backstop"\n'
224 )
227def build_get_ami_command(region: str) -> list[str]:
228 """``aws ssm get-parameter`` argv resolving the latest AL2023 x86_64 AMI."""
229 _validate(region, _REGION_RE, "region")
230 return [
231 "aws",
232 "ssm",
233 "get-parameter",
234 "--name",
235 AL2023_AMI_SSM_PARAMETER,
236 "--region",
237 region,
238 "--query",
239 "Parameter.Value",
240 "--output",
241 "text",
242 ]
245def build_describe_cluster_network_command(cluster: str, region: str) -> list[str]:
246 """``aws eks describe-cluster`` argv projecting VPC, cluster SG, and subnets."""
247 _validate(cluster, _CLUSTER_RE, "cluster name")
248 _validate(region, _REGION_RE, "region")
249 return [
250 "aws",
251 "eks",
252 "describe-cluster",
253 "--name",
254 cluster,
255 "--region",
256 region,
257 "--query",
258 ("cluster.resourcesVpcConfig.{vpc:vpcId,sg:clusterSecurityGroupId,subnets:subnetIds}"),
259 "--output",
260 "json",
261 ]
264def build_describe_private_cluster_subnet_command(subnet_ids: list[str], region: str) -> list[str]:
265 """Find the first private subnet among the EKS control-plane subnets."""
266 if not subnet_ids:
267 raise ValueError("At least one cluster subnet id is required")
268 validated_subnets = [
269 _validate(subnet_id, _SUBNET_RE, "cluster subnet id") for subnet_id in subnet_ids
270 ]
271 _validate(region, _REGION_RE, "region")
272 return [
273 "aws",
274 "ec2",
275 "describe-subnets",
276 "--region",
277 region,
278 "--subnet-ids",
279 *validated_subnets,
280 "--filters",
281 "Name=map-public-ip-on-launch,Values=false",
282 "--query",
283 "Subnets[0].SubnetId",
284 "--output",
285 "text",
286 ]
289def build_create_role_command(
290 role_name: str = BASTION_ROLE_NAME,
291 region: str = "us-east-1",
292) -> list[str]:
293 """``aws iam create-role`` argv with the partition-correct EC2 trust."""
294 _validate(role_name, _CLUSTER_RE, "role name")
295 _validate(region, _REGION_RE, "region")
296 return [
297 "aws",
298 "iam",
299 "create-role",
300 "--region",
301 region,
302 "--role-name",
303 role_name,
304 "--assume-role-policy-document",
305 json.dumps(bastion_trust_policy(region)),
306 "--description",
307 "GCO ephemeral SSM bastion (cluster-observability); safe to delete.",
308 "--tags",
309 f"Key={TAG_EPHEMERAL_KEY},Value=true",
310 f"Key={TAG_PURPOSE_KEY},Value={BASTION_PURPOSE}",
311 ]
314def build_attach_role_policy_command(
315 role_name: str = BASTION_ROLE_NAME,
316 region: str = "us-east-1",
317) -> list[str]:
318 """``aws iam attach-role-policy`` argv attaching the partition policy."""
319 _validate(role_name, _CLUSTER_RE, "role name")
320 _validate(region, _REGION_RE, "region")
321 return [
322 "aws",
323 "iam",
324 "attach-role-policy",
325 "--region",
326 region,
327 "--role-name",
328 role_name,
329 "--policy-arn",
330 ssm_managed_policy_arn(region),
331 ]
334def build_create_instance_profile_command(
335 profile_name: str = BASTION_PROFILE_NAME,
336 region: str = "us-east-1",
337) -> list[str]:
338 """``aws iam create-instance-profile`` argv."""
339 _validate(profile_name, _CLUSTER_RE, "instance-profile name")
340 _validate(region, _REGION_RE, "region")
341 return [
342 "aws",
343 "iam",
344 "create-instance-profile",
345 "--region",
346 region,
347 "--instance-profile-name",
348 profile_name,
349 ]
352def build_add_role_to_profile_command(
353 role_name: str = BASTION_ROLE_NAME,
354 profile_name: str = BASTION_PROFILE_NAME,
355 region: str = "us-east-1",
356) -> list[str]:
357 """``aws iam add-role-to-instance-profile`` argv."""
358 _validate(role_name, _CLUSTER_RE, "role name")
359 _validate(profile_name, _CLUSTER_RE, "instance-profile name")
360 _validate(region, _REGION_RE, "region")
361 return [
362 "aws",
363 "iam",
364 "add-role-to-instance-profile",
365 "--region",
366 region,
367 "--instance-profile-name",
368 profile_name,
369 "--role-name",
370 role_name,
371 ]
374def _tag_specification(
375 ttl_minutes: int,
376 instance_name: str = BASTION_NAME,
377 project_name: str = DEFAULT_PROJECT_NAME,
378) -> str:
379 """Build the ``--tag-specifications`` value stamping the ephemeral markers."""
380 project = _validate_project(project_name)
381 tags = (
382 f"{{Key=Name,Value={instance_name}}},"
383 f"{{Key={TAG_EPHEMERAL_KEY},Value=true}},"
384 f"{{Key={TAG_PURPOSE_KEY},Value={BASTION_PURPOSE}}},"
385 f"{{Key={TAG_PROJECT_KEY},Value={project}}},"
386 f"{{Key={TAG_TTL_KEY},Value={ttl_minutes}}}"
387 )
388 return f"ResourceType=instance,Tags=[{tags}]"
391def build_run_instances_command(
392 *,
393 ami_id: str,
394 instance_type: str,
395 subnet_id: str,
396 security_group_id: str,
397 profile_name: str,
398 region: str,
399 user_data: str,
400 ttl_minutes: int = DEFAULT_TTL_MINUTES,
401 instance_name: str = BASTION_NAME,
402 project_name: str = DEFAULT_PROJECT_NAME,
403) -> list[str]:
404 """Build the validated ``aws ec2 run-instances`` argv, safeguards included.
406 The safeguards (IMDSv2, shutdown-behaviour terminate, TTL user-data, and the
407 ``gco:ephemeral`` tag set) are not optional — they are the contract that
408 keeps a forgotten teardown from turning into a paid orphan.
409 """
410 _validate(ami_id, _AMI_RE, "AMI id")
411 _validate(subnet_id, _SUBNET_RE, "subnet id")
412 _validate(security_group_id, _SG_RE, "security group id")
413 _validate(profile_name, _CLUSTER_RE, "instance-profile name")
414 _validate(region, _REGION_RE, "region")
415 ttl = _validate_ttl(ttl_minutes)
416 if not _INSTANCE_TYPE_RE.match(instance_type):
417 raise ValueError(f"Invalid instance type {instance_type!r}")
419 cmd = [
420 "aws",
421 "ec2",
422 "run-instances",
423 "--region",
424 region,
425 "--image-id",
426 ami_id,
427 "--instance-type",
428 instance_type,
429 "--count",
430 "1",
431 "--subnet-id",
432 subnet_id,
433 "--security-group-ids",
434 security_group_id,
435 "--iam-instance-profile",
436 f"Name={profile_name}",
437 # IMDSv2 required.
438 "--metadata-options",
439 "HttpTokens=required,HttpEndpoint=enabled",
440 # Orphan safeguard #1: an OS-initiated shutdown terminates the instance.
441 "--instance-initiated-shutdown-behavior",
442 "terminate",
443 # Orphan safeguard #2: schedule that shutdown from boot.
444 "--user-data",
445 user_data,
446 # Orphan safeguard #3: greppable ephemeral tags.
447 "--tag-specifications",
448 _tag_specification(ttl, instance_name, project_name),
449 ]
450 cmd.append("--no-associate-public-ip-address")
451 # Ask only for the instance id back.
452 cmd += ["--query", "Instances[0].InstanceId", "--output", "text"]
453 return cmd
456def build_describe_ssm_ping_command(instance_id: str, region: str) -> list[str]:
457 """``aws ssm describe-instance-information`` argv projecting the PingStatus."""
458 _validate(instance_id, _INSTANCE_RE, "instance id")
459 _validate(region, _REGION_RE, "region")
460 return [
461 "aws",
462 "ssm",
463 "describe-instance-information",
464 "--region",
465 region,
466 "--filters",
467 f"Key=InstanceIds,Values={instance_id}",
468 "--query",
469 "InstanceInformationList[0].PingStatus",
470 "--output",
471 "text",
472 ]
475def build_terminate_instances_command(instance_id: str, region: str) -> list[str]:
476 """``aws ec2 terminate-instances`` argv."""
477 _validate(instance_id, _INSTANCE_RE, "instance id")
478 _validate(region, _REGION_RE, "region")
479 return [
480 "aws",
481 "ec2",
482 "terminate-instances",
483 "--region",
484 region,
485 "--instance-ids",
486 instance_id,
487 "--query",
488 "TerminatingInstances[0].CurrentState.Name",
489 "--output",
490 "text",
491 ]
494def build_iam_teardown_commands(
495 role_name: str = BASTION_ROLE_NAME,
496 profile_name: str = BASTION_PROFILE_NAME,
497 region: str = "us-east-1",
498) -> list[list[str]]:
499 """Return ordered, partition-aware IAM teardown argvs."""
500 _validate(role_name, _CLUSTER_RE, "role name")
501 _validate(profile_name, _CLUSTER_RE, "instance-profile name")
502 _validate(region, _REGION_RE, "region")
503 policy_arn = ssm_managed_policy_arn(region)
504 region_args = ["--region", region]
505 return [
506 [
507 "aws",
508 "iam",
509 "remove-role-from-instance-profile",
510 *region_args,
511 "--instance-profile-name",
512 profile_name,
513 "--role-name",
514 role_name,
515 ],
516 [
517 "aws",
518 "iam",
519 "delete-instance-profile",
520 *region_args,
521 "--instance-profile-name",
522 profile_name,
523 ],
524 [
525 "aws",
526 "iam",
527 "detach-role-policy",
528 *region_args,
529 "--role-name",
530 role_name,
531 "--policy-arn",
532 policy_arn,
533 ],
534 ["aws", "iam", "delete-role", *region_args, "--role-name", role_name],
535 ]
538# --------------------------------------------------------------------------
539# Pure parsers (unit-tested).
540# --------------------------------------------------------------------------
543def parse_cluster_network(stdout: str) -> tuple[str, str, list[str]]:
544 """Parse ``build_describe_cluster_network_command`` JSON → (vpc, sg, subnets)."""
545 data = json.loads(stdout or "{}")
546 vpc = data.get("vpc") or ""
547 sg = data.get("sg") or ""
548 subnets = data.get("subnets") or []
549 if not vpc or not sg:
550 raise RuntimeError(
551 "Cluster VPC or cluster security group not found in describe-cluster output; "
552 "cannot place an ephemeral bastion."
553 )
554 return vpc, sg, list(subnets)
557def _clean_scalar(stdout: str) -> str:
558 """Normalise an ``--output text`` scalar (strip; treat 'None' as empty)."""
559 value = (stdout or "").strip()
560 return "" if value in ("", "None") else value
563# --------------------------------------------------------------------------
564# Runtime wrappers (thin; shell out to the AWS CLI).
565# --------------------------------------------------------------------------
568def _run_aws(cmd: list[str], *, allow_exists: bool = False) -> str:
569 """Run an AWS CLI argv, returning stdout. Raise ``RuntimeError`` on failure.
571 ``allow_exists`` swallows idempotent "already exists" IAM errors so repeated
572 runs reuse the standing role/profile instead of failing.
573 """
574 try:
575 result = subprocess.run( # nosemgrep: dangerous-subprocess-use-audit - argv built by validated builders; list form, no shell=True
576 cmd,
577 capture_output=True,
578 text=True,
579 check=False,
580 )
581 except FileNotFoundError as exc:
582 raise RuntimeError(
583 "AWS CLI not found. Please install the AWS CLI and ensure it's in your PATH."
584 ) from exc
585 if result.returncode != 0:
586 stderr = result.stderr or ""
587 if allow_exists and (
588 "EntityAlreadyExists" in stderr
589 or "already exists" in stderr
590 # IAM reports an idempotent add-role-to-instance-profile call as
591 # this quota error when the profile already contains its one role.
592 or "Cannot exceed quota for InstanceSessionsPerInstanceProfile" in stderr
593 ):
594 return result.stdout or ""
595 raise RuntimeError(f"AWS CLI command failed ({' '.join(cmd[:3])}): {stderr.strip()}")
596 return result.stdout or ""
599def resolve_bastion_ami(region: str) -> str:
600 """Resolve the latest AL2023 x86_64 AMI id for ``region``."""
601 ami = _clean_scalar(_run_aws(build_get_ami_command(region)))
602 return _validate(ami, _AMI_RE, "resolved AMI id")
605def resolve_bastion_network(cluster: str, region: str) -> BastionNetwork:
606 """Discover private VPC placement for the bastion.
608 GCO gives EKS private control-plane subnets with NAT egress. We select one
609 of those subnets and refuse a public-subnet fallback, ensuring the bastion
610 never receives a public IP. A deployment without NAT can provide Systems
611 Manager interface VPC endpoints instead.
612 """
613 vpc, sg, subnets = parse_cluster_network(
614 _run_aws(build_describe_cluster_network_command(cluster, region))
615 )
616 _validate(vpc, _VPC_RE, "cluster vpc id")
617 _validate(sg, _SG_RE, "cluster security group id")
619 if not subnets:
620 raise RuntimeError(
621 f"No cluster subnets found in VPC {vpc}; cannot place a private bastion."
622 )
623 private_subnet = _clean_scalar(
624 _run_aws(build_describe_private_cluster_subnet_command(list(subnets), region))
625 )
626 if not private_subnet:
627 raise RuntimeError(
628 f"No private EKS subnet found in VPC {vpc}; refusing to launch a public bastion."
629 )
630 return BastionNetwork(
631 vpc,
632 _validate(private_subnet, _SUBNET_RE, "private cluster subnet id"),
633 sg,
634 )
637def ensure_bastion_iam(
638 project_name: str = DEFAULT_PROJECT_NAME,
639 region: str = "us-east-1",
640) -> None:
641 """Idempotently create the bastion role + profile in one partition."""
642 role_name = bastion_role_name(project_name)
643 profile_name = bastion_profile_name(project_name)
644 _run_aws(build_create_role_command(role_name, region), allow_exists=True)
645 _run_aws(build_attach_role_policy_command(role_name, region), allow_exists=True)
646 _run_aws(build_create_instance_profile_command(profile_name, region), allow_exists=True)
647 _run_aws(
648 build_add_role_to_profile_command(role_name, profile_name, region),
649 allow_exists=True,
650 )
653def launch_bastion(
654 *,
655 network: BastionNetwork,
656 ami_id: str,
657 region: str,
658 ttl_minutes: int,
659 project_name: str = DEFAULT_PROJECT_NAME,
660 instance_type: str = BASTION_INSTANCE_TYPE,
661) -> str:
662 """Run the instance, retrying briefly while the instance profile propagates."""
663 cmd = build_run_instances_command(
664 ami_id=ami_id,
665 instance_type=instance_type,
666 subnet_id=network.subnet_id,
667 security_group_id=network.security_group_id,
668 profile_name=bastion_profile_name(project_name),
669 region=region,
670 user_data=render_user_data(ttl_minutes),
671 ttl_minutes=ttl_minutes,
672 instance_name=bastion_instance_name(project_name),
673 project_name=project_name,
674 )
675 last_error: Exception | None = None
676 for attempt in range(_PROFILE_PROPAGATION_RETRIES):
677 try:
678 instance_id = _clean_scalar(_run_aws(cmd))
679 return _validate(instance_id, _INSTANCE_RE, "launched instance id")
680 except RuntimeError as exc: # bounded retry loop
681 last_error = exc
682 if (
683 "Invalid IAM Instance Profile" not in str(exc)
684 and "instance profile" not in str(exc).lower()
685 ):
686 raise
687 time.sleep(_PROFILE_PROPAGATION_WAIT_SECONDS)
688 logger.info(
689 "Instance profile not yet visible to EC2 (attempt %d); retrying launch.",
690 attempt + 1,
691 )
692 raise RuntimeError(
693 f"run-instances failed after {_PROFILE_PROPAGATION_RETRIES} attempts "
694 f"waiting for instance-profile propagation: {last_error}"
695 )
698def _is_instance_type_unavailable(error: Exception) -> bool:
699 """Return whether ``error`` means the type can't launch in this subnet.
701 Matches EC2's ``InsufficientInstanceCapacity`` (transient capacity) and
702 ``Unsupported`` family ("not supported in your requested Availability
703 Zone", Regions that never offered the type). Deliberately narrow:
704 authorization, vCPU-quota, and malformed-request errors do not match, so
705 they fail immediately instead of burning the fallback chain.
706 """
707 message = str(error)
708 return any(marker in message for marker in _INSTANCE_TYPE_UNAVAILABLE_MARKERS)
711def launch_bastion_with_fallback(
712 *,
713 network: BastionNetwork,
714 ami_id: str,
715 region: str,
716 ttl_minutes: int,
717 project_name: str = DEFAULT_PROJECT_NAME,
718 instance_types: tuple[str, ...] = (BASTION_INSTANCE_TYPE, *BASTION_INSTANCE_TYPE_FALLBACKS),
719) -> tuple[str, str]:
720 """Launch the bastion, walking the type chain when a type is unavailable.
722 Tries each candidate in order and returns ``(instance_id, instance_type)``
723 for the first successful launch. Only unavailability errors advance the
724 chain; every other failure propagates unchanged from :func:`launch_bastion`
725 (which still owns the instance-profile propagation retry per attempt).
726 Raises the last unavailability error when every candidate is exhausted.
727 """
728 if not instance_types:
729 raise ValueError("instance_types must name at least one candidate")
730 last_error: Exception | None = None
731 for candidate in instance_types:
732 try:
733 instance_id = launch_bastion(
734 network=network,
735 ami_id=ami_id,
736 region=region,
737 ttl_minutes=ttl_minutes,
738 project_name=project_name,
739 instance_type=candidate,
740 )
741 except RuntimeError as exc: # bounded fallback chain
742 if not _is_instance_type_unavailable(exc):
743 raise
744 last_error = exc
745 logger.warning(
746 "Instance type %s is unavailable in %s (%s); trying the next fallback.",
747 candidate,
748 region,
749 network.subnet_id,
750 )
751 continue
752 if candidate != instance_types[0]:
753 logger.info(
754 "Launched ephemeral bastion on fallback instance type %s "
755 "(preferred %s unavailable).",
756 candidate,
757 instance_types[0],
758 )
759 return instance_id, candidate
760 raise RuntimeError(
761 f"No bastion instance type in {list(instance_types)} could be launched in "
762 f"{region} ({network.subnet_id}); last error: {last_error}"
763 )
766def wait_until_ssm_online(
767 instance_id: str,
768 region: str,
769 *,
770 timeout_seconds: float = 240.0,
771 poll_interval_seconds: float = 10.0,
772) -> None:
773 """Block until the instance registers with SSM (PingStatus=Online) or time out."""
774 deadline = time.monotonic() + timeout_seconds
775 cmd = build_describe_ssm_ping_command(instance_id, region)
776 while time.monotonic() < deadline:
777 status = _clean_scalar(_run_aws(cmd))
778 if status == "Online":
779 return
780 time.sleep(poll_interval_seconds)
781 raise RuntimeError(
782 f"Instance {instance_id} did not come Online in SSM within "
783 f"{int(timeout_seconds)}s. Check that the private subnet can reach the SSM service "
784 "through NAT egress or Systems Manager interface VPC endpoints."
785 )
788def create_ephemeral_bastion(
789 cluster: str,
790 region: str,
791 *,
792 project_name: str = DEFAULT_PROJECT_NAME,
793 ttl_minutes: int = DEFAULT_TTL_MINUTES,
794 wait_online: bool = True,
795) -> str:
796 """Provision a self-terminating SSM bastion in the cluster VPC. Returns its id.
798 The IAM role / instance profile are named for ``project_name`` (default
799 ``gco``) so they match the deployment's other project-scoped resources.
800 """
801 _validate(cluster, _CLUSTER_RE, "cluster name")
802 _validate(region, _REGION_RE, "region")
803 _validate_project(project_name)
804 ttl = _validate_ttl(ttl_minutes)
806 ami_id = resolve_bastion_ami(region)
807 network = resolve_bastion_network(cluster, region)
808 ensure_bastion_iam(project_name, region)
809 instance_id, launched_type = launch_bastion_with_fallback(
810 network=network, ami_id=ami_id, region=region, ttl_minutes=ttl, project_name=project_name
811 )
812 logger.info(
813 "Launched ephemeral bastion %s (%s) in %s (%s).",
814 instance_id,
815 launched_type,
816 region,
817 network.subnet_id,
818 )
819 if wait_online:
820 try:
821 wait_until_ssm_online(instance_id, region)
822 except Exception:
823 # Atomic create: never leak the instance we just launched if it
824 # fails to register with SSM. The self-terminate user-data is only a
825 # last-resort backstop, not the normal cleanup path.
826 try:
827 destroy_ephemeral_bastion(instance_id, region, project_name=project_name)
828 except Exception: # pragma: no cover - best effort
829 logger.exception(
830 "Failed to clean up bastion %s after online-wait failure", instance_id
831 )
832 raise
833 return instance_id
836def destroy_ephemeral_bastion(
837 instance_id: str,
838 region: str,
839 *,
840 project_name: str = DEFAULT_PROJECT_NAME,
841 delete_iam: bool = True,
842) -> None:
843 """Terminate the bastion and (best-effort) delete its IAM role + profile.
845 Termination is the cost-critical step and is always attempted. IAM cleanup is
846 best-effort: a leftover role/instance-profile costs nothing and is greppable
847 by its ``gco:ephemeral`` tag, so a failure here is logged, not raised. The
848 role / profile deleted are the ones named for ``project_name``.
849 """
850 _validate(instance_id, _INSTANCE_RE, "instance id")
851 _validate(region, _REGION_RE, "region")
852 _validate_project(project_name)
854 _run_aws(build_terminate_instances_command(instance_id, region))
855 logger.info("Terminating ephemeral bastion %s.", instance_id)
857 if not delete_iam:
858 return
859 teardown = build_iam_teardown_commands(
860 bastion_role_name(project_name),
861 bastion_profile_name(project_name),
862 region,
863 )
864 for step in teardown:
865 try:
866 _run_aws(step, allow_exists=True)
867 except RuntimeError as exc: # best-effort — never mask the termination
868 logger.warning("IAM teardown step failed (%s): %s", " ".join(step[:3]), exc)
871@contextmanager
872def ephemeral_bastion(
873 cluster: str,
874 region: str,
875 *,
876 project_name: str = DEFAULT_PROJECT_NAME,
877 ttl_minutes: int = DEFAULT_TTL_MINUTES,
878) -> Iterator[str]:
879 """Context manager: create a bastion, yield its id, guarantee teardown."""
880 instance_id = create_ephemeral_bastion(
881 cluster, region, project_name=project_name, ttl_minutes=ttl_minutes
882 )
883 try:
884 yield instance_id
885 finally:
886 destroy_ephemeral_bastion(instance_id, region, project_name=project_name)