Coverage for cli/ephemeral_bastion.py: 100.00%

1"""Ephemeral SSM bastion lifecycle for reaching a private EKS API endpoint.

3``gco monitoring open --via-ssm <instance-id>`` tunnels to a private EKS

4endpoint through an *existing* SSM-managed instance. This module lets the CLI

5**create that instance on demand** — a minimal, self-terminating ``t3.micro``

6(falling back through equivalent burstable types when a Region or AZ can't

7launch one) in the cluster VPC — and tear it down when the port-forward

8session ends, so an operator doesn't have to keep a standing bastion around

9just to view Grafana.

11Orphan safeguards (defence in depth — a crash, ``Ctrl-C``, or a forgotten

12teardown must never leave a paid instance running):

14* launched with ``--instance-initiated-shutdown-behavior terminate``;

15* user-data schedules ``shutdown -h +<ttl>`` as an unconditional backstop;

16* IMDSv2 required (``HttpTokens=required``);

17* tagged ``gco:ephemeral=true`` + ``gco:purpose`` so any orphan is greppable;

18* the CLI tears it down in a ``finally:`` block.

20Network posture: the instance reuses the cluster's own security group (which is

21self-referencing, so it can reach the private API endpoint) and is placed in one

22of the cluster's private subnets **without a public IP**. GCO private subnets

23have NAT egress, which lets the SSM agent reach Systems Manager over HTTPS while

24keeping the instance unreachable from the internet. No inbound ports are opened;

25SSM is agent-initiated outbound only.

27Style matches :mod:`cli.ssm_tunnel`: pure, validated argv builders (list form,

28never a shell string) that are fully unit-testable, plus thin runtime wrappers

29that shell out to the AWS CLI (the ``cli`` package does not depend on boto3).

30"""

32from __future__ import annotations

34import json

35import logging

36import re

37import subprocess

38import time

39from collections.abc import Iterator

40from contextlib import contextmanager

41from dataclasses import dataclass

43from ._image_uri import aws_partition, aws_url_suffix

45logger = logging.getLogger(__name__)

47# --------------------------------------------------------------------------

48# Constants — the fixed identity of the ephemeral bastion.

49# --------------------------------------------------------------------------

51# The bastion's IAM role, instance profile, and Name tag are scoped to the

52# deployment's project key (cdk.json context.project_name, default "gco" — the

53# same key cli/config.py derives cluster and stack names from), so a non-default

54# deployment addresses its own resources and two differently named deployments in

55# one account don't collide on a shared name. See bastion_role_name() below.

56DEFAULT_PROJECT_NAME = "gco"

57_ROLE_NAME_SUFFIX = "-ephemeral-bastion-role"

58_PROFILE_NAME_SUFFIX = "-ephemeral-bastion-profile"

59_INSTANCE_NAME_SUFFIX = "-ephemeral-ssm-bastion"

62# AmazonSSMManagedInstanceCore is AWS-managed and therefore not project-scoped,

63# but its ARN partition must follow the deployment Region.

64def ssm_managed_policy_arn(region: str) -> str:

65 return f"arn:{aws_partition(region)}:iam::aws:policy/AmazonSSMManagedInstanceCore"

68def bastion_trust_policy(region: str) -> dict[str, object]:

69 """Return an EC2 trust policy using the partition's service DNS suffix."""

70 return {

71 "Version": "2012-10-17",

72 "Statement": [

73 {

74 "Effect": "Allow",

75 "Principal": {"Service": f"ec2.{aws_url_suffix(region)}"},

76 "Action": "sts:AssumeRole",

77 }

78 ],

79 }

82# Back-compatible commercial defaults for callers that imported the constants.

83SSM_MANAGED_POLICY_ARN = ssm_managed_policy_arn("us-east-1")

85BASTION_INSTANCE_TYPE = "t3.micro"

87# Fallback chain when the preferred type cannot be launched in the target

88# subnet — a Region/AZ that lacks the type entirely, or transient

89# InsufficientInstanceCapacity. Ordered cheapest-first and all x86_64 (the

90# resolved AL2023 AMI is x86_64, so an arm64 type can never be substituted

91# without also changing the AMI). The bastion only forwards an SSM tunnel, so

92# any burstable nano/micro/small class is dimensionally identical for the job.

93BASTION_INSTANCE_TYPE_FALLBACKS: tuple[str, ...] = (

94 "t3a.micro",

95 "t3.small",

96 "t2.micro",

97)

99# EC2 error markers that mean "this instance type won't launch here" — the

100# cue to try the next type rather than fail the tunnel. Anything else

101# (auth, quota on vCPUs, malformed request) still raises immediately.

102_INSTANCE_TYPE_UNAVAILABLE_MARKERS = (

103 "InsufficientInstanceCapacity",

104 "Unsupported",

105 "InstanceTypeNotSupported",

106 "unsupported instance type",

107 "not supported in your requested Availability Zone",

108)

109

110# Public SSM parameter that always resolves to the latest Amazon Linux 2023

111# x86_64 AMI in the target region (t3.micro is x86_64).

112AL2023_AMI_SSM_PARAMETER = "/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64"

113

114# Tags stamped on every ephemeral instance so orphans are trivially discoverable:

115# aws ec2 describe-instances \

116# --filters Name=tag:gco:ephemeral,Values=true \

117# Name=instance-state-name,Values=running,pending

118TAG_EPHEMERAL_KEY = "gco:ephemeral"

119TAG_PURPOSE_KEY = "gco:purpose"

120TAG_PROJECT_KEY = "gco:project"

121TAG_TTL_KEY = "gco:ttl-minutes"

122BASTION_PURPOSE = "cluster-observability"

123

124DEFAULT_TTL_MINUTES = 120

125

126# EC2 instance-profile propagation to the RunInstances API is eventually

127# consistent; retry the launch for a short window on the "not found" error.

128_PROFILE_PROPAGATION_RETRIES = 6

129_PROFILE_PROPAGATION_WAIT_SECONDS = 5.0

130

131# EC2 trust policy default; builders derive the principal from their target Region.

132BASTION_TRUST_POLICY: dict[str, object] = bastion_trust_policy("us-east-1")

133

134# --------------------------------------------------------------------------

135# Validators — every AWS-supplied id is re-validated before it enters an argv.

136# --------------------------------------------------------------------------

137

138_REGION_RE = re.compile(r"^[a-z]{2,4}(?:-[a-z0-9]+)+-[0-9]+$")

139_CLUSTER_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9\-]{0,99}$")

140_INSTANCE_RE = re.compile(r"^i-[0-9a-f]{8}([0-9a-f]{9})?$")

141_VPC_RE = re.compile(r"^vpc-[0-9a-f]{8}([0-9a-f]{9})?$")

142_SUBNET_RE = re.compile(r"^subnet-[0-9a-f]{8}([0-9a-f]{9})?$")

143_SG_RE = re.compile(r"^sg-[0-9a-f]{8}([0-9a-f]{9})?$")

144_AMI_RE = re.compile(r"^ami-[0-9a-f]{8}([0-9a-f]{9})?$")

145_INSTANCE_TYPE_RE = re.compile(r"^[a-z0-9]+\.[a-z0-9]+$")

146

147

148def _validate(value: str, pattern: re.Pattern[str], what: str) -> str:

149 if not isinstance(value, str) or not pattern.match(value):

150 raise ValueError(f"Invalid {what}: {value!r}")

151 return value

152

153

154def _validate_ttl(ttl_minutes: int) -> int:

155 try:

156 value = int(ttl_minutes)

157 except (TypeError, ValueError) as exc:

158 raise ValueError(f"Invalid ttl-minutes {ttl_minutes!r}: must be an integer") from exc

159 if not 5 <= value <= 1440:

160 raise ValueError(f"Invalid ttl-minutes {value}: must be between 5 and 1440")

161 return value

162

163

164# Project keys follow the cdk.json context.project_name shape (alnum + hyphen).

165_PROJECT_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9\-]{0,40}$")

166

167

168def _validate_project(project_name: str) -> str:

169 return _validate(project_name, _PROJECT_RE, "project name")

170

171

172def bastion_role_name(project_name: str = DEFAULT_PROJECT_NAME) -> str:

173 """IAM role name for ``project_name``'s ephemeral bastion."""

174 return f"{_validate_project(project_name)}{_ROLE_NAME_SUFFIX}"

175

176

177def bastion_profile_name(project_name: str = DEFAULT_PROJECT_NAME) -> str:

178 """Instance-profile name for ``project_name``'s ephemeral bastion."""

179 return f"{_validate_project(project_name)}{_PROFILE_NAME_SUFFIX}"

180

181

182def bastion_instance_name(project_name: str = DEFAULT_PROJECT_NAME) -> str:

183 """EC2 ``Name`` tag for ``project_name``'s ephemeral bastion."""

184 return f"{_validate_project(project_name)}{_INSTANCE_NAME_SUFFIX}"

185

186

187# Back-compat convenience constants for the default project. The helpers above

188# are the source of truth; these are the default-project values that the builder

189# defaults (and tests) reference.

190BASTION_ROLE_NAME = bastion_role_name()

191BASTION_PROFILE_NAME = bastion_profile_name()

192BASTION_NAME = bastion_instance_name()

193

194

195@dataclass(frozen=True)

196class BastionNetwork:

197 """Private VPC placement for an ephemeral bastion."""

198

199 vpc_id: str

200 subnet_id: str

201 security_group_id: str

202

203

204# --------------------------------------------------------------------------

205# Pure argv / payload builders (unit-tested; list form, no shell).

206# --------------------------------------------------------------------------

207

208

209def render_user_data(ttl_minutes: int = DEFAULT_TTL_MINUTES) -> str:

210 """Return the bastion boot script: an unconditional self-terminate backstop."""

211 ttl = _validate_ttl(ttl_minutes)

212 return (

213 "#!/bin/bash\n"

214 "# GCO ephemeral SSM bastion — self-terminate backstop.\n"

215 "# The instance is launched with --instance-initiated-shutdown-behavior\n"

216 "# terminate, so this scheduled halt terminates it even if no explicit\n"

217 "# teardown ever runs.\n"

218 f'shutdown -h +{ttl} "gco ephemeral bastion self-terminate backstop"\n'

219 )

220

221

222def build_get_ami_command(region: str) -> list[str]:

223 """``aws ssm get-parameter`` argv resolving the latest AL2023 x86_64 AMI."""

224 _validate(region, _REGION_RE, "region")

225 return [

226 "aws",

227 "ssm",

228 "get-parameter",

229 "--name",

230 AL2023_AMI_SSM_PARAMETER,

231 "--region",

232 region,

233 "--query",

234 "Parameter.Value",

235 "--output",

236 "text",

237 ]

238

239

240def build_describe_cluster_network_command(cluster: str, region: str) -> list[str]:

241 """``aws eks describe-cluster`` argv projecting VPC, cluster SG, and subnets."""

242 _validate(cluster, _CLUSTER_RE, "cluster name")

243 _validate(region, _REGION_RE, "region")

244 return [

245 "aws",

246 "eks",

247 "describe-cluster",

248 "--name",

249 cluster,

250 "--region",

251 region,

252 "--query",

253 ("cluster.resourcesVpcConfig.{vpc:vpcId,sg:clusterSecurityGroupId,subnets:subnetIds}"),

254 "--output",

255 "json",

256 ]

257

258

259def build_describe_private_cluster_subnet_command(subnet_ids: list[str], region: str) -> list[str]:

260 """Find the first private subnet among the EKS control-plane subnets."""

261 if not subnet_ids:

262 raise ValueError("At least one cluster subnet id is required")

263 validated_subnets = [

264 _validate(subnet_id, _SUBNET_RE, "cluster subnet id") for subnet_id in subnet_ids

265 ]

266 _validate(region, _REGION_RE, "region")

267 return [

268 "aws",

269 "ec2",

270 "describe-subnets",

271 "--region",

272 region,

273 "--subnet-ids",

274 *validated_subnets,

275 "--filters",

276 "Name=map-public-ip-on-launch,Values=false",

277 "--query",

278 "Subnets[0].SubnetId",

279 "--output",

280 "text",

281 ]

282

283

284def build_create_role_command(

285 role_name: str = BASTION_ROLE_NAME,

286 region: str = "us-east-1",

287) -> list[str]:

288 """``aws iam create-role`` argv with the partition-correct EC2 trust."""

289 _validate(role_name, _CLUSTER_RE, "role name")

290 _validate(region, _REGION_RE, "region")

291 return [

292 "aws",

293 "iam",

294 "create-role",

295 "--region",

296 region,

297 "--role-name",

298 role_name,

299 "--assume-role-policy-document",

300 json.dumps(bastion_trust_policy(region)),

301 "--description",

302 "GCO ephemeral SSM bastion (cluster-observability); safe to delete.",

303 "--tags",

304 f"Key={TAG_EPHEMERAL_KEY},Value=true",

305 f"Key={TAG_PURPOSE_KEY},Value={BASTION_PURPOSE}",

306 ]

307

308

309def build_attach_role_policy_command(

310 role_name: str = BASTION_ROLE_NAME,

311 region: str = "us-east-1",

312) -> list[str]:

313 """``aws iam attach-role-policy`` argv attaching the partition policy."""

314 _validate(role_name, _CLUSTER_RE, "role name")

315 _validate(region, _REGION_RE, "region")

316 return [

317 "aws",

318 "iam",

319 "attach-role-policy",

320 "--region",

321 region,

322 "--role-name",

323 role_name,

324 "--policy-arn",

325 ssm_managed_policy_arn(region),

326 ]

327

328

329def build_create_instance_profile_command(

330 profile_name: str = BASTION_PROFILE_NAME,

331 region: str = "us-east-1",

332) -> list[str]:

333 """``aws iam create-instance-profile`` argv."""

334 _validate(profile_name, _CLUSTER_RE, "instance-profile name")

335 _validate(region, _REGION_RE, "region")

336 return [

337 "aws",

338 "iam",

339 "create-instance-profile",

340 "--region",

341 region,

342 "--instance-profile-name",

343 profile_name,

344 ]

345

346

347def build_add_role_to_profile_command(

348 role_name: str = BASTION_ROLE_NAME,

349 profile_name: str = BASTION_PROFILE_NAME,

350 region: str = "us-east-1",

351) -> list[str]:

352 """``aws iam add-role-to-instance-profile`` argv."""

353 _validate(role_name, _CLUSTER_RE, "role name")

354 _validate(profile_name, _CLUSTER_RE, "instance-profile name")

355 _validate(region, _REGION_RE, "region")

356 return [

357 "aws",

358 "iam",

359 "add-role-to-instance-profile",

360 "--region",

361 region,

362 "--instance-profile-name",

363 profile_name,

364 "--role-name",

365 role_name,

366 ]

367

368

369def _tag_specification(

370 ttl_minutes: int,

371 instance_name: str = BASTION_NAME,

372 project_name: str = DEFAULT_PROJECT_NAME,

373) -> str:

374 """Build the ``--tag-specifications`` value stamping the ephemeral markers."""

375 project = _validate_project(project_name)

376 tags = (

377 f"{{Key=Name,Value={instance_name}}},"

378 f"{{Key={TAG_EPHEMERAL_KEY},Value=true}},"

379 f"{{Key={TAG_PURPOSE_KEY},Value={BASTION_PURPOSE}}},"

380 f"{{Key={TAG_PROJECT_KEY},Value={project}}},"

381 f"{{Key={TAG_TTL_KEY},Value={ttl_minutes}}}"

382 )

383 return f"ResourceType=instance,Tags=[{tags}]"

384

385

386def build_run_instances_command(

387 *,

388 ami_id: str,

389 instance_type: str,

390 subnet_id: str,

391 security_group_id: str,

392 profile_name: str,

393 region: str,

394 user_data: str,

395 ttl_minutes: int = DEFAULT_TTL_MINUTES,

396 instance_name: str = BASTION_NAME,

397 project_name: str = DEFAULT_PROJECT_NAME,

398) -> list[str]:

399 """Build the validated ``aws ec2 run-instances`` argv, safeguards included.

400

401 The safeguards (IMDSv2, shutdown-behaviour terminate, TTL user-data, and the

402 ``gco:ephemeral`` tag set) are not optional — they are the contract that

403 keeps a forgotten teardown from turning into a paid orphan.

404 """

405 _validate(ami_id, _AMI_RE, "AMI id")

406 _validate(subnet_id, _SUBNET_RE, "subnet id")

407 _validate(security_group_id, _SG_RE, "security group id")

408 _validate(profile_name, _CLUSTER_RE, "instance-profile name")

409 _validate(region, _REGION_RE, "region")

410 ttl = _validate_ttl(ttl_minutes)

411 if not _INSTANCE_TYPE_RE.match(instance_type):

412 raise ValueError(f"Invalid instance type {instance_type!r}")

413

414 cmd = [

415 "aws",

416 "ec2",

417 "run-instances",

418 "--region",

419 region,

420 "--image-id",

421 ami_id,

422 "--instance-type",

423 instance_type,

424 "--count",

425 "1",

426 "--subnet-id",

427 subnet_id,

428 "--security-group-ids",

429 security_group_id,

430 "--iam-instance-profile",

431 f"Name={profile_name}",

432 # IMDSv2 required.

433 "--metadata-options",

434 "HttpTokens=required,HttpEndpoint=enabled",

435 # Orphan safeguard #1: an OS-initiated shutdown terminates the instance.

436 "--instance-initiated-shutdown-behavior",

437 "terminate",

438 # Orphan safeguard #2: schedule that shutdown from boot.

439 "--user-data",

440 user_data,

441 # Orphan safeguard #3: greppable ephemeral tags.

442 "--tag-specifications",

443 _tag_specification(ttl, instance_name, project_name),

444 ]

445 cmd.append("--no-associate-public-ip-address")

446 # Ask only for the instance id back.

447 cmd += ["--query", "Instances[0].InstanceId", "--output", "text"]

448 return cmd

449

450

451def build_describe_ssm_ping_command(instance_id: str, region: str) -> list[str]:

452 """``aws ssm describe-instance-information`` argv projecting the PingStatus."""

453 _validate(instance_id, _INSTANCE_RE, "instance id")

454 _validate(region, _REGION_RE, "region")

455 return [

456 "aws",

457 "ssm",

458 "describe-instance-information",

459 "--region",

460 region,

461 "--filters",

462 f"Key=InstanceIds,Values={instance_id}",

463 "--query",

464 "InstanceInformationList[0].PingStatus",

465 "--output",

466 "text",

467 ]

468

469

470def build_terminate_instances_command(instance_id: str, region: str) -> list[str]:

471 """``aws ec2 terminate-instances`` argv."""

472 _validate(instance_id, _INSTANCE_RE, "instance id")

473 _validate(region, _REGION_RE, "region")

474 return [

475 "aws",

476 "ec2",

477 "terminate-instances",

478 "--region",

479 region,

480 "--instance-ids",

481 instance_id,

482 "--query",

483 "TerminatingInstances[0].CurrentState.Name",

484 "--output",

485 "text",

486 ]

487

488

489def build_iam_teardown_commands(

490 role_name: str = BASTION_ROLE_NAME,

491 profile_name: str = BASTION_PROFILE_NAME,

492 region: str = "us-east-1",

493) -> list[list[str]]:

494 """Return ordered, partition-aware IAM teardown argvs."""

495 _validate(role_name, _CLUSTER_RE, "role name")

496 _validate(profile_name, _CLUSTER_RE, "instance-profile name")

497 _validate(region, _REGION_RE, "region")

498 policy_arn = ssm_managed_policy_arn(region)

499 region_args = ["--region", region]

500 return [

501 [

502 "aws",

503 "iam",

504 "remove-role-from-instance-profile",

505 *region_args,

506 "--instance-profile-name",

507 profile_name,

508 "--role-name",

509 role_name,

510 ],

511 [

512 "aws",

513 "iam",

514 "delete-instance-profile",

515 *region_args,

516 "--instance-profile-name",

517 profile_name,

518 ],

519 [

520 "aws",

521 "iam",

522 "detach-role-policy",

523 *region_args,

524 "--role-name",

525 role_name,

526 "--policy-arn",

527 policy_arn,

528 ],

529 ["aws", "iam", "delete-role", *region_args, "--role-name", role_name],

530 ]

531

532

533# --------------------------------------------------------------------------

534# Pure parsers (unit-tested).

535# --------------------------------------------------------------------------

536

537

538def parse_cluster_network(stdout: str) -> tuple[str, str, list[str]]:

539 """Parse ``build_describe_cluster_network_command`` JSON → (vpc, sg, subnets)."""

540 data = json.loads(stdout or "{}")

541 vpc = data.get("vpc") or ""

542 sg = data.get("sg") or ""

543 subnets = data.get("subnets") or []

544 if not vpc or not sg:

545 raise RuntimeError(

546 "Cluster VPC or cluster security group not found in describe-cluster output; "

547 "cannot place an ephemeral bastion."

548 )

549 return vpc, sg, list(subnets)

550

551

552def _clean_scalar(stdout: str) -> str:

553 """Normalise an ``--output text`` scalar (strip; treat 'None' as empty)."""

554 value = (stdout or "").strip()

555 return "" if value in ("", "None") else value

556

557

558# --------------------------------------------------------------------------

559# Runtime wrappers (thin; shell out to the AWS CLI).

560# --------------------------------------------------------------------------

561

562

563def _run_aws(cmd: list[str], *, allow_exists: bool = False) -> str:

564 """Run an AWS CLI argv, returning stdout. Raise ``RuntimeError`` on failure.

565

566 ``allow_exists`` swallows idempotent "already exists" IAM errors so repeated

567 runs reuse the standing role/profile instead of failing.

568 """

569 try:

570 result = subprocess.run( # nosemgrep: dangerous-subprocess-use-audit - argv built by validated builders; list form, no shell=True

571 cmd, capture_output=True, text=True

572 )

573 except FileNotFoundError as exc:

574 raise RuntimeError(

575 "AWS CLI not found. Please install the AWS CLI and ensure it's in your PATH."

576 ) from exc

577 if result.returncode != 0:

578 stderr = result.stderr or ""

579 if allow_exists and (

580 "EntityAlreadyExists" in stderr

581 or "already exists" in stderr

582 # IAM reports an idempotent add-role-to-instance-profile call as

583 # this quota error when the profile already contains its one role.

584 or "Cannot exceed quota for InstanceSessionsPerInstanceProfile" in stderr

585 ):

586 return result.stdout or ""

587 raise RuntimeError(f"AWS CLI command failed ({' '.join(cmd[:3])}): {stderr.strip()}")

588 return result.stdout or ""

589

590

591def resolve_bastion_ami(region: str) -> str:

592 """Resolve the latest AL2023 x86_64 AMI id for ``region``."""

593 ami = _clean_scalar(_run_aws(build_get_ami_command(region)))

594 return _validate(ami, _AMI_RE, "resolved AMI id")

595

596

597def resolve_bastion_network(cluster: str, region: str) -> BastionNetwork:

598 """Discover private VPC placement for the bastion.

599

600 GCO gives EKS private control-plane subnets with NAT egress. We select one

601 of those subnets and refuse a public-subnet fallback, ensuring the bastion

602 never receives a public IP. A deployment without NAT can provide Systems

603 Manager interface VPC endpoints instead.

604 """

605 vpc, sg, subnets = parse_cluster_network(

606 _run_aws(build_describe_cluster_network_command(cluster, region))

607 )

608 _validate(vpc, _VPC_RE, "cluster vpc id")

609 _validate(sg, _SG_RE, "cluster security group id")

610

611 if not subnets:

612 raise RuntimeError(

613 f"No cluster subnets found in VPC {vpc}; cannot place a private bastion."

614 )

615 private_subnet = _clean_scalar(

616 _run_aws(build_describe_private_cluster_subnet_command(list(subnets), region))

617 )

618 if not private_subnet:

619 raise RuntimeError(

620 f"No private EKS subnet found in VPC {vpc}; refusing to launch a public bastion."

621 )

622 return BastionNetwork(

623 vpc,

624 _validate(private_subnet, _SUBNET_RE, "private cluster subnet id"),

625 sg,

626 )

627

628

629def ensure_bastion_iam(

630 project_name: str = DEFAULT_PROJECT_NAME,

631 region: str = "us-east-1",

632) -> None:

633 """Idempotently create the bastion role + profile in one partition."""

634 role_name = bastion_role_name(project_name)

635 profile_name = bastion_profile_name(project_name)

636 _run_aws(build_create_role_command(role_name, region), allow_exists=True)

637 _run_aws(build_attach_role_policy_command(role_name, region), allow_exists=True)

638 _run_aws(build_create_instance_profile_command(profile_name, region), allow_exists=True)

639 _run_aws(

640 build_add_role_to_profile_command(role_name, profile_name, region),

641 allow_exists=True,

642 )

643

644

645def launch_bastion(

646 *,

647 network: BastionNetwork,

648 ami_id: str,

649 region: str,

650 ttl_minutes: int,

651 project_name: str = DEFAULT_PROJECT_NAME,

652 instance_type: str = BASTION_INSTANCE_TYPE,

653) -> str:

654 """Run the instance, retrying briefly while the instance profile propagates."""

655 cmd = build_run_instances_command(

656 ami_id=ami_id,

657 instance_type=instance_type,

658 subnet_id=network.subnet_id,

659 security_group_id=network.security_group_id,

660 profile_name=bastion_profile_name(project_name),

661 region=region,

662 user_data=render_user_data(ttl_minutes),

663 ttl_minutes=ttl_minutes,

664 instance_name=bastion_instance_name(project_name),

665 project_name=project_name,

666 )

667 last_error: Exception | None = None

668 for attempt in range(_PROFILE_PROPAGATION_RETRIES):

669 try:

670 instance_id = _clean_scalar(_run_aws(cmd))

671 return _validate(instance_id, _INSTANCE_RE, "launched instance id")

672 except RuntimeError as exc: # noqa: PERF203 — bounded retry loop

673 last_error = exc

674 if (

675 "Invalid IAM Instance Profile" not in str(exc)

676 and "instance profile" not in str(exc).lower()

677 ):

678 raise

679 time.sleep(_PROFILE_PROPAGATION_WAIT_SECONDS)

680 logger.info(

681 "Instance profile not yet visible to EC2 (attempt %d); retrying launch.",

682 attempt + 1,

683 )

684 raise RuntimeError(

685 f"run-instances failed after {_PROFILE_PROPAGATION_RETRIES} attempts "

686 f"waiting for instance-profile propagation: {last_error}"

687 )

688

689

690def _is_instance_type_unavailable(error: Exception) -> bool:

691 """Return whether ``error`` means the type can't launch in this subnet.

692

693 Matches EC2's ``InsufficientInstanceCapacity`` (transient capacity) and

694 ``Unsupported`` family ("not supported in your requested Availability

695 Zone", Regions that never offered the type). Deliberately narrow:

696 authorization, vCPU-quota, and malformed-request errors do not match, so

697 they fail immediately instead of burning the fallback chain.

698 """

699 message = str(error)

700 return any(marker in message for marker in _INSTANCE_TYPE_UNAVAILABLE_MARKERS)

701

702

703def launch_bastion_with_fallback(

704 *,

705 network: BastionNetwork,

706 ami_id: str,

707 region: str,

708 ttl_minutes: int,

709 project_name: str = DEFAULT_PROJECT_NAME,

710 instance_types: tuple[str, ...] = (BASTION_INSTANCE_TYPE, *BASTION_INSTANCE_TYPE_FALLBACKS),

711) -> tuple[str, str]:

712 """Launch the bastion, walking the type chain when a type is unavailable.

713

714 Tries each candidate in order and returns ``(instance_id, instance_type)``

715 for the first successful launch. Only unavailability errors advance the

716 chain; every other failure propagates unchanged from :func:`launch_bastion`

717 (which still owns the instance-profile propagation retry per attempt).

718 Raises the last unavailability error when every candidate is exhausted.

719 """

720 if not instance_types:

721 raise ValueError("instance_types must name at least one candidate")

722 last_error: Exception | None = None

723 for candidate in instance_types:

724 try:

725 instance_id = launch_bastion(

726 network=network,

727 ami_id=ami_id,

728 region=region,

729 ttl_minutes=ttl_minutes,

730 project_name=project_name,

731 instance_type=candidate,

732 )

733 except RuntimeError as exc: # noqa: PERF203 — bounded fallback chain

734 if not _is_instance_type_unavailable(exc):

735 raise

736 last_error = exc

737 logger.warning(

738 "Instance type %s is unavailable in %s (%s); trying the next fallback.",

739 candidate,

740 region,

741 network.subnet_id,

742 )

743 continue

744 if candidate != instance_types[0]:

745 logger.info(

746 "Launched ephemeral bastion on fallback instance type %s "

747 "(preferred %s unavailable).",

748 candidate,

749 instance_types[0],

750 )

751 return instance_id, candidate

752 raise RuntimeError(

753 f"No bastion instance type in {list(instance_types)} could be launched in "

754 f"{region} ({network.subnet_id}); last error: {last_error}"

755 )

756

757

758def wait_until_ssm_online(

759 instance_id: str,

760 region: str,

761 *,

762 timeout_seconds: float = 240.0,

763 poll_interval_seconds: float = 10.0,

764) -> None:

765 """Block until the instance registers with SSM (PingStatus=Online) or time out."""

766 deadline = time.monotonic() + timeout_seconds

767 cmd = build_describe_ssm_ping_command(instance_id, region)

768 while time.monotonic() < deadline:

769 status = _clean_scalar(_run_aws(cmd))

770 if status == "Online":

771 return

772 time.sleep(poll_interval_seconds)

773 raise RuntimeError(

774 f"Instance {instance_id} did not come Online in SSM within "

775 f"{int(timeout_seconds)}s. Check that the private subnet can reach the SSM service "

776 "through NAT egress or Systems Manager interface VPC endpoints."

777 )

778

779

780def create_ephemeral_bastion(

781 cluster: str,

782 region: str,

783 *,

784 project_name: str = DEFAULT_PROJECT_NAME,

785 ttl_minutes: int = DEFAULT_TTL_MINUTES,

786 wait_online: bool = True,

787) -> str:

788 """Provision a self-terminating SSM bastion in the cluster VPC. Returns its id.

789

790 The IAM role / instance profile are named for ``project_name`` (default

791 ``gco``) so they match the deployment's other project-scoped resources.

792 """

793 _validate(cluster, _CLUSTER_RE, "cluster name")

794 _validate(region, _REGION_RE, "region")

795 _validate_project(project_name)

796 ttl = _validate_ttl(ttl_minutes)

797

798 ami_id = resolve_bastion_ami(region)

799 network = resolve_bastion_network(cluster, region)

800 ensure_bastion_iam(project_name, region)

801 instance_id, launched_type = launch_bastion_with_fallback(

802 network=network, ami_id=ami_id, region=region, ttl_minutes=ttl, project_name=project_name

803 )

804 logger.info(

805 "Launched ephemeral bastion %s (%s) in %s (%s).",

806 instance_id,

807 launched_type,

808 region,

809 network.subnet_id,

810 )

811 if wait_online:

812 try:

813 wait_until_ssm_online(instance_id, region)

814 except Exception:

815 # Atomic create: never leak the instance we just launched if it

816 # fails to register with SSM. The self-terminate user-data is only a

817 # last-resort backstop, not the normal cleanup path.

818 try:

819 destroy_ephemeral_bastion(instance_id, region, project_name=project_name)

820 except Exception: # pragma: no cover - best effort

821 logger.exception(

822 "Failed to clean up bastion %s after online-wait failure", instance_id

823 )

824 raise

825 return instance_id

826

827

828def destroy_ephemeral_bastion(

829 instance_id: str,

830 region: str,

831 *,

832 project_name: str = DEFAULT_PROJECT_NAME,

833 delete_iam: bool = True,

834) -> None:

835 """Terminate the bastion and (best-effort) delete its IAM role + profile.

836

837 Termination is the cost-critical step and is always attempted. IAM cleanup is

838 best-effort: a leftover role/instance-profile costs nothing and is greppable

839 by its ``gco:ephemeral`` tag, so a failure here is logged, not raised. The

840 role / profile deleted are the ones named for ``project_name``.

841 """

842 _validate(instance_id, _INSTANCE_RE, "instance id")

843 _validate(region, _REGION_RE, "region")

844 _validate_project(project_name)

845

846 _run_aws(build_terminate_instances_command(instance_id, region))

847 logger.info("Terminating ephemeral bastion %s.", instance_id)

848

849 if not delete_iam:

850 return

851 teardown = build_iam_teardown_commands(

852 bastion_role_name(project_name),

853 bastion_profile_name(project_name),

854 region,

855 )

856 for step in teardown:

857 try:

858 _run_aws(step, allow_exists=True)

859 except RuntimeError as exc: # best-effort — never mask the termination

860 logger.warning("IAM teardown step failed (%s): %s", " ".join(step[:3]), exc)

861

862

863@contextmanager

864def ephemeral_bastion(

865 cluster: str,

866 region: str,

867 *,

868 project_name: str = DEFAULT_PROJECT_NAME,

869 ttl_minutes: int = DEFAULT_TTL_MINUTES,

870) -> Iterator[str]:

871 """Context manager: create a bastion, yield its id, guarantee teardown."""

872 instance_id = create_ephemeral_bastion(

873 cluster, region, project_name=project_name, ttl_minutes=ttl_minutes

874 )

875 try:

876 yield instance_id

877 finally:

878 destroy_ephemeral_bastion(instance_id, region, project_name=project_name)

Coverage for cli / ephemeral_bastion.py: 100.00%

258 statements