Coverage for cli/upgrade.py: 100.00%

1"""Whole-deployment upgrade orchestration behind ``gco upgrade``.

3An upgrade moves a GCO installation — the checked-out source, the locally

4installed CLI and toolchain, the ``gco-dev`` container image, and the deployed

5CloudFormation stacks — to a tagged release in one pass:

71. **Resolve the release.** Fetch the tags of the configured git remote and pick

8 the highest ``vMAJOR.MINOR.PATCH`` (or the tag named with ``--ref``).

92. **Move the checkout.** ``git checkout --detach <tag>`` with ``cdk.json``

10 preserved byte-for-byte: the deployment's configuration is the operator's,

11 not the release's, so it is snapshotted before the checkout and written back

12 afterwards.

133. **Refresh the local install.** ``pip install -e '.[cdk,...]'`` (``cdk`` plus

14 every extra already installed) when the running CLI is

15 the editable install of this checkout, ``npm ci`` when the checkout carries

16 its own CDK toolchain, and a rebuild of the ``gco-dev`` image when a

17 container runtime and the image are both present.

184. **Cycle the stacks.** Check that the checkout's deployable sources are

19 readable by every user (the redeploy would refuse them otherwise, after the

20 teardown), then scale the workload tier to zero

21 (:meth:`cli.stacks.StackManager.destroy_orchestrated` with

22 ``keep_control_plane=True`` — the monitoring stack is first updated in place

23 so it stops referencing the regional stacks), then run the ordinary

24 deploy-all: the global and API Gateway stacks are updated in place and the

25 regional stacks, their API bridges and the monitoring stack are recreated

26 from the new release.

28Step 4 deletes every regional stack, and with it the data those stacks own

29(EFS and FSx file systems, Valkey, Aurora, in-cluster volumes, the

30regional-shared bucket unless retained). ``docs/UPGRADING.md`` describes the

31back-up-to-the-cluster-shared-bucket / upgrade / restore procedure; the

32command's confirmation prompt repeats the essentials.

34The functions here take plain arguments and return dataclasses so the Click

35layer (:mod:`cli.commands.upgrade_cmd`) stays a thin veneer and every branch

36can be exercised without git, pip, or AWS.

37"""

39from __future__ import annotations

41import re

42import shutil

43import subprocess # fixed argv only, never a shell string

44import sys

45from collections.abc import Callable, Iterable, Sequence

46from dataclasses import asdict, dataclass, field

47from importlib import metadata

48from pathlib import Path

49from typing import Any

51from . import __version__

52from ._container_runtime import container_runtime_error_message, detect_container_runtime

54# <pyflowchart-code-diagram> BEGIN - auto-inserted, do not edit

55# Generated at (UTC): 2026-09-29T11:35:23Z

56# Generated from Git commit: a081fbf678ca3f3fa3f7edb74687de923efe6657

57# Flowchart(s) generated from this file:

58# * ``run_stack_cycle`` -> ``diagrams/code_diagrams/cli/upgrade.run_stack_cycle.html``

59# (PNG: ``diagrams/code_diagrams/cli/upgrade.run_stack_cycle.png``)

60# Regenerate with ``SOURCE_DATE_EPOCH=<unix-seconds> GCO_DIAGRAM_SOURCE_COMMIT=<40-char-sha> python diagrams/generate.py --code-only``.

61# <pyflowchart-code-diagram> END

64#: Release tags are the immutable ``vMAJOR.MINOR.PATCH`` tags release-publish.yml creates.

65RELEASE_TAG_RE = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")

67#: The git remote whose tags define "the latest release" unless ``--remote`` says otherwise.

68DEFAULT_REMOTE = "origin"

70#: The dev-container image ``scripts/setup-dev-alias.sh`` builds and the shell function runs.

71DEFAULT_DEV_IMAGE = "gco-dev"

73#: Files that belong to the deployment rather than the release and survive the checkout.

74PRESERVED_FILES: tuple[str, ...] = ("cdk.json",)

76#: The distribution the editable install registers (``[project] name``).

77DISTRIBUTION = "gco-cli"

79#: Extras the stack cycle needs whatever else is installed: the deploy synthesizes the CDK app.

80REQUIRED_EXTRAS: tuple[str, ...] = ("cdk",)

82_EXTRA_MARKER_RE = re.compile(r"""extra\s*==\s*["']([^"']+)["']""")

83_REQUIREMENT_NAME_RE = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)")

85#: Wall-clock cap for one git/pip/npm/image-build subprocess.

86_TOOL_TIMEOUT_SECONDS = 1800.0

88Logger = Callable[[str], None]

91class UpgradeError(RuntimeError):

92 """A precondition or step of the upgrade failed; the message is operator-facing."""

95def _quiet(_message: str) -> None:

96 """Default progress sink."""

99# =============================================================================

100# Release tags

101# =============================================================================

102

103

104@dataclass(frozen=True, order=True)

105class ReleaseTag:

106 """One ``vMAJOR.MINOR.PATCH`` tag, ordered by version."""

107

108 version: tuple[int, int, int]

109 name: str

110

111 @classmethod

112 def parse(cls, name: str) -> ReleaseTag | None:

113 """Return the tag when ``name`` has the release shape, else ``None``."""

114 match = RELEASE_TAG_RE.match(name.strip())

115 if match is None:

116 return None

117 major, minor, patch = (int(part) for part in match.groups())

118 return cls(version=(major, minor, patch), name=name.strip())

119

120 @property

121 def dotted(self) -> str:

122 """The version without its ``v`` prefix, as ``VERSION`` and ``gco --version`` print it."""

123 return ".".join(str(part) for part in self.version)

124

125

126def parse_release_tags(names: Iterable[str]) -> list[ReleaseTag]:

127 """Keep only release-shaped tags, sorted ascending by version."""

128 tags = [tag for tag in (ReleaseTag.parse(name) for name in names) if tag is not None]

129 return sorted(set(tags))

130

131

132def latest_release(tags: Sequence[ReleaseTag]) -> ReleaseTag:

133 """Return the highest release tag or explain that there is none."""

134 if not tags:

135 raise UpgradeError(

136 "No release tags (vMAJOR.MINOR.PATCH) were found on the remote. Check that the "

137 "remote is the GCO repository (or a fork that carries its tags) and that "

138 "'git fetch --tags' succeeds."

139 )

140 return tags[-1]

141

142

143def resolve_target(tags: Sequence[ReleaseTag], ref: str | None) -> ReleaseTag:

144 """Pick the upgrade target: the latest release, or the explicitly named one."""

145 if ref is None:

146 return latest_release(tags)

147 wanted = ReleaseTag.parse(ref)

148 if wanted is None:

149 raise UpgradeError(

150 f"--ref must name a release tag such as v8.1.0 (got {ref!r}); gco upgrade only "

151 "moves between tagged releases."

152 )

153 for tag in tags:

154 if tag == wanted:

155 return tag

156 known = ", ".join(tag.name for tag in tags[-5:]) or "none"

157 raise UpgradeError(f"Release tag {wanted.name} does not exist on the remote (latest: {known}).")

158

159

160# =============================================================================

161# The checkout

162# =============================================================================

163

164

165@dataclass

166class CheckoutState:

167 """What ``git`` says about the checkout before anything is changed."""

168

169 root: Path

170 head: str

171 ref: str

172 version_file: str | None

173 dirty_paths: list[str] = field(default_factory=list)

174 preserved_modified: list[str] = field(default_factory=list)

175

176 def to_dict(self) -> dict[str, Any]:

177 data = asdict(self)

178 data["root"] = str(self.root)

179 return data

180

181

182def _run(

183 argv: Sequence[str],

184 *,

185 cwd: Path | None = None,

186 timeout: float = _TOOL_TIMEOUT_SECONDS,

187) -> subprocess.CompletedProcess[str]:

188 """Run a fixed-argv tool, translating a missing binary or a timeout into UpgradeError."""

189 try:

190 return subprocess.run( # fixed argv, no shell

191 list(argv),

192 cwd=str(cwd) if cwd is not None else None,

193 capture_output=True,

194 text=True,

195 timeout=timeout,

196 check=False,

197 )

198 except FileNotFoundError as exc:

199 raise UpgradeError(f"{argv[0]} is not installed or not on PATH") from exc

200 except subprocess.TimeoutExpired as exc:

201 raise UpgradeError(f"{' '.join(argv)} did not finish within {timeout:.0f}s") from exc

202

203

204def run_git(root: Path, *args: str) -> str:

205 """Run ``git -C <root> <args>`` and return stdout, raising UpgradeError on failure."""

206 result = _run(["git", "-C", str(root), *args])

207 if result.returncode != 0:

208 detail = (result.stderr or result.stdout).strip() or f"exit status {result.returncode}"

209 raise UpgradeError(f"git {' '.join(args)} failed: {detail}")

210 return result.stdout

211

212

213def find_checkout_root(start: Path | None = None) -> Path:

214 """Locate the GCO checkout: the nearest ancestor holding both ``cdk.json`` and ``.git``.

215

216 Deploying needs the CDK app (``cdk.json``/``app.py``) and moving between

217 releases needs git history, so an installed-only CLI (``uv tool``, ``pipx``

218 from a git URL) running outside a clone is refused with the fix spelled out.

219 """

220 current = (start or Path.cwd()).resolve()

221 for candidate in (current, *current.parents):

222 if (candidate / "cdk.json").is_file() and (candidate / ".git").exists():

223 return candidate

224 raise UpgradeError(

225 "gco upgrade must run inside a git checkout of the GCO repository (a directory tree "

226 "holding cdk.json and .git). Clone the repository, cd into it, and run the command "

227 "again; the CLI you are running does not need to come from that checkout."

228 )

229

230

231def inspect_checkout(root: Path) -> CheckoutState:

232 """Record HEAD, the symbolic ref, the VERSION file, and every local modification."""

233 head = run_git(root, "rev-parse", "--short", "HEAD").strip()

234 ref = run_git(root, "rev-parse", "--abbrev-ref", "HEAD").strip()

235 if ref == "HEAD":

236 exact = _run(["git", "-C", str(root), "describe", "--tags", "--exact-match", "HEAD"])

237 ref = exact.stdout.strip() if exact.returncode == 0 and exact.stdout.strip() else head

238 version_path = root / "VERSION"

239 version_file = (

240 version_path.read_text(encoding="utf-8").strip() if version_path.is_file() else None

241 )

242

243 dirty: list[str] = []

244 preserved: list[str] = []

245 for line in run_git(root, "status", "--porcelain", "--untracked-files=no").splitlines():

246 # Porcelain v1: two status letters, a space, then the path (renames as

247 # ``old -> new``; the new name is the one that is modified).

248 path = line[3:].split(" -> ")[-1].strip()

249 if path in PRESERVED_FILES:

250 preserved.append(path)

251 else:

252 dirty.append(path)

253 return CheckoutState(

254 root=root,

255 head=head,

256 ref=ref,

257 version_file=version_file,

258 dirty_paths=sorted(dirty),

259 preserved_modified=sorted(preserved),

260 )

261

262

263def fetch_release_tags(root: Path, remote: str) -> list[ReleaseTag]:

264 """Fetch the remote's tags and return the release-shaped ones, ascending."""

265 run_git(root, "fetch", "--tags", "--quiet", remote)

266 listed = run_git(root, "tag", "--list", "v*")

267 return parse_release_tags(listed.splitlines())

268

269

270def checkout_is_at(root: Path, tag: ReleaseTag) -> bool:

271 """Return whether HEAD already is the commit the release tag points at."""

272 head = run_git(root, "rev-parse", "HEAD").strip()

273 target = run_git(root, "rev-parse", f"{tag.name}^{{commit}}").strip()

274 return head == target

275

276

277def checkout_release(root: Path, tag: ReleaseTag, *, log: Logger = _quiet) -> dict[str, Any]:

278 """Detach the checkout onto ``tag`` while preserving the deployment's own files.

279

280 ``cdk.json`` is tracked, but its contents (project name, Regions, feature

281 toggles) describe this deployment rather than the release, so its exact

282 bytes are captured first, reset so git will switch branches, and written

283 back once the release is checked out. Any other local modification aborts

284 before the checkout — the operator decides what to do with real work.

285 """

286 state = inspect_checkout(root)

287 if state.dirty_paths:

288 raise UpgradeError(

289 "The checkout has local modifications that gco upgrade will not touch: "

290 + ", ".join(state.dirty_paths)

291 + ". Commit or stash them (cdk.json is preserved automatically) and run again."

292 )

293

294 snapshots: dict[str, bytes] = {}

295 for name in PRESERVED_FILES:

296 path = root / name

297 if path.is_file():

298 snapshots[name] = path.read_bytes()

299 if state.preserved_modified:

300 # git refuses to switch when a modified tracked file differs between the

301 # two commits; the bytes are safe in ``snapshots`` and come right back.

302 run_git(root, "checkout", "--quiet", "--", *state.preserved_modified)

303

304 log(f"Checking out {tag.name} (was {state.ref} at {state.head})...")

305 run_git(root, "checkout", "--quiet", "--detach", tag.name)

306

307 restored: list[str] = []

308 for name, data in snapshots.items():

309 path = root / name

310 if not path.is_file() or path.read_bytes() != data:

311 path.write_bytes(data)

312 restored.append(name)

313 return {

314 "previous_ref": state.ref,

315 "previous_head": state.head,

316 "checked_out": tag.name,

317 "preserved": sorted(snapshots),

318 "restored": restored,

319 }

320

321

322# =============================================================================

323# The local install (CLI, CDK toolchain, dev container)

324# =============================================================================

325

326

327@dataclass

328class InstallProbe:

329 """How the running CLI and its toolchain relate to the checkout."""

330

331 cli_version: str

332 cli_path: str

333 cli_editable_from_checkout: bool

334 python: str

335 node_toolchain: bool

336 container_runtime: str | None

337 dev_image: str

338 dev_image_present: bool

339

340 def to_dict(self) -> dict[str, Any]:

341 return asdict(self)

342

343

344def _cli_source_root() -> Path:

345 """The directory the running ``cli`` package is imported from."""

346 return Path(__file__).resolve().parent.parent

347

348

349def image_exists(runtime: str, image: str) -> bool:

350 """Return whether ``image`` is present in the runtime's local image store."""

351 result = _run([runtime, "image", "inspect", image], timeout=60)

352 return result.returncode == 0

353

354

355def probe_install(root: Path, *, image: str = DEFAULT_DEV_IMAGE) -> InstallProbe:

356 """Describe the running CLI, the checkout's toolchain, and the dev image."""

357 runtime = detect_container_runtime()

358 return InstallProbe(

359 cli_version=__version__,

360 cli_path=str(_cli_source_root()),

361 cli_editable_from_checkout=_cli_source_root() == root.resolve(),

362 python=sys.executable,

363 node_toolchain=(root / "node_modules" / ".bin" / "cdk").exists(),

364 container_runtime=runtime,

365 dev_image=image,

366 dev_image_present=bool(runtime) and image_exists(str(runtime), image),

367 )

368

369

370def _installed(name: str) -> bool:

371 try:

372 metadata.distribution(name)

373 except metadata.PackageNotFoundError:

374 return False

375 return True

376

377

378def installed_extras(distribution: str = DISTRIBUTION) -> list[str]:

379 """The optional-dependency groups to reinstall: the ones already installed, plus ``cdk``.

380

381 ``pip install -e .`` alone never touches an extra, so a release that moves

382 the ``cdk`` pins (aws-cdk-lib, cdk-nag, constructs) would otherwise be

383 deployed with the previous release's CDK library. An editable install

384 keeps the metadata it was installed with, so this reads the groups the

385 operator had before the checkout moved: every extra whose requirements are

386 all installed is asked for again, which only re-pins packages already

387 present. ``cdk`` is always included because the stack cycle synthesizes

388 the CDK app.

389 """

390 extras = set(REQUIRED_EXTRAS)

391 try:

392 requirements = metadata.distribution(distribution).requires or []

393 except metadata.PackageNotFoundError:

394 return sorted(extras)

395 self_name = _canonical_name(distribution)

396 by_extra: dict[str, list[str]] = {}

397 for requirement in requirements:

398 extra = _EXTRA_MARKER_RE.search(requirement)

399 name = _REQUIREMENT_NAME_RE.match(requirement)

400 if extra is None or name is None:

401 continue

402 names = by_extra.setdefault(extra.group(1), [])

403 if _canonical_name(name.group(1)) != self_name:

404 names.append(name.group(1))

405 for extra_name, names in by_extra.items():

406 if all(_installed(name) for name in names):

407 extras.add(extra_name)

408 return sorted(extras)

409

410

411def _canonical_name(name: str) -> str:

412 return re.sub(r"[-_.]+", "-", name).lower()

413

414

415def refresh_python_install(root: Path, *, log: Logger = _quiet) -> dict[str, Any]:

416 """Reinstall the checkout into the running interpreter so new pins take effect.

417

418 The reinstall asks for the extras :func:`installed_extras` returns, so the

419 CDK toolchain and any other installed group follow the release's pins.

420 ``python -m pip`` is tried first; interpreters created by ``uv`` ship

421 without pip, so ``uv pip`` is the fallback when ``uv`` is on PATH.

422 """

423 extras = installed_extras()

424 target = f"{root}[{','.join(extras)}]"

425 pip_argv = [sys.executable, "-m", "pip", "install", "--quiet", "--no-input", "-e", target]

426 log(f"Refreshing the editable CLI install with its extras ({', '.join(extras)})...")

427 result = _run(pip_argv, cwd=root)

428 if result.returncode == 0:

429 return {"tool": "pip", "argv": pip_argv, "extras": extras, "status": "ok"}

430

431 uv = shutil.which("uv")

432 if uv is not None:

433 uv_argv = [uv, "pip", "install", "--quiet", "--python", sys.executable, "-e", target]

434 log("pip is unavailable in this interpreter; retrying with uv pip...")

435 uv_result = _run(uv_argv, cwd=root)

436 if uv_result.returncode == 0:

437 return {"tool": "uv", "argv": uv_argv, "extras": extras, "status": "ok"}

438 result = uv_result

439 detail = (result.stderr or result.stdout).strip()[-800:]

440 raise UpgradeError(

441 "Could not refresh the editable CLI install after checking out the release; the "

442 "deploy would run the new app against old dependencies. Run "

443 f"\"{sys.executable} -m pip install -e '{target}'\" (or uv pip install) yourself, "

444 f"then rerun 'gco upgrade --skip-checkout'. Installer output:\n{detail}"

445 )

446

447

448def refresh_node_toolchain(root: Path, *, log: Logger = _quiet) -> dict[str, Any]:

449 """Reinstall the checkout-local CDK toolchain from the release's lockfile (best effort)."""

450 npm = shutil.which("npm")

451 if npm is None:

452 return {"status": "skipped", "reason": "npm is not on PATH"}

453 argv = [npm, "ci", "--ignore-scripts", "--no-audit", "--no-fund"]

454 log("Refreshing the checkout's CDK toolchain (npm ci)...")

455 result = _run(argv, cwd=root)

456 if result.returncode == 0:

457 return {"status": "ok", "argv": argv}

458 detail = (result.stderr or result.stdout).strip()[-400:]

459 return {

460 "status": "warning",

461 "argv": argv,

462 "message": (

463 "npm ci failed; the previously installed CDK CLI stays in place. If 'cdk' "

464 f"versions now mismatch, run 'npm ci' in {root} by hand. Output:\n{detail}"

465 ),

466 }

467

468

469def rebuild_dev_image(

470 root: Path,

471 *,

472 runtime: str,

473 image: str = DEFAULT_DEV_IMAGE,

474 log: Logger = _quiet,

475) -> dict[str, Any]:

476 """Rebuild the dev-container image from the release's Dockerfile.dev (best effort).

477

478 Mirrors ``scripts/setup-dev-alias.sh``'s build step. A failure is reported

479 rather than fatal: the stack cycle does not depend on the image, and the

480 setup script can be rerun to retry with its own registry retries.

481 """

482 dockerfile = root / "Dockerfile.dev"

483 if not dockerfile.is_file():

484 return {"status": "skipped", "reason": "Dockerfile.dev is not part of this release"}

485 argv = [runtime, "build", "-f", str(dockerfile), "-t", image, str(root)]

486 log(f"Rebuilding the {image} container image with {runtime} (this can take a few minutes)...")

487 result = _run(argv, cwd=root, timeout=3600)

488 if result.returncode == 0:

489 return {"status": "ok", "image": image, "runtime": runtime}

490 detail = (result.stderr or result.stdout).strip()[-400:]

491 return {

492 "status": "warning",

493 "image": image,

494 "runtime": runtime,

495 "message": (

496 f"{runtime} could not rebuild {image}; the current image keeps running the "

497 "previous release until you rerun './scripts/setup-dev-alias.sh'. "

498 f"Output:\n{detail}"

499 ),

500 }

501

502

503# =============================================================================

504# The plan and the stack cycle

505# =============================================================================

506

507

508@dataclass

509class UpgradePlan:

510 """Everything the confirmation prompt shows and the JSON document reports."""

511

512 current_version: str

513 target: ReleaseTag

514 latest: ReleaseTag

515 checkout: CheckoutState

516 install: InstallProbe

517 already_at_target: bool

518 control_plane_stacks: list[str]

519 workload_stacks: list[str]

520 remote: str

521

522 def to_dict(self) -> dict[str, Any]:

523 return {

524 "current_version": self.current_version,

525 "checkout_version": self.checkout.version_file,

526 "target": self.target.name,

527 "latest": self.latest.name,

528 "remote": self.remote,

529 "already_at_target": self.already_at_target,

530 "checkout": self.checkout.to_dict(),

531 "install": self.install.to_dict(),

532 "control_plane_stacks": list(self.control_plane_stacks),

533 "workload_stacks": list(self.workload_stacks),

534 }

535

536

537def split_stacks(stacks: Sequence[str], project_name: str) -> tuple[list[str], list[str]]:

538 """Return ``(control_plane, workload)`` in the order the upgrade touches them."""

539 from .stacks import get_stack_destroy_order

540

541 workload = get_stack_destroy_order(

542 list(stacks), project_name=project_name, keep_control_plane=True

543 )

544 control_plane = [stack for stack in stacks if stack not in set(workload)]

545 return control_plane, workload

546

547

548def build_plan(

549 root: Path,

550 *,

551 project_name: str,

552 stacks: Sequence[str],

553 remote: str = DEFAULT_REMOTE,

554 ref: str | None = None,

555 image: str = DEFAULT_DEV_IMAGE,

556 skip_fetch: bool = False,

557) -> UpgradePlan:

558 """Inspect the checkout, fetch the release tags, and lay out the upgrade."""

559 checkout = inspect_checkout(root)

560 if skip_fetch:

561 tags = parse_release_tags(run_git(root, "tag", "--list", "v*").splitlines())

562 else:

563 tags = fetch_release_tags(root, remote)

564 target = resolve_target(tags, ref)

565 latest = latest_release(tags)

566 control_plane, workload = split_stacks(stacks, project_name)

567 return UpgradePlan(

568 current_version=__version__,

569 target=target,

570 latest=latest,

571 checkout=checkout,

572 install=probe_install(root, image=image),

573 already_at_target=checkout_is_at(root, target),

574 control_plane_stacks=control_plane,

575 workload_stacks=workload,

576 remote=remote,

577 )

578

579

580def require_container_runtime() -> str:

581 """Deploys build Lambda assets and mirror images; refuse early when nothing can."""

582 runtime = detect_container_runtime()

583 if not runtime:

584 raise UpgradeError(container_runtime_error_message())

585 return runtime

586

587

588@dataclass

589class StackCycleResult:

590 """Outcome of the teardown/redeploy half of the upgrade."""

591

592 destroyed: list[str] = field(default_factory=list)

593 deployed: list[str] = field(default_factory=list)

594 failed: list[str] = field(default_factory=list)

595 teardown_attempts: int = 0

596 phase_failed: str | None = None

597

598 @property

599 def ok(self) -> bool:

600 return self.phase_failed is None

601

602 def to_dict(self) -> dict[str, Any]:

603 data = asdict(self)

604 data["ok"] = self.ok

605 return data

606

607

608def run_stack_cycle(

609 manager: Any,

610 *,

611 parallel: bool = False,

612 max_workers: int = 4,

613 on_stack_start: Callable[[str], None] | None = None,

614 on_stack_complete: Callable[[str, bool], None] | None = None,

615 log: Logger = _quiet,

616 sleep: Callable[[float], None] | None = None,

617 max_attempts: int = 3,

618 retry_wait_seconds: float = 30.0,

619) -> StackCycleResult:

620 """Scale the workload tier to zero, then deploy-all from the new release.

621

622 The checkout's deployable sources are checked first

623 (:meth:`cli.stacks.StackManager.ensure_asset_sources_readable`): the

624 redeploy refuses owner-only sources, so a checkout it could not deploy —

625 for example one whose files the upgrade's own ``git checkout`` wrote under

626 a restrictive umask — must stop the upgrade before the teardown, not

627 after it. The teardown's monitoring detach would catch it too, but only

628 when the monitoring stack is deployed.

629

630 The teardown uses the same retry loop as ``gco stacks destroy-all``: a

631 failed attempt clears the orphaned network interfaces that typically block

632 VPC deletion, waits, and tries again. The redeploy is a single

633 ``deploy_orchestrated`` pass — global and API Gateway stacks update in

634 place, regional stacks and their bridges are recreated, monitoring is

635 updated back to the full topology.

636

637 Raises:

638 cli.stacks.AssetPermissionError: before any stack is touched, when a

639 deployable source is owner-only.

640 """

641 import time

642

643 wait = sleep or time.sleep

644 result = StackCycleResult()

645

646 manager.ensure_asset_sources_readable()

647

648 log("Phase 1/2: scaling the workload tier to zero (control plane stays)...")

649 ok = False

650 failed: list[str] = []

651 for attempt in range(1, max_attempts + 1):

652 result.teardown_attempts = attempt

653 if attempt > 1:

654 log("Clearing orphaned network interfaces that can block VPC deletion...")

655 manager.cleanup_orphaned_network_interfaces()

656 log(f"Teardown attempt {attempt}/{max_attempts} in {retry_wait_seconds:.0f}s...")

657 wait(retry_wait_seconds)

658 ok, destroyed, failed = manager.destroy_orchestrated(

659 force=True,

660 keep_control_plane=True,

661 parallel=parallel,

662 max_workers=max_workers,

663 on_stack_start=on_stack_start,

664 on_stack_complete=on_stack_complete,

665 )

666 for stack in destroyed:

667 if stack not in result.destroyed:

668 result.destroyed.append(stack)

669 if ok:

670 break

671 if not ok:

672 result.failed = list(failed)

673 result.phase_failed = "teardown"

674 return result

675

676 log("Phase 2/2: updating the control plane in place and recreating the regional stacks...")

677 ok, deployed, failed = manager.deploy_orchestrated(

678 require_approval=False,

679 parallel=parallel,

680 max_workers=max_workers,

681 on_stack_start=on_stack_start,

682 on_stack_complete=on_stack_complete,

683 )

684 result.deployed = list(deployed)

685 if not ok:

686 result.failed = list(failed)

687 result.phase_failed = "deploy"

688 return result

Coverage for cli / upgrade.py: 100.00%

297 statements