Coverage for cli/commands/upgrade

1"""``gco upgrade`` — move a whole GCO deployment to the latest tagged release.

3Thin Click veneer over :mod:`cli.upgrade`. The command's job is to make the

4blast radius unmistakable before anything happens: it prints the release it

5will move to, what happens to the checkout and the local install, and — in

6capitals — that every regional stack and the data inside it is destroyed and

7recreated. The typed project-name confirmation (skipped by ``-y``) exists

8because the command deletes data in every workload Region.

9"""

11from __future__ import annotations

13import sys

14from collections.abc import Mapping

15from typing import Any

17import click

19from ..config import GCOConfig

20from ..output import get_output_formatter, interactive_echo, prompt

21from ..upgrade import (

22 DEFAULT_DEV_IMAGE,

23 DEFAULT_REMOTE,

24 UpgradeError,

25 UpgradePlan,

26 build_plan,

27 checkout_release,

28 find_checkout_root,

29 rebuild_dev_image,

30 refresh_node_toolchain,

31 refresh_python_install,

32 require_container_runtime,

33 run_stack_cycle,

34)

35from .stacks_cmd import _ENABLE_OPTION_HELP, _apply_enable_overrides, _validate_enable

37pass_config = click.make_pass_decorator(GCOConfig, ensure=True)

39#: Reminder printed with the plan and again after a successful run.

40_DATA_WARNING = (

41 "Every regional stack is DESTROYED and recreated. EFS and FSx file systems, Valkey, "

42 "Aurora, in-cluster volumes (Prometheus, Grafana, MLflow) and, unless retained, the "

43 "regional-shared bucket in those Regions are deleted with them. Back that data up to "

44 "the cluster-shared bucket first and restore it afterwards — see docs/UPGRADING.md."

45)

48def _describe_install_steps(

49 plan: UpgradePlan, *, skip_checkout: bool, skip_container: bool

50) -> list[str]:

51 """Human lines for the local half of the plan."""

52 lines: list[str] = []

53 if skip_checkout:

54 lines.append(

55 f"Keep the current checkout ({plan.checkout.ref} at {plan.checkout.head}, "

56 f"VERSION {plan.checkout.version_file or 'unknown'}) — --skip-checkout"

57 )

58 elif plan.already_at_target:

59 lines.append(

60 f"Checkout is already at {plan.target.name} ({plan.checkout.head}); nothing to fetch"

61 )

62 else:

63 lines.append(

64 f"git checkout --detach {plan.target.name} in {plan.checkout.root} "

65 f"(currently {plan.checkout.ref} at {plan.checkout.head}); cdk.json is preserved byte-for-byte"

66 )

67 if plan.install.cli_editable_from_checkout:

68 lines.append(

69 "Refresh the editable CLI install with the cdk extra and every extra already "

70 "installed (pip install -e '.[cdk,...]')"

71 )

72 else:

73 lines.append(

74 f"The running gco ({plan.current_version} from {plan.install.cli_path}) is not the "

75 "editable install of this checkout — reinstall it yourself after the upgrade"

76 )

77 if plan.install.node_toolchain:

78 lines.append("Refresh the checkout's CDK toolchain (npm ci)")

79 if skip_container:

80 lines.append("Leave the dev-container image alone — --skip-container")

81 elif plan.install.dev_image_present:

82 lines.append(

83 f"Rebuild the {plan.install.dev_image} container image with {plan.install.container_runtime}"

84 )

85 else:

86 lines.append(f"No local {plan.install.dev_image} image to rebuild")

87 return lines

90def _render_plan(

91 plan: UpgradePlan,

92 *,

93 skip_checkout: bool,

94 skip_container: bool,

95 echo: Any,

96) -> None:

97 """Print the plan through ``echo`` (stdout in table mode, stderr in machine modes)."""

98 echo(f"gco upgrade — {plan.current_version} → {plan.target.name} (latest: {plan.latest.name})")

99 echo("")

100 echo("Local install:")

101 for line in _describe_install_steps(

102 plan, skip_checkout=skip_checkout, skip_container=skip_container

103 ):

104 echo(f" - {line}")

105 echo("")

106 echo("Stacks (in order):")

107 echo(" 1. DESTROY the workload tier (regional API bridges, then regional stacks):")

108 if plan.workload_stacks:

109 for stack in plan.workload_stacks:

110 echo(f" - {stack}")

111 else:

112 echo(" (no workload Regions are configured — nothing to destroy)")

113 echo(" 2. Update the control-plane stacks in place: " + ", ".join(plan.control_plane_stacks))

114 echo(" 3. Recreate the workload tier from cdk.json on the new release")

115 echo("")

116 echo(f"⚠ {_DATA_WARNING}")

117 echo(

118 "Expect 45-90 minutes for a single Region; the command is safe to rerun with --skip-checkout."

119 )

120

121

122@click.command("upgrade")

123@click.option("--check", is_flag=True, help="Show the plan and the latest release; change nothing")

124@click.option("--yes", "-y", is_flag=True, help="Skip the typed confirmation")

125@click.option("--ref", metavar="vX.Y.Z", help="Upgrade to this release tag instead of the latest")

126@click.option(

127 "--remote",

128 default=DEFAULT_REMOTE,

129 show_default=True,

130 help="Git remote whose tags define the releases",

131)

132@click.option(

133 "--skip-checkout",

134 is_flag=True,

135 help="Keep the current checkout and toolchain; only cycle the stacks",

136)

137@click.option("--skip-container", is_flag=True, help="Do not rebuild the dev-container image")

138@click.option(

139 "--image",

140 default=DEFAULT_DEV_IMAGE,

141 show_default=True,

142 help="Dev-container image to rebuild when it exists locally",

143)

144@click.option(

145 "--force",

146 is_flag=True,

147 help="Cycle the stacks even when the checkout is already at the target release",

148)

149@click.option(

150 "--parallel", "-p", is_flag=True, help="Destroy and recreate regional stacks in parallel"

151)

152@click.option("--max-workers", "-w", default=4, help="Max parallel stack operations (default: 4)")

153@click.option(

154 "--enable",

155 "enable",

156 multiple=True,

157 metavar="NAME[,NAME...]",

158 callback=_validate_enable,

159 help=_ENABLE_OPTION_HELP,

160)

161@pass_config

162def upgrade(

163 config: Any,

164 check: Any,

165 yes: Any,

166 ref: Any,

167 remote: Any,

168 skip_checkout: Any,

169 skip_container: Any,

170 image: Any,

171 force: Any,

172 parallel: Any,

173 max_workers: Any,

174 enable: Mapping[str, str],

175) -> None:

176 """Upgrade this GCO deployment to the latest tagged release.

177

178 Runs inside a git checkout of the repository and, in one pass:

179

180 \b

181 1. fetches the release tags and checks out the latest vMAJOR.MINOR.PATCH

182 (cdk.json — your deployment's configuration — is preserved as-is);

183 2. refreshes the local install: pip install -e '.[cdk,...]' (the cdk

184 extra plus every extra already installed) for an editable CLI,

185 npm ci for a checkout-local CDK toolchain, and a rebuild of the

186 gco-dev container image when it exists;

187 3. checks that the checkout's deployable sources are readable by every

188 user (stopping, with no stack touched, when they are not), then

189 scales the workload tier to zero — the monitoring stack is updated

190 in place so it stops referencing the regional stacks, then every

191 regional API bridge and regional stack is DESTROYED;

192 4. updates the global, API Gateway and monitoring stacks in place and

193 recreates the regional stacks from cdk.json on the new release.

194

195 Step 3 deletes the data inside the regional stacks (EFS and FSx file

196 systems, Valkey, Aurora, in-cluster volumes, the regional-shared bucket

197 unless retained). Back it up to the cluster-shared bucket first and

198 restore it afterwards: docs/UPGRADING.md walks through the procedure.

199 The shared state in the control plane — DynamoDB tables, the model and

200 cluster-shared buckets, the image registry, EFS recovery points, cost

201 reports — is untouched.

202

203 --check prints the plan without changing anything. --skip-checkout reruns

204 only the stack cycle (to finish an interrupted upgrade, or on a fork that

205 merged the release itself); --force does the same when the checkout is

206 already at the latest release.

207

208 \b

209 Examples:

210 gco upgrade --check

211 gco upgrade

212 gco upgrade -y --parallel

213 gco upgrade --ref v8.1.0

214 gco upgrade --skip-checkout -y

215 """

216 from ..stacks import AssetPermissionError, get_stack_manager

217

218 formatter = get_output_formatter(config)

219 machine = config.output_format != "table"

220

221 try:

222 root = find_checkout_root()

223 manager = get_stack_manager(config)

224 _apply_enable_overrides(formatter, manager, enable)

225 stacks = manager.list_stacks()

226 plan = build_plan(

227 root,

228 project_name=config.project_name,

229 stacks=stacks,

230 remote=remote,

231 ref=ref,

232 image=image,

233 skip_fetch=skip_checkout,

234 )

235 except UpgradeError as e:

236 formatter.print_error(str(e))

237 sys.exit(1)

238 except Exception as e:

239 formatter.print_error(f"Could not prepare the upgrade: {e}")

240 sys.exit(1)

241

242 if check:

243 if machine:

244 formatter.print(

245 {

246 "status": "ok",

247 "up_to_date": plan.already_at_target,

248 "plan": plan.to_dict(),

249 }

250 )

251 else:

252 _render_plan(

253 plan,

254 skip_checkout=skip_checkout,

255 skip_container=skip_container,

256 echo=formatter.print_info,

257 )

258 if plan.already_at_target:

259 formatter.print_success(

260 f"Checkout is already at {plan.target.name}; --check made no changes"

261 )

262 else:

263 formatter.print_info(

264 f"Run 'gco upgrade' to move to {plan.target.name}; --check made no changes"

265 )

266 return

267

268 if plan.already_at_target and not force and not skip_checkout:

269 if machine:

270 formatter.print({"status": "up-to-date", "plan": plan.to_dict()})

271 else:

272 formatter.print_success(

273 f"Already at {plan.target.name} — nothing to upgrade. Use --force to run the "

274 "stack cycle anyway (for example to finish an interrupted upgrade)."

275 )

276 return

277

278 if not yes:

279 _render_plan(

280 plan,

281 skip_checkout=skip_checkout,

282 skip_container=skip_container,

283 echo=interactive_echo,

284 )

285 answer = prompt(

286 f"\nType the project name ({config.project_name}) to confirm",

287 default="",

288 show_default=False,

289 )

290 if str(answer).strip() != config.project_name:

291 formatter.print_error(

292 "Confirmation did not match the project name; nothing was changed."

293 )

294 sys.exit(1)

295

296 result: dict[str, Any] = {"status": "ok", "plan": plan.to_dict(), "steps": {}}

297 steps = result["steps"]

298 cycle_started = False

299 try:

300 runtime = require_container_runtime()

301

302 if skip_checkout:

303 steps["checkout"] = {"status": "skipped", "reason": "--skip-checkout"}

304 else:

305 if plan.already_at_target:

306 steps["checkout"] = {"status": "already-at-target", "checked_out": plan.target.name}

307 else:

308 steps["checkout"] = checkout_release(root, plan.target, log=formatter.print_info)

309 if plan.install.cli_editable_from_checkout:

310 steps["python"] = refresh_python_install(root, log=formatter.print_info)

311 else:

312 steps["python"] = {

313 "status": "skipped",

314 "reason": "the running gco is not the editable install of this checkout",

315 }

316 formatter.print_warning(

317 f"The running gco ({plan.current_version}) is installed from "

318 f"{plan.install.cli_path}, not from this checkout; reinstall it from "

319 f"{plan.target.name} yourself once the upgrade finishes."

320 )

321 if plan.install.node_toolchain:

322 steps["node"] = refresh_node_toolchain(root, log=formatter.print_info)

323 if steps["node"].get("status") == "warning":

324 formatter.print_warning(steps["node"]["message"])

325

326 if skip_container:

327 steps["container"] = {"status": "skipped", "reason": "--skip-container"}

328 elif plan.install.dev_image_present:

329 steps["container"] = rebuild_dev_image(

330 root, runtime=runtime, image=image, log=formatter.print_info

331 )

332 if steps["container"].get("status") == "warning":

333 formatter.print_warning(steps["container"]["message"])

334 else:

335 steps["container"] = {

336 "status": "skipped",

337 "reason": f"image {image} is not present locally",

338 }

339

340 # A fresh manager: the checkout may have moved, and the CDK app it

341 # synthesizes is now the new release's. Run-scoped --enable overrides

342 # ride along exactly as they did for the plan.

343 manager = get_stack_manager(config)

344 if enable:

345 manager.set_extra_cdk_context(dict(enable))

346

347 def on_start(stack_name: str) -> None:

348 formatter.print_info(f" {stack_name}...")

349

350 def on_complete(stack_name: str, success: bool) -> None:

351 if success:

352 formatter.print_success(f" ✓ {stack_name}")

353 else:

354 formatter.print_error(f" ✗ {stack_name} failed")

355

356 cycle_started = True

357 cycle = run_stack_cycle(

358 manager,

359 parallel=parallel,

360 max_workers=max_workers,

361 on_stack_start=on_start,

362 on_stack_complete=on_complete,

363 log=formatter.print_info,

364 )

365 steps["stacks"] = cycle.to_dict()

366 except AssetPermissionError as e:

367 # run_stack_cycle checks this before its teardown, so no stack has been

368 # touched; the checkout and local install have already moved.

369 formatter.print_error(str(e))

370 formatter.print_error(

371 "No stack was changed. Once the modes are fixed, run "

372 "'gco upgrade --skip-checkout' to cycle the stacks on this checkout."

373 )

374 sys.exit(1)

375 except UpgradeError as e:

376 formatter.print_error(str(e))

377 sys.exit(1)

378 except Exception as e:

379 if cycle_started:

380 reason = str(e).rstrip(".")

381 formatter.print_error(

382 f"Upgrade failed during the stack cycle: {reason}. Stacks may be partly "

383 "torn down or recreated; fix the cause and rerun "

384 "'gco upgrade --skip-checkout' to finish the cycle (the teardown half is "

385 "idempotent)."

386 )

387 else:

388 formatter.print_error(f"Upgrade failed: {e}")

389 sys.exit(1)

390

391 if not cycle.ok:

392 formatter.print_error(

393 f"Upgrade stopped during the {cycle.phase_failed} phase; failed stacks: "

394 f"{', '.join(cycle.failed) or 'none reported'}. The checkout and local install are "

395 f"already on {plan.target.name}; fix the cause and rerun 'gco upgrade --skip-checkout' "

396 "to finish the stack cycle."

397 )

398 sys.exit(1)

399

400 if machine:

401 formatter.print(result)

402 return

403

404 formatter.print_success(f"Upgrade to {plan.target.name} complete.")

405 formatter.print_info(f" destroyed and recreated: {', '.join(cycle.destroyed) or 'none'}")

406 formatter.print_info(f" deployed: {', '.join(cycle.deployed) or 'none'}")

407 formatter.print_info(

408 "Restore any data you staged from the regional stacks now (see docs/UPGRADING.md). "

409 "Open a new shell so `gco --version` reports the new release."

410 )

Coverage for cli / commands / upgrade_cmd.py: 100.00%

155 statements