Coverage for gco / slurm_config.py: 100.00%

1"""Schema, defaults and rendering for Slurm on GCO (``helm.slurm``). 

2 

3GCO runs Slurm on each regional EKS cluster through the Slinky slurm-operator 

4when ``cdk.json`` ``helm.slurm.enabled`` is true (off by default; 

5``helm_enabled_overrides=slurm`` forces it on for one deploy). The block is 

6the whole description of the cluster an operator gets, the Slurm equivalent of 

7an AWS PCS cluster definition: 

8 

9- ``partitions`` are the compute node groups. Each one becomes a Slinky 

10 ``NodeSet`` (a pool of identically sized ``slurmd`` pods scheduled onto one 

11 of GCO's Karpenter NodePools) and the Slurm partition that spans it. Pod 

12 size is node size: Slinky sizes the Slurm node from the pod's resources and 

13 Karpenter picks an EC2 instance that fits it, so ``cpu``/``memory``/``gpus`` 

14 stand in for PCS's instance type. 

15- ``min_nodes``/``max_nodes`` are the scaling bounds. A partition whose bounds 

16 differ is autoscaled on Slurm's own queue metrics by a KEDA ``ScaledObject`` 

17 the chart renders through ``extraObjects`` (:func:`slurm_chart_values`), 

18 scaling the NodeSet to zero when idle; Karpenter then consolidates the 

19 nodes. 

20- ``accounting`` is slurmdbd with an in-cluster MariaDB (or an external 

21 database the operator already runs), ``metrics`` is slurmctld's native 

22 OpenMetrics endpoint, ``login`` is the sackd login pod ``gco slurm shell`` 

23 execs into, and ``slurm_conf``/``slurmdbd_conf`` pass extra configuration 

24 through to the daemons. 

25 

26This module owns the block's shape so the config loader, the regional stack, 

27the kubectl-applier manifests, the ``gco slurm`` CLI and the live validation 

28harness agree on names and defaults. Like :mod:`gco.argocd_config` it imports 

29nothing from ``aws_cdk``: the CLI runs without the CDK toolchain installed. It 

30raises :class:`SlurmConfigError` (a ``ValueError``); the loader re-raises with 

31its own exception type. 

32""" 

33 

34from __future__ import annotations 

35 

36import copy 

37import re 

38from collections.abc import Iterable, Mapping 

39from typing import Any 

40 

41from gco.resource_governance import parse_k8s_quantity 

42 

43# --------------------------------------------------------------------------- 

44# Fixed names shared by the chart values, the manifests, the API and the CLI 

45# --------------------------------------------------------------------------- 

46 

47#: ``cdk.json`` ``helm`` block key (and ``helm_enabled_overrides`` name). 

48SLURM_HELM_KEY = "slurm" 

49 

50#: charts.yaml entries (the Helm release names). 

51SLURM_OPERATOR_CHART_NAME = "slinky-slurm-operator" 

52SLURM_CLUSTER_CHART_NAME = "slinky-slurm" 

53 

54#: The Slurm cluster lives with the user workloads; the operator has its own 

55#: namespace (charts.yaml ``slinky-slurm-operator.namespace``). 

56SLURM_NAMESPACE = "gco-jobs" 

57SLURM_OPERATOR_NAMESPACE = "slurm-operator" 

58 

59#: Objects the ``slurm`` chart and the operator derive from the release name 

60#: ``slinky-slurm``: the Controller CR is the release name, its StatefulSet and 

61#: Service are ``<release>-controller``, the RestApi Deployment and Service are 

62#: ``<release>-restapi``, NodeSets are ``<release>-worker-<key>`` and the 

63#: LoginSet Deployment/Service is ``<release>-login-<key>``. The two auth 

64#: Secrets hold the cluster key (``slurm.key``) and the JWT signing key 

65#: (``jwt.key``). 

66SLURM_CONTROLLER_NAME = "slinky-slurm-controller" 

67SLURM_RESTAPI_NAME = "slinky-slurm-restapi" 

68SLURM_ACCOUNTING_NAME = "slinky-slurm-accounting" 

69SLURM_LOGIN_SET_KEY = "gco" 

70SLURM_LOGIN_NAME = f"{SLURM_CLUSTER_CHART_NAME}-login-{SLURM_LOGIN_SET_KEY}" 

71SLURM_JWT_SECRET = "slinky-slurm-auth-jwt" # nosec B105 # Kubernetes Secret name, not its contents 

72SLURM_KEY_SECRET = "slinky-slurm-auth-slurm" # nosec B105 # Kubernetes Secret name, not its contents 

73 

74#: Daemon ports (Slurm defaults; the operator does not change them). slurmctld 

75#: also serves the OpenMetrics endpoints on its own port. 

76SLURM_CONTROLLER_PORT = 6817 

77SLURM_SLURMD_PORT = 6818 

78SLURM_SLURMDBD_PORT = 6819 

79SLURM_RESTAPI_PORT = 6820 

80 

81#: Pod labels the operator stamps (``app.kubernetes.io/name`` per component; 

82#: every Slurm pod also carries the chart's ``app.kubernetes.io/part-of: slurm``). 

83SLURM_COMPONENT_LABEL = "app.kubernetes.io/name" 

84SLURM_CONTROLLER_COMPONENT = "slurmctld" 

85SLURM_RESTAPI_COMPONENT = "slurmrestd" 

86SLURM_WORKER_COMPONENT = "slurmd" 

87SLURM_ACCOUNTING_COMPONENT = "slurmdbd" 

88SLURM_LOGIN_COMPONENT = "login" 

89SLURM_PART_OF_LABEL = "app.kubernetes.io/part-of" 

90SLURM_PART_OF_VALUE = "slurm" 

91 

92#: The in-cluster accounting database (29-slurm-accounting-db.yaml): 

93#: one StatefulSet, Service and generated-password Secret of this name, and 

94#: the slurmdbd connection constants the chart values point at it with. 

95SLURM_ACCOUNTING_DB_NAME = "slinky-slurm-mariadb" 

96SLURM_ACCOUNTING_DB_SECRET = SLURM_ACCOUNTING_DB_NAME # nosec B105 # Kubernetes Secret name 

97SLURM_ACCOUNTING_DB_PASSWORD_KEY = "password" # nosec B105 # Secret data key, not a password 

98SLURM_ACCOUNTING_DB_ROOT_PASSWORD_KEY = "root-password" # nosec B105 # Secret data key 

99SLURM_ACCOUNTING_DB_PORT = 3306 

100SLURM_ACCOUNTING_DATABASE = "slurm_acct_db" 

101SLURM_ACCOUNTING_USER = "slurm" 

102 

103#: Where the accounting backup CronJob writes its dumps in the regional shared 

104#: bucket (``<prefix><cluster name>/<UTC timestamp>.sql.gz``); the regional 

105#: stack's lifecycle rule expires objects under this prefix. 

106SLURM_ACCOUNTING_BACKUP_PREFIX = "slurm/accounting-backups/" 

107 

108#: Block storage for slurmctld's state directory and the accounting database: 

109#: a gp3 StorageClass shipped by the gated base manifest 28-storage-slurm-gp3.yaml. 

110SLURM_STORAGE_CLASS = "gco-slurm-gp3" 

111 

112#: Shared filesystems mounted into worker and login pods. The EFS claim is the 

113#: always-present ``gco-jobs`` claim; ``/home`` is a subdirectory of it so 

114#: Slurm home directories never mix with job output written to the claim root. 

115#: The FSx claim exists only when ``fsx_lustre`` is enabled. 

116SLURM_EFS_CLAIM = "gco-shared-storage" 

117SLURM_HOME_SUBPATH = "slurm/home" 

118SLURM_HOME_MOUNT_PATH = "/home" 

119SLURM_FSX_CLAIM = "gco-fsx-storage" 

120SLURM_SCRATCH_MOUNT_PATH = "/scratch" 

121 

122#: The control-plane pods (slurmctld, slurmrestd, slurmdbd) run at the 

123#: platform priority (05-priority-classes.yaml) so they preempt user pods 

124#: under pressure instead of being starved; workers deliberately do not. 

125SLURM_PLATFORM_PRIORITY_CLASS = "gco-platform-critical" 

126 

127#: Where the KEDA Prometheus scaler reads Slurm's metrics (kube-prometheus-stack). 

128SLURM_PROMETHEUS_URL = "http://kube-prometheus-stack-prometheus.monitoring.svc.cluster.local:9090" 

129 

130#: The Karpenter NodePools a partition may target (lambda/kubectl-applier-simple/ 

131#: manifests/4x-nodepool-*.yaml): the node label that selects the pool, the 

132#: taints its nodes carry (the slurmd pods tolerate exactly those), and what 

133#: accelerators its instances have. ``neuron`` is deliberately absent: Slurm's 

134#: gres plugin has no Trainium support. 

135NODE_POOLS: dict[str, dict[str, Any]] = { 

136 "cpu": {"selector": {"workload-type": "cpu"}, "taints": (), "gpu": False, "efa": False}, 

137 "gpu": { 

138 "selector": {"workload-type": "gpu"}, 

139 "taints": ("nvidia.com/gpu",), 

140 "gpu": True, 

141 "efa": False, 

142 }, 

143 "gpu-efa": { 

144 "selector": {"workload-type": "gpu-efa"}, 

145 "taints": ("nvidia.com/gpu", "vpc.amazonaws.com/efa"), 

146 "gpu": True, 

147 "efa": True, 

148 }, 

149} 

150 

151#: ``karpenter.sh/capacity-type`` choices; ``any`` leaves the pool's own mix. 

152CAPACITY_TYPES: tuple[str, ...] = ("any", "spot", "on-demand") 

153CAPACITY_TYPE_LABEL = "karpenter.sh/capacity-type" 

154 

155#: Slurm's ``MaxNodeCount`` as the operator configures it; a NodeSet cannot 

156#: usefully grow past it. 

157MAX_NODES_LIMIT = 1024 

158 

159#: Partition names double as NodeSet keys: the operator names pods 

160#: ``<name>-<ordinal>`` and the NodeSet ``slinky-slurm-worker-<name>``, so the 

161#: name must be a DNS label short enough for both. 

162PARTITION_NAME_PATTERN = re.compile(r"^[a-z][a-z0-9-]{0,30}[a-z0-9]$|^[a-z]$") 

163#: The chart's own ``partitions.all`` key (GCO disables it, see values). 

164RESERVED_PARTITION_NAMES: frozenset[str] = frozenset({"all"}) 

165 

166#: Slurm time limits: minutes, min:sec, hours:min:sec, days-hours, 

167#: days-hours:min, days-hours:min:sec, or unlimited. 

168SLURM_TIME_PATTERN = re.compile( 

169 r"^(UNLIMITED|INFINITE|\d+|\d+:\d{2}|\d+:\d{2}:\d{2}|\d+-\d{1,2}|\d+-\d{1,2}:\d{2}|\d+-\d{1,2}:\d{2}:\d{2})$" 

170) 

171 

172#: Kubernetes label values: ``features`` become Slurm node Features and are 

173#: also the only free-form text that reaches slurm.conf from a partition. 

174FEATURE_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,62}$") 

175 

176#: Slurm ``ClusterName``: letters, digits, ``_`` and ``-``; slurmdbd uses it 

177#: as a table-name prefix, hence the conservative character set and length. 

178CLUSTER_NAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,62}$") 

179#: Keys an operator may set in ``slurm_conf`` / ``slurmdbd_conf``. 

180CONF_KEY_PATTERN = re.compile(r"^[A-Za-z][A-Za-z0-9]{0,63}$") 

181 

182#: slurm.conf keys the operator writes itself; overriding them through 

183#: ``extraConf`` either breaks the operator's reconciliation (auth, ports, 

184#: hosts) or duplicates a key Slurm rejects. The webhook rejects whole files 

185#: named slurm.conf/slurmdbd.conf; this is the per-key equivalent. 

186OPERATOR_OWNED_SLURM_CONF_KEYS: frozenset[str] = frozenset( 

187 { 

188 "ClusterName", 

189 "SlurmUser", 

190 "SlurmdUser", 

191 "SlurmctldHost", 

192 "SlurmctldPort", 

193 "SlurmdPort", 

194 "SlurmdSpoolDir", 

195 "StateSaveLocation", 

196 "SlurmctldLogFile", 

197 "SlurmSchedLogFile", 

198 "SlurmdLogFile", 

199 "AuthType", 

200 "CredType", 

201 "AuthAltTypes", 

202 "MetricsType", 

203 "AccountingStorageType", 

204 "AccountingStorageHost", 

205 "AccountingStoragePort", 

206 "Include", 

207 "NodeName", 

208 "NodeSet", 

209 "PartitionName", 

210 "Prolog", 

211 "Epilog", 

212 "PrologSlurmctld", 

213 "EpilogSlurmctld", 

214 } 

215) 

216OPERATOR_OWNED_SLURMDBD_CONF_KEYS: frozenset[str] = frozenset( 

217 { 

218 "DbdHost", 

219 "DbdPort", 

220 "SlurmUser", 

221 "StorageType", 

222 "StorageHost", 

223 "StoragePort", 

224 "StorageLoc", 

225 "StorageUser", 

226 "StoragePass", 

227 "AuthType", 

228 "AuthAltTypes", 

229 "AuthAltParameters", 

230 "LogFile", 

231 "PidFile", 

232 } 

233) 

234 

235#: Per-partition defaults for keys an operator leaves out. 

236PARTITION_DEFAULTS: dict[str, Any] = { 

237 "enabled": True, 

238 "node_pool": "cpu", 

239 "cpu": "4", 

240 "memory": "16Gi", 

241 "gpus": 0, 

242 "efa_interfaces": 0, 

243 "capacity_type": "any", 

244 "min_nodes": 0, 

245 "max_nodes": 4, 

246 "default": False, 

247 "max_time": "UNLIMITED", 

248 "exclusive": False, 

249 "features": [], 

250} 

251 

252#: The block with every knob at its default. Slurm is off; when it is on, the 

253#: shipped partitions give a CPU partition and a single-GPU partition that 

254#: both scale from zero, plus a disabled template for multi-node EFA training 

255#: (p5-class: eight GPUs, 32 EFA interfaces, whole-node exclusive jobs). 

256SLURM_DEFAULTS: dict[str, Any] = { 

257 "enabled": False, 

258 # Slurm's ClusterName; also the accounting cluster record. 

259 "cluster_name": "gco-slurm", 

260 "controller": { 

261 # slurmctld StateSaveLocation: a dedicated gp3 volume (the chart 

262 # creates the claim on SLURM_STORAGE_CLASS). 

263 "state_storage_size": "20Gi", 

264 }, 

265 "restapi": { 

266 # Two slurmrestd replicas behind one Service, with a PDB keeping one 

267 # up through node drains: the GCO API's path into Slurm. 

268 "replicas": 2, 

269 }, 

270 "accounting": { 

271 "enabled": True, 

272 # In-cluster MariaDB (29-slurm-accounting-db.yaml). 

273 "storage_size": "20Gi", 

274 "backup": { 

275 # Nightly mariadb-dump of slurm_acct_db to the regional shared 

276 # bucket under slurm/accounting-backups/<cluster>/. 

277 "enabled": True, 

278 "schedule": "0 3 * * *", 

279 "retention_days": 30, 

280 }, 

281 # Point slurmdbd at a database the operator already runs (RDS for 

282 # MariaDB, Aurora MySQL, ...) instead of the in-cluster one. host set 

283 # = external; password_secret names a Secret in gco-jobs carrying the 

284 # password under the given key. 

285 "external": { 

286 "host": "", 

287 "port": SLURM_ACCOUNTING_DB_PORT, 

288 "database": SLURM_ACCOUNTING_DATABASE, 

289 "username": SLURM_ACCOUNTING_USER, 

290 "password_secret": "", # nosec B105 # a Secret *name*; empty = in-cluster database 

291 "password_key": SLURM_ACCOUNTING_DB_PASSWORD_KEY, 

292 }, 

293 }, 

294 "metrics": { 

295 # slurmctld OpenMetrics (MetricsType=metrics/openmetrics); scraped by 

296 # the gco-slurm-controller ServiceMonitor when observability is on and 

297 # read by the autoscaler. Only turn off for a cluster that has no 

298 # autoscaled partition. 

299 "enabled": True, 

300 }, 

301 "login": { 

302 # One sackd login pod (ClusterIP, no load balancer): the target of 

303 # `gco slurm shell`, with /home mounted like the workers. 

304 "enabled": True, 

305 "replicas": 1, 

306 }, 

307 "autoscaling": { 

308 # Pending jobs per additional node (KEDA threshold for the pending-jobs 

309 # trigger; 1 = one node per pending job). 

310 "pending_jobs_threshold": 1, 

311 # Idle time before an autoscaled partition is scaled to min_nodes (and 

312 # to zero when min_nodes is 0). 

313 "cooldown_seconds": 300, 

314 "polling_interval_seconds": 30, 

315 }, 

316 "partitions": { 

317 "cpu": { 

318 "node_pool": "cpu", 

319 "cpu": "8", 

320 "memory": "32Gi", 

321 "gpus": 0, 

322 "capacity_type": "any", 

323 "min_nodes": 0, 

324 "max_nodes": 8, 

325 "default": True, 

326 "max_time": "UNLIMITED", 

327 "exclusive": False, 

328 }, 

329 "gpu": { 

330 "node_pool": "gpu", 

331 "cpu": "8", 

332 "memory": "32Gi", 

333 "gpus": 1, 

334 "capacity_type": "any", 

335 "min_nodes": 0, 

336 "max_nodes": 4, 

337 "default": False, 

338 "max_time": "2-00:00:00", 

339 "exclusive": True, 

340 }, 

341 "gpu-efa": { 

342 "enabled": False, 

343 "node_pool": "gpu-efa", 

344 "cpu": "180", 

345 "memory": "1800Gi", 

346 "gpus": 8, 

347 "efa_interfaces": 32, 

348 "capacity_type": "on-demand", 

349 "min_nodes": 0, 

350 "max_nodes": 2, 

351 "default": False, 

352 "max_time": "7-00:00:00", 

353 "exclusive": True, 

354 }, 

355 }, 

356 "storage": { 

357 # /home for workers and login pods: "efs" (the shared claim) or "none". 

358 "home": "efs", 

359 # /scratch: "auto" (FSx Lustre when fsx_lustre is enabled), "fsx" 

360 # (require it) or "none". 

361 "scratch": "auto", 

362 }, 

363 # Extra slurm.conf / slurmdbd.conf lines (key -> value or list of values). 

364 "slurm_conf": {}, 

365 "slurmdbd_conf": {}, 

366} 

367 

368#: Image pins GCO keeps in lockstep with the chart version in charts.yaml. 

369#: The Slinky sidecars default to ``alpine:latest`` upstream; GCO pins them. 

370#: ``tests/test_helm_charts_validation.py`` holds every charts.yaml image tag 

371#: to these two constants. 

372SLURM_IMAGE_TAG = "26.05.5-ubuntu24.04" 

373SLURM_SIDECAR_IMAGE = {"repository": "public.ecr.aws/docker/library/alpine", "tag": "3.24.3"} 

374 

375#: The slurmrestd OpenAPI plugin version every GCO surface addresses 

376#: (``/slurm/<version>/...``): the API client, the example job, the live 

377#: validation probe and the API doc. It is a property of the Slurm release 

378#: behind ``SLURM_IMAGE_TAG``: each release ships its newest ``data_parser`` 

379#: plugin plus the three before it and drops the oldest, so a tag bump must 

380#: re-check this pin. ``validate_helm_charts.py --mode online`` does that in 

381#: CI by running ``slurmrestd -d list`` inside the pinned image; the offline 

382#: half holds the other surfaces to this constant. Request and response 

383#: bodies differ between plugin versions, which is why the client is written 

384#: against exactly one instead of negotiating at runtime. 

385SLURM_API_VERSION = "v0.0.44" 

386 

387 

388class SlurmConfigError(ValueError): 

389 """Raised when the ``helm.slurm`` block is malformed.""" 

390 

391 

392# --------------------------------------------------------------------------- 

393# Validation 

394# --------------------------------------------------------------------------- 

395 

396 

397def _deep_merge(defaults: Mapping[str, Any], overrides: Mapping[str, Any]) -> dict[str, Any]: 

398 merged: dict[str, Any] = copy.deepcopy(dict(defaults)) 

399 for key, value in overrides.items(): 

400 if value is None and isinstance(merged.get(key), Mapping): 

401 # JSON null for a nested block reads as "absent": keep its defaults. 

402 continue 

403 if isinstance(value, Mapping) and isinstance(merged.get(key), Mapping): 

404 merged[key] = _deep_merge(merged[key], value) 

405 else: 

406 merged[key] = copy.deepcopy(value) 

407 return merged 

408 

409 

410def _reject_unknown_keys(block: Mapping[str, Any], allowed: Iterable[str], path: str) -> None: 

411 allowed_set = set(allowed) 

412 unknown = sorted(str(key) for key in block if key not in allowed_set) 

413 if unknown: 

414 raise SlurmConfigError( 

415 f"{path} contains unknown key(s): {', '.join(unknown)}; " 

416 f"allowed keys: {', '.join(sorted(allowed_set))}" 

417 ) 

418 

419 

420def _require_object(value: object, path: str) -> Mapping[str, Any]: 

421 if not isinstance(value, Mapping): 

422 raise SlurmConfigError(f"{path} must be an object, got {value!r}") 

423 return value 

424 

425 

426def _require_bool(value: object, path: str) -> bool: 

427 if type(value) is not bool: 

428 raise SlurmConfigError(f"{path} must be a boolean, got {value!r}") 

429 return value 

430 

431 

432def _require_int(value: object, bounds: tuple[int, int], path: str) -> int: 

433 """An exact integer (``bool`` rejected) inside the inclusive ``bounds``.""" 

434 low, high = bounds 

435 if type(value) is not int or not low <= value <= high: 

436 raise SlurmConfigError(f"{path} must be an integer between {low} and {high}, got {value!r}") 

437 return value 

438 

439 

440def _require_quantity(value: object, path: str) -> float: 

441 """A positive Kubernetes quantity (``"8"``, ``"500m"``, ``"32Gi"``, ``8``).""" 

442 if type(value) is bool or not isinstance(value, str | int | float): 

443 raise SlurmConfigError(f"{path} must be a Kubernetes quantity, got {value!r}") 

444 try: 

445 parsed = parse_k8s_quantity(str(value)) 

446 except ValueError as exc: 

447 raise SlurmConfigError(f"{path}={value!r} is not a valid Kubernetes quantity") from exc 

448 if parsed <= 0: 

449 raise SlurmConfigError(f"{path} must be greater than zero, got {value!r}") 

450 return parsed 

451 

452 

453def _require_nonempty_string(value: object, path: str) -> str: 

454 if not isinstance(value, str) or not value.strip(): 

455 raise SlurmConfigError(f"{path} must be a non-empty string, got {value!r}") 

456 return value.strip() 

457 

458 

459def _validate_conf_map(raw: object, path: str, owned: frozenset[str]) -> dict[str, list[str]]: 

460 """Validate an ``extraConfMap``-shaped block: key -> string or list of strings.""" 

461 block = _require_object(raw, path) 

462 normalized: dict[str, list[str]] = {} 

463 for key, value in block.items(): 

464 key_text = str(key) 

465 if not CONF_KEY_PATTERN.match(key_text): 

466 raise SlurmConfigError( 

467 f"{path} key {key_text!r} is not a Slurm configuration parameter name" 

468 ) 

469 if key_text in owned: 

470 raise SlurmConfigError( 

471 f"{path}.{key_text} is managed by the slurm-operator and cannot be overridden" 

472 ) 

473 values = value if isinstance(value, list) else [value] 

474 rendered: list[str] = [] 

475 for item in values: 

476 if type(item) is bool or not isinstance(item, str | int | float): 

477 raise SlurmConfigError( 

478 f"{path}.{key_text} must be a string, a number or a list of them, got {value!r}" 

479 ) 

480 text = str(item).strip() 

481 if not text or "\n" in text: 

482 raise SlurmConfigError( 

483 f"{path}.{key_text} values must be non-empty single-line strings, got {item!r}" 

484 ) 

485 rendered.append(text) 

486 if not rendered: 

487 raise SlurmConfigError(f"{path}.{key_text} must not be an empty list") 

488 normalized[key_text] = rendered 

489 return normalized 

490 

491 

492def _validate_partition(name: str, raw: object) -> dict[str, Any]: 

493 path = f"helm.slurm.partitions.{name}" 

494 if not PARTITION_NAME_PATTERN.match(name): 

495 raise SlurmConfigError( 

496 f"helm.slurm.partitions has an invalid partition name {name!r}: use a lowercase " 

497 "DNS label of at most 32 characters (it names the NodeSet and its pods)" 

498 ) 

499 if name in RESERVED_PARTITION_NAMES: 

500 raise SlurmConfigError( 

501 f"helm.slurm.partitions.{name} is reserved by the Slinky chart; choose another name" 

502 ) 

503 block = _require_object(raw, path) 

504 _reject_unknown_keys(block, PARTITION_DEFAULTS.keys(), path) 

505 partition = _deep_merge(PARTITION_DEFAULTS, block) 

506 

507 _require_bool(partition["enabled"], f"{path}.enabled") 

508 _require_bool(partition["default"], f"{path}.default") 

509 _require_bool(partition["exclusive"], f"{path}.exclusive") 

510 pool_name = partition["node_pool"] 

511 if pool_name not in NODE_POOLS: 

512 raise SlurmConfigError( 

513 f"{path}.node_pool must be one of {', '.join(sorted(NODE_POOLS))}, got {pool_name!r}" 

514 ) 

515 pool = NODE_POOLS[pool_name] 

516 cpu = _require_quantity(partition["cpu"], f"{path}.cpu") 

517 if cpu < 1 or cpu != int(cpu): 

518 raise SlurmConfigError( 

519 f"{path}.cpu must be a whole number of cores (a Slurm node advertises whole CPUs), " 

520 f"got {partition['cpu']!r}" 

521 ) 

522 memory = _require_quantity(partition["memory"], f"{path}.memory") 

523 if memory < 1024 * 1024 * 1024: 

524 raise SlurmConfigError( 

525 f"{path}.memory must be at least 1Gi (slurmd and the job need room inside the " 

526 f"pod's limit), got {partition['memory']!r}" 

527 ) 

528 gpus = _require_int(partition["gpus"], (0, 64), f"{path}.gpus") 

529 efa = _require_int(partition["efa_interfaces"], (0, 64), f"{path}.efa_interfaces") 

530 if gpus and not pool["gpu"]: 

531 raise SlurmConfigError( 

532 f"{path}.gpus is {gpus} but node_pool {pool_name!r} has no GPUs; use gpu or gpu-efa" 

533 ) 

534 if pool["gpu"] and not gpus: 

535 raise SlurmConfigError( 

536 f"{path}.node_pool {pool_name!r} schedules onto GPU nodes but gpus is 0; request at " 

537 "least one GPU or use the cpu pool" 

538 ) 

539 if efa and not pool["efa"]: 

540 raise SlurmConfigError( 

541 f"{path}.efa_interfaces is {efa} but node_pool {pool_name!r} has no EFA; use gpu-efa" 

542 ) 

543 if pool["efa"] and not efa: 

544 raise SlurmConfigError( 

545 f"{path}.node_pool {pool_name!r} is the EFA pool but efa_interfaces is 0; request the " 

546 "instance's EFA interfaces (32 on p5-class instances) or use the gpu pool" 

547 ) 

548 if partition["capacity_type"] not in CAPACITY_TYPES: 

549 raise SlurmConfigError( 

550 f"{path}.capacity_type must be one of {', '.join(CAPACITY_TYPES)}, " 

551 f"got {partition['capacity_type']!r}" 

552 ) 

553 min_nodes = _require_int(partition["min_nodes"], (0, MAX_NODES_LIMIT), f"{path}.min_nodes") 

554 max_nodes = _require_int(partition["max_nodes"], (1, MAX_NODES_LIMIT), f"{path}.max_nodes") 

555 if min_nodes > max_nodes: 

556 raise SlurmConfigError( 

557 f"{path}.min_nodes must not exceed max_nodes, got {min_nodes} > {max_nodes}" 

558 ) 

559 max_time = partition["max_time"] 

560 if not isinstance(max_time, str) or not SLURM_TIME_PATTERN.match(max_time.strip()): 

561 raise SlurmConfigError( 

562 f"{path}.max_time must be a Slurm time limit (minutes, HH:MM:SS, D-HH:MM:SS or " 

563 f"UNLIMITED), got {max_time!r}" 

564 ) 

565 partition["max_time"] = max_time.strip() 

566 features = partition["features"] 

567 if not isinstance(features, list) or not all( 

568 isinstance(item, str) and FEATURE_PATTERN.match(item) for item in features 

569 ): 

570 raise SlurmConfigError( 

571 f"{path}.features must be a list of Slurm feature names (letters, digits, _ . -), " 

572 f"got {features!r}" 

573 ) 

574 if len(set(features)) != len(features): 

575 raise SlurmConfigError(f"{path}.features lists a feature twice") 

576 partition["_cpu_cores"] = cpu 

577 partition["_memory_bytes"] = memory 

578 return partition 

579 

580 

581def _validate_partitions(raw: object) -> dict[str, dict[str, Any]]: 

582 block = _require_object(raw, "helm.slurm.partitions") 

583 partitions = {str(name): _validate_partition(str(name), spec) for name, spec in block.items()} 

584 enabled = {name: spec for name, spec in partitions.items() if spec["enabled"]} 

585 if not enabled: 

586 raise SlurmConfigError( 

587 "helm.slurm.partitions must declare at least one enabled partition when Slurm is on" 

588 ) 

589 defaults = sorted(name for name, spec in enabled.items() if spec["default"]) 

590 if len(defaults) != 1: 

591 raise SlurmConfigError( 

592 "helm.slurm.partitions must mark exactly one enabled partition default: true " 

593 f"(sbatch without --partition goes there), found {defaults or 'none'}" 

594 ) 

595 return partitions 

596 

597 

598def _quota_quantity(resource_quota: Mapping[str, Any], key: str) -> float | None: 

599 """A quota ceiling as a float, ``None`` when absent or unparseable. 

600 

601 The regional stack validates ``resource_quota`` itself and reports a bad 

602 value with its own message; this module only compares against values it 

603 can read, so a malformed quota never produces a second, Slurm-flavored 

604 error for the same typo. 

605 """ 

606 value = resource_quota.get(key) 

607 if value is None: 

608 return None 

609 try: 

610 return parse_k8s_quantity(str(value)) 

611 except ValueError: 

612 return None 

613 

614 

615def _validate_quota_fit( 

616 config: Mapping[str, Any], resource_quota: Mapping[str, Any] | None 

617) -> None: 

618 """Fail when the declared partitions can never fit the gco-jobs quota. 

619 

620 Slurm worker pods live in ``gco-jobs`` and count against its 

621 ``ResourceQuota`` and ``LimitRange`` (04-resource-quotas.yaml). A partition 

622 whose node is larger than the per-container ceiling, or whose 

623 ``max_nodes`` worth of nodes exceeds the namespace ceiling on its own, 

624 would deploy fine and then sit PENDING forever with the reason visible only 

625 in namespace events. Check the certain failures at synth; the quota is 

626 shared with other job pods, so fitting here is necessary, not sufficient. 

627 """ 

628 if not resource_quota: 

629 return 

630 enabled = {n: p for n, p in config["partitions"].items() if p["enabled"]} 

631 container_cpu = _quota_quantity(resource_quota, "container_max_cpu") 

632 container_memory = _quota_quantity(resource_quota, "container_max_memory") 

633 container_gpu = _quota_quantity(resource_quota, "container_max_gpu") 

634 for name, spec in enabled.items(): 

635 path = f"helm.slurm.partitions.{name}" 

636 if container_cpu is not None and spec["_cpu_cores"] > container_cpu: 

637 raise SlurmConfigError( 

638 f"{path}.cpu={spec['cpu']!r} exceeds resource_quota.container_max_cpu=" 

639 f"{resource_quota['container_max_cpu']!r}: the LimitRange would reject every " 

640 "worker pod" 

641 ) 

642 if container_memory is not None and spec["_memory_bytes"] > container_memory: 

643 raise SlurmConfigError( 

644 f"{path}.memory={spec['memory']!r} exceeds resource_quota.container_max_memory=" 

645 f"{resource_quota['container_max_memory']!r}: the LimitRange would reject every " 

646 "worker pod" 

647 ) 

648 if container_gpu is not None and spec["gpus"] > container_gpu: 

649 raise SlurmConfigError( 

650 f"{path}.gpus={spec['gpus']} exceeds resource_quota.container_max_gpu=" 

651 f"{resource_quota['container_max_gpu']!r}: the LimitRange would reject every " 

652 "worker pod" 

653 ) 

654 totals = { 

655 "max_cpu": sum(p["_cpu_cores"] * p["max_nodes"] for p in enabled.values()), 

656 "max_memory": sum(p["_memory_bytes"] * p["max_nodes"] for p in enabled.values()), 

657 "max_gpu": float(sum(p["gpus"] * p["max_nodes"] for p in enabled.values())), 

658 "max_pods": float( 

659 sum(p["max_nodes"] for p in enabled.values()) + control_plane_pods(config) 

660 ), 

661 } 

662 for key, needed in totals.items(): 

663 ceiling = _quota_quantity(resource_quota, key) 

664 if ceiling is not None and needed > ceiling: 

665 raise SlurmConfigError( 

666 f"helm.slurm.partitions at max_nodes need {needed:g} of resource_quota.{key} " 

667 f"but the gco-jobs ceiling is {resource_quota[key]!r}; lower max_nodes or raise " 

668 "the quota (workers share it with every other job pod)" 

669 ) 

670 

671 

672def control_plane_pods(config: Mapping[str, Any]) -> int: 

673 """Pods the cluster runs besides workers: slurmctld, slurmrestd, slurmdbd, MariaDB, login.""" 

674 count = 1 + int(config["restapi"]["replicas"]) 

675 if config["accounting"]["enabled"]: 

676 count += 1 

677 if accounting_in_cluster(config): 

678 count += 1 

679 if config["login"]["enabled"]: 

680 count += int(config["login"]["replicas"]) 

681 return count 

682 

683 

684def validate_slurm_config( 

685 raw: object, 

686 *, 

687 resource_quota: Mapping[str, Any] | None = None, 

688 observability_enabled: bool = True, 

689 fsx_enabled: bool = False, 

690) -> dict[str, Any]: 

691 """Validate the raw ``helm.slurm`` block and return it with defaults filled. 

692 

693 ``None`` means "absent" and yields the defaults (Slurm off). Unknown keys 

694 fail by name at every level. The deployment-wide facts that decide whether 

695 the block can work are passed in: the effective ``resource_quota`` (worker 

696 pods must fit the gco-jobs ceilings), whether ``cluster_observability`` is 

697 on (autoscaling reads Slurm's metrics through Prometheus) and whether 

698 ``fsx_lustre`` is on (``storage.scratch: fsx``). 

699 

700 The returned mapping carries two derived, underscore-prefixed numbers per 

701 partition (``_cpu_cores``, ``_memory_bytes``) for the renderer; everything 

702 else is the operator's block with defaults merged. 

703 """ 

704 if raw is None: 

705 config = copy.deepcopy(SLURM_DEFAULTS) 

706 config["partitions"] = _validate_partitions(config["partitions"]) 

707 return config 

708 block = _require_object(raw, "helm.slurm") 

709 _reject_unknown_keys(block, SLURM_DEFAULTS.keys(), "helm.slurm") 

710 for key in ( 

711 "controller", 

712 "restapi", 

713 "accounting", 

714 "metrics", 

715 "login", 

716 "autoscaling", 

717 "storage", 

718 ): 

719 sub = block.get(key) 

720 if sub is None: 

721 continue 

722 sub_block = _require_object(sub, f"helm.slurm.{key}") 

723 _reject_unknown_keys(sub_block, SLURM_DEFAULTS[key].keys(), f"helm.slurm.{key}") 

724 if key == "accounting": 

725 for nested in ("backup", "external"): 

726 nested_raw = sub_block.get(nested) 

727 if nested_raw is None: 

728 continue 

729 nested_block = _require_object(nested_raw, f"helm.slurm.accounting.{nested}") 

730 _reject_unknown_keys( 

731 nested_block, 

732 SLURM_DEFAULTS["accounting"][nested].keys(), 

733 f"helm.slurm.accounting.{nested}", 

734 ) 

735 # Partitions merge like every other block: the shipped cpu/gpu/gpu-efa 

736 # definitions are always present, an entry of the same name deep-merges 

737 # over the shipped one (``gpu: {max_nodes: 8}`` keeps the GPU pool), a new 

738 # name starts from PARTITION_DEFAULTS, and ``enabled: false`` removes one. 

739 config = _deep_merge(SLURM_DEFAULTS, block) 

740 

741 _require_bool(config["enabled"], "helm.slurm.enabled") 

742 cluster_name = _require_nonempty_string(config["cluster_name"], "helm.slurm.cluster_name") 

743 if not CLUSTER_NAME_PATTERN.match(cluster_name): 

744 raise SlurmConfigError( 

745 f"helm.slurm.cluster_name must be a Slurm ClusterName (letters, digits, _ -), " 

746 f"got {cluster_name!r}" 

747 ) 

748 config["cluster_name"] = cluster_name 

749 _require_quantity( 

750 config["controller"]["state_storage_size"], "helm.slurm.controller.state_storage_size" 

751 ) 

752 _require_int(config["restapi"]["replicas"], (1, 10), "helm.slurm.restapi.replicas") 

753 

754 accounting = config["accounting"] 

755 _require_bool(accounting["enabled"], "helm.slurm.accounting.enabled") 

756 _require_quantity(accounting["storage_size"], "helm.slurm.accounting.storage_size") 

757 backup = accounting["backup"] 

758 _require_bool(backup["enabled"], "helm.slurm.accounting.backup.enabled") 

759 _validate_cron(backup["schedule"], "helm.slurm.accounting.backup.schedule") 

760 _require_int(backup["retention_days"], (1, 3650), "helm.slurm.accounting.backup.retention_days") 

761 external = accounting["external"] 

762 for key in ("host", "database", "username", "password_secret", "password_key"): 

763 if not isinstance(external[key], str): 

764 raise SlurmConfigError( 

765 f"helm.slurm.accounting.external.{key} must be a string, got {external[key]!r}" 

766 ) 

767 external[key] = external[key].strip() 

768 _require_int(external["port"], (1, 65535), "helm.slurm.accounting.external.port") 

769 if external["host"]: 

770 for key in ("database", "username", "password_secret", "password_key"): 

771 if not external[key]: 

772 raise SlurmConfigError( 

773 f"helm.slurm.accounting.external.{key} is required when external.host is set" 

774 ) 

775 elif external["password_secret"]: 

776 raise SlurmConfigError( 

777 "helm.slurm.accounting.external.password_secret is set but external.host is empty; " 

778 "set the host to use an external database" 

779 ) 

780 

781 _require_bool(config["metrics"]["enabled"], "helm.slurm.metrics.enabled") 

782 _require_bool(config["login"]["enabled"], "helm.slurm.login.enabled") 

783 _require_int(config["login"]["replicas"], (1, 10), "helm.slurm.login.replicas") 

784 autoscaling = config["autoscaling"] 

785 _require_int( 

786 autoscaling["pending_jobs_threshold"], 

787 (1, 1000), 

788 "helm.slurm.autoscaling.pending_jobs_threshold", 

789 ) 

790 _require_int( 

791 autoscaling["cooldown_seconds"], (60, 86400), "helm.slurm.autoscaling.cooldown_seconds" 

792 ) 

793 _require_int( 

794 autoscaling["polling_interval_seconds"], 

795 (5, 3600), 

796 "helm.slurm.autoscaling.polling_interval_seconds", 

797 ) 

798 

799 storage = config["storage"] 

800 if storage["home"] not in ("efs", "none"): 

801 raise SlurmConfigError( 

802 f"helm.slurm.storage.home must be 'efs' or 'none', got {storage['home']!r}" 

803 ) 

804 if storage["scratch"] not in ("auto", "fsx", "none"): 

805 raise SlurmConfigError( 

806 f"helm.slurm.storage.scratch must be 'auto', 'fsx' or 'none', got {storage['scratch']!r}" 

807 ) 

808 if storage["scratch"] == "fsx" and not fsx_enabled: 

809 raise SlurmConfigError( 

810 "helm.slurm.storage.scratch is 'fsx' but fsx_lustre is not enabled for this " 

811 "deployment; enable fsx_lustre or use 'auto'" 

812 ) 

813 

814 config["slurm_conf"] = _validate_conf_map( 

815 config["slurm_conf"], "helm.slurm.slurm_conf", OPERATOR_OWNED_SLURM_CONF_KEYS 

816 ) 

817 config["slurmdbd_conf"] = _validate_conf_map( 

818 config["slurmdbd_conf"], "helm.slurm.slurmdbd_conf", OPERATOR_OWNED_SLURMDBD_CONF_KEYS 

819 ) 

820 

821 config["partitions"] = _validate_partitions(config["partitions"]) 

822 if config["enabled"]: 

823 if autoscaled_partitions(config) and not observability_enabled: 

824 names = ", ".join(autoscaled_partitions(config)) 

825 raise SlurmConfigError( 

826 f"helm.slurm.partitions {names} autoscale (min_nodes < max_nodes), which reads " 

827 "Slurm's queue metrics through Prometheus, but cluster_observability is disabled; " 

828 "enable it or set min_nodes equal to max_nodes for a fixed-size partition" 

829 )