Coverage for gco / resource_governance.py: 100.00%

1"""Shared gco-jobs resource-governance defaults and quantity parsing. 

2 

3Three enforcement layers govern job resources and must tell one story: the 

4manifest/queue processors cap what a single submitted manifest may total, 

5the gco-jobs ``LimitRange`` caps each container, and the namespace 

6``ResourceQuota`` caps the aggregate. The values below are the single source 

7of truth for all three, shared by: 

8 

9- the regional stack (substitutes them into ``04-resource-quotas.yaml`` and 

10 validates cdk.json overrides against the layering invariant at synth), 

11- the manifest and queue processors (their built-in runtime defaults), and 

12- the example-job validation static checks (prove every shipped example 

13 fits these defaults offline). 

14 

15This module lives at the top of the ``gco`` package — NOT under 

16``gco.stacks`` — because the service container images deliberately exclude 

17``gco/stacks/**`` from their build context (synth-only code must never 

18rebuild service images; see ``_SERVICE_IMAGE_COMMON_EXCLUDES`` in the 

19regional stack). A runtime import from ``gco.stacks`` fails the distroless 

20runtime smoke at image build. ``gco.stacks.constants`` re-exports these 

21names for synth-side callers. 

22""" 

23 

24from __future__ import annotations 

25 

26from collections.abc import Mapping 

27from types import MappingProxyType 

28 

29DEFAULT_RESOURCE_QUOTA: Mapping[str, str] = MappingProxyType( 

30 { 

31 # Namespace-wide aggregate ceilings (ResourceQuota on requests.*): 

32 # sized to hold two full accelerator-node training pods side by side 

33 # with CPU headroom for the surrounding jobs. 

34 "max_cpu": "400", 

35 "max_memory": "4096Gi", 

36 "max_gpu": "32", 

37 "max_pods": "50", 

38 # Per-container ceilings (LimitRange max): one full accelerator-node 

39 # slice — p5.48xlarge / trn2.48xlarge expose 192 vCPUs, 2 TiB memory, 

40 # and 8 GPUs, and one-pod-per-node is the standard unit of 

41 # distributed training. Anything smaller rejects the platform's own 

42 # EFA training example at pod admission (observed live: example-job 

43 # validation run ex241-df723811, where the previous 10-CPU/64Gi/4-GPU 

44 # caps left the Job permanently podless with only namespace events 

45 # explaining why). 

46 "container_max_cpu": "192", 

47 "container_max_memory": "2048Gi", 

48 "container_max_gpu": "8", 

49 } 

50) 

51"""Default ``resource_quota`` context for the gco-jobs namespace.""" 

52 

53 

54def parse_k8s_quantity(value: object) -> float: 

55 """Parse a Kubernetes resource quantity into a float of base units. 

56 

57 Supports the quantity forms GCO's manifests actually use: bare integers 

58 and decimals (``8``, ``0.5``), CPU millicores (``250m``), and the binary 

59 and decimal suffixes (``Ki Mi Gi Ti Pi`` / ``k M G T P``). 

60 

61 Raises: 

62 ValueError: If the value is not a parseable quantity. 

63 """ 

64 text = str(value).strip() 

65 if not text: 

66 raise ValueError("empty resource quantity") 

67 binary = {"Ki": 2**10, "Mi": 2**20, "Gi": 2**30, "Ti": 2**40, "Pi": 2**50} 

68 decimal = {"k": 10**3, "M": 10**6, "G": 10**9, "T": 10**12, "P": 10**15} 

69 for suffix, factor in binary.items(): 

70 if text.endswith(suffix): 

71 return float(text[: -len(suffix)]) * factor 

72 if text.endswith("m"): 

73 return float(text[:-1]) / 1000.0 

74 for suffix, factor in decimal.items(): 

75 if text.endswith(suffix): 

76 return float(text[: -len(suffix)]) * factor 

77 return float(text) 

78 

79 

80DEFAULT_MANIFEST_RESOURCE_CAPS: Mapping[str, object] = MappingProxyType( 

81 { 

82 # Front-door budget for one API/SQS-submitted manifest, enforced by 

83 # the manifest and queue processors before anything reaches the 

84 # cluster: two full accelerator-node slices, i.e. the canonical 

85 # two-node distributed-training manifest 

86 # (examples/efa-distributed-training.yaml). Layering invariant, 

87 # validated at synth: container_max_* (LimitRange) <= per-manifest 

88 # cap <= max_* (namespace ResourceQuota) on every dimension — the 

89 # front door must never reject a manifest whose pods the namespace 

90 # would admit, and must never accept one it cannot possibly run. 

91 "max_cpu_per_manifest": "384", 

92 "max_memory_per_manifest": "4096Gi", 

93 "max_gpu_per_manifest": 16, 

94 } 

95) 

96"""Default ``job_validation_policy.resource_quotas`` for submitted manifests."""