Coverage for gco_mcp / tools / semantic_progress.py: 100.00%

1"""Read-only LLM-as-judge tool that scores Mission progress. 

2 

3The single ``metrics_semantic_progress`` tool scores how close a Mission is 

4to satisfying its directive and returns that score in the canonical 

5``{"metrics": {"progress_score": <number>}}`` shape the Observe_Phase merges, 

6so a plain ``metric_threshold`` or ``metric_trend`` criterion can read it by 

7dot-path with no special handling. 

8 

9The whole tool registration is gated by ``GCO_ENABLE_SEMANTIC_PROGRESS`` so the 

10``@mcp.tool`` decorator only fires when the flag (or the umbrella 

11``GCO_ENABLE_ALL_TOOLS``) is enabled. With the flag unset this module imports 

12cleanly and FastMCP never sees the tool. Each invocation incurs one LLM call 

13via the existing sampling seam, which is why the tool is default-off. 

14 

15[gated by GCO_ENABLE_SEMANTIC_PROGRESS] 

16""" 

17 

18from __future__ import annotations 

19 

20import sys 

21from pathlib import Path 

22from typing import Any 

23 

24from audit import audit_logged 

25from feature_flags import FLAG_SEMANTIC_PROGRESS, is_enabled 

26from server import mcp 

27 

28# The pure judge package and the sampling seam live under ``gco_mcp/``; the 

29# path-injection pattern matches the rest of the MCP module surface so 

30# ``import mission_judge.*`` and ``import mission.*`` resolve without making 

31# the ``mcp`` directory a package. 

32sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) 

33 

34# The sampling seam — reused, not reconstructed. 

35from mission.sampling import ( # noqa: E402 

36 SamplingTransportError, 

37 select_sampling_backend, 

38) 

39from mission_judge import prompt as judge_prompt # noqa: E402 

40from mission_judge import rubric as judge_rubric # noqa: E402 

41from mission_judge import score as judge_score # noqa: E402 

42from mission_judge.shape import ( # noqa: E402 

43 ErrorCode, 

44 JudgeError, 

45 error_envelope, 

46 metrics_result, 

47 validate_output_name, 

48) 

49 

50# Registration is entirely gated by the feature flag. When the flag is unset, 

51# the decorator below never fires and FastMCP never sees the tool, so it does 

52# not appear in ``mcp.list_tools()``. The gate is evaluated only through 

53# ``feature_flags.is_enabled`` — never by reading ``os.environ`` here. 

54if is_enabled(FLAG_SEMANTIC_PROGRESS): 

55 

56 @mcp.tool(tags={"safe", "metrics"}) 

57 @audit_logged 

58 async def metrics_semantic_progress( 

59 directive: str, 

60 recent_context: str | None = None, 

61 output_name: str | None = None, 

62 model_id: str | None = None, 

63 ) -> dict[str, Any]: 

64 """[gated by GCO_ENABLE_SEMANTIC_PROGRESS] [read-only] Score Mission progress. 

65 

66 Scores how close a Mission is to satisfying ``directive`` against a 

67 fixed, versioned rubric via the existing sampling backend, and returns 

68 the canonical ``{"metrics": {"progress_score": <float 0.0-1.0>}}`` shape 

69 consumable by a ``metric_threshold`` (e.g. ``progress_score >= 0.8``) or 

70 ``metric_trend`` (e.g. ``progress_score`` increasing) criterion. Incurs 

71 one LLM call per invocation. Mutates nothing — it only reads its inputs 

72 and asks the model for a score. Provenance (rationale, source, 

73 backend_name, model_id, rubric_version, raw_score) is returned outside 

74 the ``metrics`` object. 

75 

76 Args: 

77 directive: The natural-language objective the Mission is pursuing. 

78 Must be non-empty and not whitespace-only. 

79 recent_context: Optional recent progress context (recent 

80 observations and/or metric-history series the caller selects). 

81 Truncated keep-newest to a fixed character budget; omit it to 

82 score from the directive alone. 

83 output_name: Optional metric key under ``metrics`` (default 

84 ``"progress_score"``). Must be a single path segment of 1..128 

85 characters with no ``.`` separator and no whitespace. 

86 model_id: Optional concrete model identifier forwarded to the 

87 sampling seam; ``None`` uses the seam's resolved default. 

88 

89 Returns the canonical metrics shape on success, or a structured 

90 ``{"code", "details"}`` error envelope (never carrying a top-level 

91 ``metrics`` key) on any failure, so the Mission loop keeps running. 

92 """ 

93 try: 

94 key = validate_output_name(output_name) if output_name else "progress_score" 

95 if not directive or not directive.strip(): 

96 raise JudgeError(ErrorCode.MISSING_DIRECTIVE) 

97 

98 prompt = judge_prompt.build_prompt( 

99 directive, recent_context, judge_rubric.RUBRIC_VERSION 

100 ) 

101 

102 # Bedrock is the only sampling transport (MCP client sampling 

103 # left the protocol with FastMCP 4), so the judge samples 

104 # server-side for CLI and MCP callers alike. 

105 backend = select_sampling_backend(model_id) 

106 if backend is None: # defensive: seam stubs may return None 

107 raise JudgeError(ErrorCode.NO_SAMPLING_BACKEND) 

108 

109 try: 

110 # The ONLY non-determinism; no retry. Both shipped backends 

111 # call only ``prompt.assemble()``, so the duck-typed JudgePrompt 

112 # drives either of them — same shim pattern as the sampling 

113 # module's own ``_PreRendered`` look-alike. 

114 raw_text = await backend.sample(prompt) # type: ignore[arg-type] 

115 except SamplingTransportError as err: 

116 raise JudgeError( 

117 ErrorCode.SAMPLING_TRANSPORT_ERROR, 

118 { 

119 "transport_code": err.code, 

120 "backend_name": backend.backend_name, 

121 "model_id": backend.model_id, 

122 }, 

123 ) from err 

124 

125 raw_score, rationale = judge_score.parse_score(raw_text) # raises INVALID_MODEL_SCORE 

126 value = judge_score.clamp_score(raw_score) 

127 

128 return metrics_result( 

129 key, 

130 value, 

131 rationale=rationale[: judge_prompt.MAX_RATIONALE_CHARS], 

132 source=f"{backend.backend_name}:{backend.model_id}", 

133 backend_name=backend.backend_name, 

134 model_id=backend.model_id, 

135 rubric_version=judge_rubric.RUBRIC_VERSION, 

136 raw_score=raw_score, 

137 ) 

138 except JudgeError as err: 

139 return error_envelope(err.code, **err.details) 

140 except Exception as err: # noqa: BLE001 - defensive: nothing escapes the tool 

141 return error_envelope( 

142 ErrorCode.SAMPLING_TRANSPORT_ERROR, reason="unexpected", detail=str(err) 

143 )