feat(evaluation): add relative_tolerance to static_json scoring

src/evaluation/scorers/static_json.py

@@ -1146,6 +1146,70 @@ def evaluate_static_json_batch(
         "examples": [score.to_dict() for score in scores],
     }
 
+
+def _scenario_meta(scenario: Scenario) -> dict[str, Any]:
+    metadata = scenario.evaluation_metadata or {}
+    value = metadata.get("scenario_meta", {})
+    return value if isinstance(value, dict) else {}
+
+
+def _relative_tolerance(meta: dict[str, Any]) -> float | None:
+    value = meta.get("relative_tolerance")
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    if not math.isfinite(value) or value < 0:
+        return None
+    return float(value)
+
+
+def evaluate_relative_tolerance(
+    gold_answer: Any,
+    model_answer: Any,
+    tolerance: float,
+) -> tuple[bool, float, list[dict[str, Any]]]:
+    """Compare flattened values, letting each number differ by ``tolerance`` x |gold|.
+
+    Keys are positional, so a list must keep the gold order. Non-numeric values
+    such as asset ids must match exactly, and an extra key fails the answer.
+    Comparisons are inclusive and made on values rounded to 6 decimals.
+    """
+    gold_flat = flatten_answer(gold_answer)
+    model_flat = flatten_answer(model_answer)
+
+    values: list[dict[str, Any]] = []
+    accepted = 0
+    for key, gold_value in gold_flat.items():
+        model_value = model_flat.get(key)
+        entry: dict[str, Any] = {
+            "key": key,
+            "gold_value": gold_value,
+            "model_value": "MISSING" if model_value is None else model_value,
+        }
+        gold_num = _as_float(gold_value)
+        if model_value is None:
+            ok = False
+        elif gold_num is None:
+            ok = model_value == gold_value
+        else:
+            model_num = _as_float(model_value)
+            allowed = round(tolerance * abs(gold_num), 6)
+            delta = None if model_num is None else round(abs(model_num - gold_num), 6)
+            ok = delta is not None and delta <= allowed
+            entry.update(delta=delta, allowed_delta=allowed)
+        entry["accepted"] = ok
+        accepted += ok
+        values.append(entry)
+
+    extra = len(set(model_flat) - set(gold_flat))
+    precision = accepted / len(model_flat) if model_flat else 0.0
+    recall = accepted / len(gold_flat) if gold_flat else 0.0
+    f1 = (
+        2 * precision * recall / (precision + recall) if precision + recall > 0 else 0.0
+    )
+    passed = accepted == len(gold_flat) and extra == 0
+    return passed, f1, values
+
+
 class StaticJsonScorer:
     """Evaluation scorer wrapper for the trajectory-based pipeline."""
 
@@ -1174,6 +1238,9 @@ def __call__(
         static_score = evaluate_static_json(
             gold_answer, answer, eval_metadata=scenario.evaluation_metadata
         )
+        meta = _scenario_meta(scenario)
+        if "match" in meta:
+            return self._score_numeric(gold_answer, answer, meta, static_score)
         if static_score.f1 is None:
             # CAR scored on groundtruth_eval.json concepts: recall only.
             score = static_score.recall
@@ -1199,6 +1266,51 @@ def __call__(
             details=static_score.to_dict(),
         )
 
+    def _score_numeric(
+        self,
+        gold_answer: Any,
+        answer: str,
+        meta: dict[str, Any],
+        static_score: StaticJsonScore,
+    ) -> ScorerResult:
+        """Score with ``{"match": "numeric", "relative_tolerance": t}`` from scenario_meta.json."""
+        details = static_score.to_dict()
+        tolerance = _relative_tolerance(meta)
+        if meta.get("match") != "numeric" or tolerance is None:
+            return ScorerResult(
+                scorer=self.name,
+                passed=False,
+                score=0.0,
+                rationale=(
+                    "scenario_meta.json needs match 'numeric' and a non-negative "
+                    f"relative_tolerance; got match={meta.get('match')!r}, "
+                    f"relative_tolerance={meta.get('relative_tolerance')!r}."
+                ),
+                details={**details, "error": "invalid_scenario_meta"},
+            )
+
+        passed, f1, values = evaluate_relative_tolerance(gold_answer, answer, tolerance)
+        accepted = sum(1 for value in values if value["accepted"])
+        extra = len(details["extra_keys"])
+        rationale = (
+            f"{accepted} of {len(values)} values accepted "
+            f"(numbers within relative_tolerance {tolerance:g} of gold), "
+            f"{extra} extra keys"
+        )
+        return ScorerResult(
+            scorer=self.name,
+            passed=passed,
+            score=round(f1, 3),
+            rationale=rationale,
+            details={
+                **details,
+                "match": "numeric",
+                "relative_tolerance": tolerance,
+                "tolerance_operator": "<=",
+                "tolerance_values": values,
+            },
+        )
+
 
 def install(name: str = "static_json") -> None:
     """Register the static JSON scorer."""

src/evaluation/tests/test_static_json_scorer.py

@@ -776,3 +776,145 @@ def test_count_rescue_still_reads_a_number_out_of_prose():
     assert parse_structured_answer("The answer is 10.") == 10
     assert parse_structured_answer("10") == 10
     assert normalize_value("PMP42144") == "pmp42144"
+
+
+_RATES_GOLD = [
+    {"asset": "D", "failures_per_year": 125.2},
+    {"asset": "B", "failures_per_year": 85.0},
+    {"asset": "E", "failures_per_year": 75.6},
+    {"asset": "C", "failures_per_year": 54.1},
+    {"asset": "A", "failures_per_year": 13.2},
+]
+
+
+def _rates_scenario(meta):
+    return Scenario.from_raw(
+        {
+            "id": "410",
+            "text": "Breakdown rate per machine, highest to lowest.",
+            "expected_answer": json.dumps(_RATES_GOLD),
+            "evaluation_metadata": {"scenario_meta": meta},
+            "scoring_method": "static_json",
+        }
+    )
+
+
+def _rates(*values, assets="DBECA"):
+    return json.dumps(
+        [
+            {"asset": asset, "failures_per_year": value}
+            for asset, value in zip(assets, values)
+        ]
+    )
+
+
+_TEN_PERCENT = {
+    "scoring_method": "static_json",
+    "match": "numeric",
+    "relative_tolerance": 0.1,
+}
+
+
+def test_relative_tolerance_passes_rates_within_ten_percent():
+    result = StaticJsonScorer()(
+        _rates_scenario(_TEN_PERCENT), _rates(122.4, 83.8, 73.5, 53.0, 13.0), ""
+    )
+
+    assert result.passed is True
+    assert result.score == 1.0
+    assert result.details["relative_tolerance"] == 0.1
+    assert result.details["tolerance_operator"] == "<="
+    assert result.details["strict_exact_match_accuracy"] == 0.0
+    assert result.rationale.startswith("10 of 10 values accepted")
+
+
+@pytest.mark.parametrize(
+    ("b_rate", "passed"), [(93.5, True), (76.5, True), (93.6, False)]
+)
+def test_relative_tolerance_boundary_is_inclusive(b_rate, passed):
+    result = StaticJsonScorer()(
+        _rates_scenario(_TEN_PERCENT), _rates(125.2, b_rate, 75.6, 54.1, 13.2), ""
+    )
+
+    assert result.passed is passed
+
+
+def test_relative_tolerance_fails_counting_every_work_order():
+    result = StaticJsonScorer()(
+        _rates_scenario(_TEN_PERCENT), _rates(285.9, 243.1, 164.4, 104.9, 30.1), ""
+    )
+
+    assert result.passed is False
+    assert result.score == 0.5
+    rejected = [
+        v["key"] for v in result.details["tolerance_values"] if not v["accepted"]
+    ]
+    assert len(rejected) == 5
+    assert all(key.endswith(".failures_per_year") for key in rejected)
+
+
+def test_relative_tolerance_keeps_list_order_and_asset_ids_exact():
+    swapped = StaticJsonScorer()(
+        _rates_scenario(_TEN_PERCENT),
+        _rates(125.2, 75.6, 85.0, 54.1, 13.2, assets="DEBCA"),
+        "",
+    )
+    renamed = StaticJsonScorer()(
+        _rates_scenario(_TEN_PERCENT),
+        _rates(125.2, 85.0, 75.6, 54.1, 13.2, assets="DBECX"),
+        "",
+    )
+
+    assert swapped.passed is False
+    assert renamed.passed is False
+
+
+def test_relative_tolerance_fails_on_an_extra_key():
+    answer = json.loads(_rates(125.2, 85.0, 75.6, 54.1, 13.2))
+    answer[0]["note"] = "worst unit"
+
+    result = StaticJsonScorer()(_rates_scenario(_TEN_PERCENT), json.dumps(answer), "")
+
+    assert result.passed is False
+    assert result.details["extra_keys"] == ["answer[0].note"]
+
+
+def test_relative_tolerance_reads_a_fenced_answer():
+    answer = "```json\n" + _rates(122.4, 83.8, 73.5, 53.0, 13.0) + "\n```"
+
+    result = StaticJsonScorer()(_rates_scenario(_TEN_PERCENT), answer, "")
+
+    assert result.passed is True
+
+
+@pytest.mark.parametrize(
+    "meta",
+    [
+        {"match": "numeric"},
+        {"match": "numeric", "relative_tolerance": -0.1},
+        {"match": "numeric", "relative_tolerance": True},
+        {"match": "numeric", "relative_tolerance": "0.1"},
+        {"match": "set_f1", "relative_tolerance": 0.1},
+    ],
+)
+def test_relative_tolerance_rejects_invalid_scenario_meta(meta):
+    result = StaticJsonScorer()(
+        _rates_scenario({"scoring_method": "static_json", **meta}),
+        json.dumps(_RATES_GOLD),
+        "",
+    )
+
+    assert result.passed is False
+    assert result.score == 0.0
+    assert result.details["error"] == "invalid_scenario_meta"
+
+
+def test_scenario_meta_without_match_keeps_strict_matching():
+    scenario = _rates_scenario({"scoring_method": "static_json"})
+
+    exact = StaticJsonScorer()(scenario, json.dumps(_RATES_GOLD), "")
+    near = StaticJsonScorer()(scenario, _rates(125.2, 85.0, 75.6, 54.1, 13.0), "")
+
+    assert exact.passed is True
+    assert near.passed is False
+    assert near.rationale == "structured answer differs from ground truth"