diff --git a/src/evaluation/scorers/static_json.py b/src/evaluation/scorers/static_json.py index 3b7d8287..dcfa7cc5 100644 --- a/src/evaluation/scorers/static_json.py +++ b/src/evaluation/scorers/static_json.py @@ -1146,6 +1146,70 @@ def evaluate_static_json_batch( "examples": [score.to_dict() for score in scores], } + +def _scenario_meta(scenario: Scenario) -> dict[str, Any]: + metadata = scenario.evaluation_metadata or {} + value = metadata.get("scenario_meta", {}) + return value if isinstance(value, dict) else {} + + +def _relative_tolerance(meta: dict[str, Any]) -> float | None: + value = meta.get("relative_tolerance") + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + if not math.isfinite(value) or value < 0: + return None + return float(value) + + +def evaluate_relative_tolerance( + gold_answer: Any, + model_answer: Any, + tolerance: float, +) -> tuple[bool, float, list[dict[str, Any]]]: + """Compare flattened values, letting each number differ by ``tolerance`` x |gold|. + + Keys are positional, so a list must keep the gold order. Non-numeric values + such as asset ids must match exactly, and an extra key fails the answer. + Comparisons are inclusive and made on values rounded to 6 decimals. + """ + gold_flat = flatten_answer(gold_answer) + model_flat = flatten_answer(model_answer) + + values: list[dict[str, Any]] = [] + accepted = 0 + for key, gold_value in gold_flat.items(): + model_value = model_flat.get(key) + entry: dict[str, Any] = { + "key": key, + "gold_value": gold_value, + "model_value": "MISSING" if model_value is None else model_value, + } + gold_num = _as_float(gold_value) + if model_value is None: + ok = False + elif gold_num is None: + ok = model_value == gold_value + else: + model_num = _as_float(model_value) + allowed = round(tolerance * abs(gold_num), 6) + delta = None if model_num is None else round(abs(model_num - gold_num), 6) + ok = delta is not None and delta <= allowed + entry.update(delta=delta, allowed_delta=allowed) + entry["accepted"] = ok + accepted += ok + values.append(entry) + + extra = len(set(model_flat) - set(gold_flat)) + precision = accepted / len(model_flat) if model_flat else 0.0 + recall = accepted / len(gold_flat) if gold_flat else 0.0 + f1 = ( + 2 * precision * recall / (precision + recall) if precision + recall > 0 else 0.0 + ) + passed = accepted == len(gold_flat) and extra == 0 + return passed, f1, values + + class StaticJsonScorer: """Evaluation scorer wrapper for the trajectory-based pipeline.""" @@ -1174,6 +1238,9 @@ def __call__( static_score = evaluate_static_json( gold_answer, answer, eval_metadata=scenario.evaluation_metadata ) + meta = _scenario_meta(scenario) + if "match" in meta: + return self._score_numeric(gold_answer, answer, meta, static_score) if static_score.f1 is None: # CAR scored on groundtruth_eval.json concepts: recall only. score = static_score.recall @@ -1199,6 +1266,51 @@ def __call__( details=static_score.to_dict(), ) + def _score_numeric( + self, + gold_answer: Any, + answer: str, + meta: dict[str, Any], + static_score: StaticJsonScore, + ) -> ScorerResult: + """Score with ``{"match": "numeric", "relative_tolerance": t}`` from scenario_meta.json.""" + details = static_score.to_dict() + tolerance = _relative_tolerance(meta) + if meta.get("match") != "numeric" or tolerance is None: + return ScorerResult( + scorer=self.name, + passed=False, + score=0.0, + rationale=( + "scenario_meta.json needs match 'numeric' and a non-negative " + f"relative_tolerance; got match={meta.get('match')!r}, " + f"relative_tolerance={meta.get('relative_tolerance')!r}." + ), + details={**details, "error": "invalid_scenario_meta"}, + ) + + passed, f1, values = evaluate_relative_tolerance(gold_answer, answer, tolerance) + accepted = sum(1 for value in values if value["accepted"]) + extra = len(details["extra_keys"]) + rationale = ( + f"{accepted} of {len(values)} values accepted " + f"(numbers within relative_tolerance {tolerance:g} of gold), " + f"{extra} extra keys" + ) + return ScorerResult( + scorer=self.name, + passed=passed, + score=round(f1, 3), + rationale=rationale, + details={ + **details, + "match": "numeric", + "relative_tolerance": tolerance, + "tolerance_operator": "<=", + "tolerance_values": values, + }, + ) + def install(name: str = "static_json") -> None: """Register the static JSON scorer.""" diff --git a/src/evaluation/tests/test_static_json_scorer.py b/src/evaluation/tests/test_static_json_scorer.py index 6b0aee11..14f255ba 100644 --- a/src/evaluation/tests/test_static_json_scorer.py +++ b/src/evaluation/tests/test_static_json_scorer.py @@ -776,3 +776,145 @@ def test_count_rescue_still_reads_a_number_out_of_prose(): assert parse_structured_answer("The answer is 10.") == 10 assert parse_structured_answer("10") == 10 assert normalize_value("PMP42144") == "pmp42144" + + +_RATES_GOLD = [ + {"asset": "D", "failures_per_year": 125.2}, + {"asset": "B", "failures_per_year": 85.0}, + {"asset": "E", "failures_per_year": 75.6}, + {"asset": "C", "failures_per_year": 54.1}, + {"asset": "A", "failures_per_year": 13.2}, +] + + +def _rates_scenario(meta): + return Scenario.from_raw( + { + "id": "410", + "text": "Breakdown rate per machine, highest to lowest.", + "expected_answer": json.dumps(_RATES_GOLD), + "evaluation_metadata": {"scenario_meta": meta}, + "scoring_method": "static_json", + } + ) + + +def _rates(*values, assets="DBECA"): + return json.dumps( + [ + {"asset": asset, "failures_per_year": value} + for asset, value in zip(assets, values) + ] + ) + + +_TEN_PERCENT = { + "scoring_method": "static_json", + "match": "numeric", + "relative_tolerance": 0.1, +} + + +def test_relative_tolerance_passes_rates_within_ten_percent(): + result = StaticJsonScorer()( + _rates_scenario(_TEN_PERCENT), _rates(122.4, 83.8, 73.5, 53.0, 13.0), "" + ) + + assert result.passed is True + assert result.score == 1.0 + assert result.details["relative_tolerance"] == 0.1 + assert result.details["tolerance_operator"] == "<=" + assert result.details["strict_exact_match_accuracy"] == 0.0 + assert result.rationale.startswith("10 of 10 values accepted") + + +@pytest.mark.parametrize( + ("b_rate", "passed"), [(93.5, True), (76.5, True), (93.6, False)] +) +def test_relative_tolerance_boundary_is_inclusive(b_rate, passed): + result = StaticJsonScorer()( + _rates_scenario(_TEN_PERCENT), _rates(125.2, b_rate, 75.6, 54.1, 13.2), "" + ) + + assert result.passed is passed + + +def test_relative_tolerance_fails_counting_every_work_order(): + result = StaticJsonScorer()( + _rates_scenario(_TEN_PERCENT), _rates(285.9, 243.1, 164.4, 104.9, 30.1), "" + ) + + assert result.passed is False + assert result.score == 0.5 + rejected = [ + v["key"] for v in result.details["tolerance_values"] if not v["accepted"] + ] + assert len(rejected) == 5 + assert all(key.endswith(".failures_per_year") for key in rejected) + + +def test_relative_tolerance_keeps_list_order_and_asset_ids_exact(): + swapped = StaticJsonScorer()( + _rates_scenario(_TEN_PERCENT), + _rates(125.2, 75.6, 85.0, 54.1, 13.2, assets="DEBCA"), + "", + ) + renamed = StaticJsonScorer()( + _rates_scenario(_TEN_PERCENT), + _rates(125.2, 85.0, 75.6, 54.1, 13.2, assets="DBECX"), + "", + ) + + assert swapped.passed is False + assert renamed.passed is False + + +def test_relative_tolerance_fails_on_an_extra_key(): + answer = json.loads(_rates(125.2, 85.0, 75.6, 54.1, 13.2)) + answer[0]["note"] = "worst unit" + + result = StaticJsonScorer()(_rates_scenario(_TEN_PERCENT), json.dumps(answer), "") + + assert result.passed is False + assert result.details["extra_keys"] == ["answer[0].note"] + + +def test_relative_tolerance_reads_a_fenced_answer(): + answer = "```json\n" + _rates(122.4, 83.8, 73.5, 53.0, 13.0) + "\n```" + + result = StaticJsonScorer()(_rates_scenario(_TEN_PERCENT), answer, "") + + assert result.passed is True + + +@pytest.mark.parametrize( + "meta", + [ + {"match": "numeric"}, + {"match": "numeric", "relative_tolerance": -0.1}, + {"match": "numeric", "relative_tolerance": True}, + {"match": "numeric", "relative_tolerance": "0.1"}, + {"match": "set_f1", "relative_tolerance": 0.1}, + ], +) +def test_relative_tolerance_rejects_invalid_scenario_meta(meta): + result = StaticJsonScorer()( + _rates_scenario({"scoring_method": "static_json", **meta}), + json.dumps(_RATES_GOLD), + "", + ) + + assert result.passed is False + assert result.score == 0.0 + assert result.details["error"] == "invalid_scenario_meta" + + +def test_scenario_meta_without_match_keeps_strict_matching(): + scenario = _rates_scenario({"scoring_method": "static_json"}) + + exact = StaticJsonScorer()(scenario, json.dumps(_RATES_GOLD), "") + near = StaticJsonScorer()(scenario, _rates(125.2, 85.0, 75.6, 54.1, 13.0), "") + + assert exact.passed is True + assert near.passed is False + assert near.rationale == "structured answer differs from ground truth"