Fix normalized multiple-choice probability underflow

src/lighteval/metrics/metrics_sample.py

@@ -347,6 +347,12 @@ def compute(
             if self.log_prob_normalization
             else choices_logprobs
         )
+        normalized_log_probs = np.asarray(normalized_log_probs, dtype=np.float64)
+        if normalized_log_probs.size > 0:
+            max_normalized_log_prob = np.max(normalized_log_probs)
+            if np.isfinite(max_normalized_log_prob):
+                normalized_log_probs = normalized_log_probs - max_normalized_log_prob
+
         normalized_probs = np.exp(normalized_log_probs)
 
         normalized_probs = safe_divide(normalized_probs[gold_ixs], np.sum(normalized_probs))

tests/test_unit_base_metrics.py

@@ -228,6 +228,16 @@ def test_mc_probability_metric(self):
         )
         assert result[prob_norm_metric.metric_name] == pytest.approx(0.7)
 
+    def test_mc_probability_metric_avoids_logprob_underflow(self):
+        doc = Doc(query="Test query", choices=["A", "B"], gold_index=0, task_name="test")
+        model_response = ModelResponse(logprobs=[-1000.0, -1001.0])
+
+        metric = NormalizedMultiChoiceProbMetric()
+        result = metric.compute_sample(doc=doc, model_response=model_response)
+
+        expected = 1 / (1 + np.exp(-1))
+        assert result[metric.metric_name] == pytest.approx(expected)
+
     def test_acc(self):
         # Test without normalization
         doc = Doc(query="Test query", choices=["A", "B", "C", "D"], gold_index=0, task_name="test")