fix(metrics): stop counting missing samples as failures in pass@k

src/lighteval/metrics/metrics_sample.py

@@ -1318,10 +1318,17 @@ def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float:
     def pass_at_k(self, all_scores: list[int]) -> float:
         """Algo from https://arxiv.org/pdf/2107.03374"""
         c: int = all_scores.count(1)
-        if self.n - c < self.k:
+        # `all_scores` only holds the samples that are actually available (`compute`
+        # passes `predictions[: self.n]`), while `self.n` is the number of samples that
+        # was requested. Using `self.n` here counts the missing samples as failures and
+        # underestimates pass@k whenever fewer than n samples came back.
+        n = len(all_scores)
+        if n == 0:
+            return 0.0
+        if n - c < self.k:
             return 1.0
 
-        return 1.0 - np.prod(1.0 - self.k / np.arange(self.n - c + 1, self.n + 1))
+        return 1.0 - np.prod(1.0 - self.k / np.arange(n - c + 1, n + 1))
 
     def num_samples(self):
         return self.n if self.n is not None else self.k

tests/test_unit_base_metrics.py

@@ -30,7 +30,7 @@
     NormalizedMultiChoiceProbMetric,
     ProbabilityMetric,
 )
-from lighteval.metrics.metrics_sample import ExactMatches
+from lighteval.metrics.metrics_sample import ExactMatches, PassAtK
 from lighteval.metrics.normalizations import LogProbCharNorm, helm_normalizer
 from lighteval.models.model_output import ModelResponse
 from lighteval.tasks.requests import Doc
@@ -331,6 +331,25 @@ def test_exact_match_dynamic_metric(self):
     def test_pass_at_k_estimator(self):
         assert False
 
+    def test_pass_at_k_with_fewer_samples_than_n(self):
+        # 10 samples were requested, only 5 came back and 3 of them are correct.
+        scores = [1, 1, 1, 0, 0]
+        assert PassAtK(k=1, n=10).pass_at_k(scores) == pytest.approx(0.6)
+        assert PassAtK(k=2, n=10).pass_at_k(scores) == pytest.approx(0.9)
+        assert PassAtK(k=3, n=10).pass_at_k(scores) == pytest.approx(1.0)
+
+    def test_pass_at_k_with_n_samples_is_unchanged(self):
+        # Same 3 correct out of 5, but 5 samples is what was requested.
+        scores = [1, 1, 1, 0, 0]
+        assert PassAtK(k=1, n=5).pass_at_k(scores) == pytest.approx(0.6)
+        assert PassAtK(k=2, n=5).pass_at_k(scores) == pytest.approx(0.9)
+        assert PassAtK(k=3, n=5).pass_at_k(scores) == pytest.approx(1.0)
+        # n larger than k and more samples than needed: unchanged too.
+        assert PassAtK(k=1, n=10).pass_at_k([1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) == pytest.approx(0.2)
+
+    def test_pass_at_k_without_any_sample_is_zero(self):
+        assert PassAtK(k=1, n=10).pass_at_k([]) == 0.0
+
     @pytest.mark.skip(reason="Using nltk metric function, no need to test.")
     def test_f1_score_quasi(self):
         assert False