Combine subtask stderrs in quadrature for the _average and all rows

src/lighteval/logging/info_loggers.py

@@ -372,7 +372,8 @@ def aggregate(self, task_dict: dict[str, LightevalTask], bootstrap_iters: int =
                         logger.warning(f"{task_name}, {metric_name} got an OVERFLOW ERROR when computing stderr.")
 
         # We group subtasks which belong to the same parent task, like MMLU, to compute an average on them
-        # and compute an average of all metrics
+        # and compute an average of all metrics. The subtasks are evaluated on disjoint samples, so the stderr of
+        # an average of G subtask scores is sqrt(sum_i stderr_i**2) / G.
         grouped_tasks = collections.defaultdict(list)
         suite_average = {}
         suite_nb = {}
@@ -383,21 +384,28 @@ def aggregate(self, task_dict: dict[str, LightevalTask], bootstrap_iters: int =
                 task, fewshot = k.split("|")
                 grouped_tasks[f"{task.split(':')[0]}:_average|{fewshot}"].append(k)
             for metric, value in metrics.items():
-                suite_average[metric] = suite_average.get(metric, 0) + value
+                suite_average[metric] = suite_average.get(metric, 0) + (
+                    value**2 if metric.endswith("_stderr") else value
+                )
                 suite_nb[metric] = suite_nb.get(metric, 0) + 1
 
         # Compute average for sub groups
         for average_task, list_of_subtasks in grouped_tasks.items():
             if len(list_of_subtasks) > 1:
                 metrics = list(self.metric_aggregated[list_of_subtasks[0]].keys())
                 self.metric_aggregated[average_task] = {
-                    metric: sum(self.metric_aggregated[k][metric] for k in list_of_subtasks) / len(list_of_subtasks)
+                    metric: (
+                        sum(self.metric_aggregated[k][metric] ** 2 for k in list_of_subtasks) ** 0.5
+                        if metric.endswith("_stderr")
+                        else sum(self.metric_aggregated[k][metric] for k in list_of_subtasks)
+                    )
+                    / len(list_of_subtasks)
                     for metric in metrics
                 }
 
         # Compute average for all
         for metric, value in suite_average.items():
-            suite_average[metric] = value / suite_nb[metric]
+            suite_average[metric] = (value**0.5 if metric.endswith("_stderr") else value) / suite_nb[metric]
 
         self.metric_aggregated["all"] = suite_average
 

tests/unit/logging/test_info_loggers.py

@@ -0,0 +1,47 @@
+# MIT License
+
+# Copyright (c) 2024 The HuggingFace Team
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import math
+
+import pytest
+
+from lighteval.logging.info_loggers import MetricsLogger
+
+
+def test_average_and_all_rows_combine_subtask_stderrs_in_quadrature():
+    # The `_average` and `all` rows are unweighted means of G subtask scores computed on disjoint samples,
+    # so their standard error is sqrt(sum_i stderr_i**2) / G, not the mean of the subtask stderrs.
+    logger = MetricsLogger()
+    subtasks = {
+        "mmlu:anatomy|5": {"acc": 0.5, "acc_stderr": 0.05},
+        "mmlu:astronomy|5": {"acc": 0.7, "acc_stderr": 0.03},
+        "mmlu:virology|5": {"acc": 0.6, "acc_stderr": 0.04},
+    }
+    for task, metrics in subtasks.items():
+        logger.metric_aggregated[task] = dict(metrics)
+
+    logger.aggregate(task_dict={}, bootstrap_iters=1000)
+
+    expected_stderr = math.sqrt(0.05**2 + 0.03**2 + 0.04**2) / 3
+    for row in ("mmlu:_average|5", "all"):
+        assert logger.metric_aggregated[row]["acc"] == pytest.approx(0.6)
+        assert logger.metric_aggregated[row]["acc_stderr"] == pytest.approx(expected_stderr)