Add HAE-RAE Bench (five original subsets)

src/lighteval/tasks/tasks/haerae.py

@@ -0,0 +1,108 @@
+"""
+name:
+HAE-RAE Bench
+
+dataset:
+HAERAE-HUB/HAE_RAE_BENCH_1.1
+
+abstract:
+HAE-RAE Bench is a Korean knowledge benchmark built from original Korean
+sources rather than translated from English. This port covers the five
+multiple-choice subsets of the original benchmark (standard nomenclature,
+loan words, rare words, general knowledge, history; 1,091 questions), which are
+the same subsets as the `haerae` group of EleutherAI lm-evaluation-harness.
+`haerae` scores the answer label like lm-evaluation-harness does. `haerae_cf`
+scores the text of each option appended to the question. The paper describes
+its evaluation as the log-likelihood of each option concatenated to the
+question, run with lm-evaluation-harness; it does not give the prompt text, so
+the "질문: ... 정답:" wording here is ours.
+
+languages:
+korean
+
+tags:
+knowledge, multiple-choice
+
+paper:
+https://arxiv.org/abs/2309.02706
+"""
+
+import ast
+import re
+
+from lighteval.metrics.dynamic_metrics import LogLikelihoodAccMetric
+from lighteval.metrics.metrics import Metrics
+from lighteval.metrics.normalizations import LogProbCharNorm
+from lighteval.tasks.lighteval_task import LightevalTaskConfig
+from lighteval.tasks.requests import Doc
+
+
+# Dataset commit pinned so scores stay comparable (main on 2026-11-03).
+HAERAE_REVISION = "b480e81024913f27783d2b05d2f0b10089db19ad"
+
+HAERAE_SUBSETS = ["standard_nomenclature", "loan_words", "rare_words", "general_knowledge", "history"]
+
+_LABELS = ["(A)", "(B)", "(C)", "(D)", "(E)"]
+
+
+def haerae_prompt(line, task_name: str = None):
+    # `query` already holds the instruction, the question, the lettered options and "### 정답:", as in
+    # lm-eval's doc_to_text. lm-eval joins context and choice with target_delimiter=" ".
+    return Doc(
+        task_name=task_name,
+        query=line["query"],
+        choices=[f" {label}" for label in _LABELS],
+        gold_index=_LABELS.index(line["answer"]),
+    )
+
+
+# The question block of `query` sits between "### 질문:" and "### 선택지:".
+_QUESTION_RE = re.compile(r"### 질문:\s*(.*?)\s*### 선택지:", re.DOTALL)
+_EMPTY_NOTE_RE = re.compile(r"^### 참고:nan[ \t]*\n?", re.MULTILINE)
+
+
+def haerae_cf_prompt(line, task_name: str = None):
+    # Closed-form: the question followed by the option text. The general_knowledge `query` can carry a
+    # literal "### 참고:nan" line (an empty note, 98 of 176 questions); it is dropped here. The label prompt
+    # keeps the `query` untouched. Notes with real text are kept.
+    options = ast.literal_eval(line["options"])
+    question = _EMPTY_NOTE_RE.sub("", _QUESTION_RE.search(line["query"]).group(1)).strip()
+    return Doc(
+        task_name=task_name,
+        query=f"질문: {question}\n정답:",
+        choices=[f" {option}" for option in options],
+        gold_index=_LABELS.index(line["answer"]),
+    )
+
+
+def _task(name: str, prompt_function, subset: str, metrics: list) -> LightevalTaskConfig:
+    return LightevalTaskConfig(
+        name=name,
+        prompt_function=prompt_function,
+        hf_repo="HAERAE-HUB/HAE_RAE_BENCH_1.1",
+        hf_subset=subset,
+        hf_revision=HAERAE_REVISION,
+        hf_avail_splits=["test"],
+        evaluation_splits=["test"],
+        few_shots_split=None,
+        few_shots_select=None,
+        metrics=metrics,
+        version=0,
+    )
+
+
+HAERAE_TASKS = [
+    _task(f"haerae:{subset}", haerae_prompt, subset, [Metrics.loglikelihood_acc]) for subset in HAERAE_SUBSETS
+]
+
+HAERAE_CF_TASKS = [
+    _task(
+        f"haerae_cf:{subset}",
+        haerae_cf_prompt,
+        subset,
+        [Metrics.loglikelihood_acc, LogLikelihoodAccMetric(normalization=LogProbCharNorm())],
+    )
+    for subset in HAERAE_SUBSETS
+]
+
+TASKS_TABLE = HAERAE_TASKS + HAERAE_CF_TASKS

tests/unit/tasks/test_haerae.py

@@ -0,0 +1,92 @@
+# MIT License
+
+# Copyright (c) 2024 The HuggingFace Team
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+import pytest
+
+from lighteval.tasks.tasks.haerae import (
+    HAERAE_CF_TASKS,
+    HAERAE_SUBSETS,
+    HAERAE_TASKS,
+    haerae_cf_prompt,
+    haerae_prompt,
+)
+
+
+QUERY = (
+    "다음 질문에 대한 정답을 선택지에서 고르시오.\n"
+    "### 질문: 우리나라의 수도는 어디인가?\n"
+    "### 선택지:\n(A) 부산\n(B) 서울\n(C) 대구\n(D) 인천\n(E) 광주\n"
+    "### 정답:"
+)
+
+
+def make_line(answer: str) -> dict:
+    return {
+        "query": QUERY,
+        "options": str(["부산", "서울", "대구", "인천", "광주"]),
+        "answer": answer,
+    }
+
+
+@pytest.mark.parametrize("answer, index", [("(A)", 0), ("(B)", 1), ("(C)", 2), ("(D)", 3), ("(E)", 4)])
+def test_gold_index(answer, index):
+    assert haerae_prompt(make_line(answer), "haerae:history").gold_index == index
+    assert haerae_cf_prompt(make_line(answer), "haerae_cf:history").gold_index == index
+
+
+def test_label_prompt():
+    doc = haerae_prompt(make_line("(B)"), "haerae:history")
+    assert doc.query == QUERY
+    assert doc.choices == [" (A)", " (B)", " (C)", " (D)", " (E)"]
+
+
+def test_closed_form_prompt():
+    doc = haerae_cf_prompt(make_line("(B)"), "haerae_cf:history")
+    assert doc.query == "질문: 우리나라의 수도는 어디인가?\n정답:"
+    assert doc.choices == [" 부산", " 서울", " 대구", " 인천", " 광주"]
+    assert doc.choices[doc.gold_index] == " 서울"
+
+
+def make_note_line(note: str) -> dict:
+    line = make_line("(B)")
+    line["query"] = QUERY.replace("\n### 선택지:", f"\n### 참고:{note}\n\n### 선택지:")
+    return line
+
+
+def test_empty_note_dropped_in_closed_form_only():
+    line = make_note_line("nan")
+    assert "### 참고:nan" in haerae_prompt(line, "haerae:general_knowledge").query
+    doc = haerae_cf_prompt(line, "haerae_cf:general_knowledge")
+    assert doc.query == "질문: 우리나라의 수도는 어디인가?\n정답:"
+
+
+def test_real_note_kept_in_closed_form():
+    doc = haerae_cf_prompt(make_note_line("서울은 한강이 흐른다."), "haerae_cf:general_knowledge")
+    assert doc.query == "질문: 우리나라의 수도는 어디인가?\n### 참고:서울은 한강이 흐른다.\n정답:"
+
+
+def test_task_table():
+    assert len(HAERAE_TASKS) == len(HAERAE_CF_TASKS) == len(HAERAE_SUBSETS) == 5
+    assert [t.name for t in HAERAE_TASKS] == [f"haerae:{s}" for s in HAERAE_SUBSETS]
+    assert [t.name for t in HAERAE_CF_TASKS] == [f"haerae_cf:{s}" for s in HAERAE_SUBSETS]
+    assert all(task.hf_revision for task in HAERAE_TASKS + HAERAE_CF_TASKS)