diff --git a/src/lighteval/tasks/tasks/haerae.py b/src/lighteval/tasks/tasks/haerae.py new file mode 100644 index 000000000..4f4ef6222 --- /dev/null +++ b/src/lighteval/tasks/tasks/haerae.py @@ -0,0 +1,108 @@ +""" +name: +HAE-RAE Bench + +dataset: +HAERAE-HUB/HAE_RAE_BENCH_1.1 + +abstract: +HAE-RAE Bench is a Korean knowledge benchmark built from original Korean +sources rather than translated from English. This port covers the five +multiple-choice subsets of the original benchmark (standard nomenclature, +loan words, rare words, general knowledge, history; 1,091 questions), which are +the same subsets as the `haerae` group of EleutherAI lm-evaluation-harness. +`haerae` scores the answer label like lm-evaluation-harness does. `haerae_cf` +scores the text of each option appended to the question. The paper describes +its evaluation as the log-likelihood of each option concatenated to the +question, run with lm-evaluation-harness; it does not give the prompt text, so +the "질문: ... 정답:" wording here is ours. + +languages: +korean + +tags: +knowledge, multiple-choice + +paper: +https://arxiv.org/abs/2309.02706 +""" + +import ast +import re + +from lighteval.metrics.dynamic_metrics import LogLikelihoodAccMetric +from lighteval.metrics.metrics import Metrics +from lighteval.metrics.normalizations import LogProbCharNorm +from lighteval.tasks.lighteval_task import LightevalTaskConfig +from lighteval.tasks.requests import Doc + + +# Dataset commit pinned so scores stay comparable (main on 2026-11-03). +HAERAE_REVISION = "b480e81024913f27783d2b05d2f0b10089db19ad" + +HAERAE_SUBSETS = ["standard_nomenclature", "loan_words", "rare_words", "general_knowledge", "history"] + +_LABELS = ["(A)", "(B)", "(C)", "(D)", "(E)"] + + +def haerae_prompt(line, task_name: str = None): + # `query` already holds the instruction, the question, the lettered options and "### 정답:", as in + # lm-eval's doc_to_text. lm-eval joins context and choice with target_delimiter=" ". + return Doc( + task_name=task_name, + query=line["query"], + choices=[f" {label}" for label in _LABELS], + gold_index=_LABELS.index(line["answer"]), + ) + + +# The question block of `query` sits between "### 질문:" and "### 선택지:". +_QUESTION_RE = re.compile(r"### 질문:\s*(.*?)\s*### 선택지:", re.DOTALL) +_EMPTY_NOTE_RE = re.compile(r"^### 참고:nan[ \t]*\n?", re.MULTILINE) + + +def haerae_cf_prompt(line, task_name: str = None): + # Closed-form: the question followed by the option text. The general_knowledge `query` can carry a + # literal "### 참고:nan" line (an empty note, 98 of 176 questions); it is dropped here. The label prompt + # keeps the `query` untouched. Notes with real text are kept. + options = ast.literal_eval(line["options"]) + question = _EMPTY_NOTE_RE.sub("", _QUESTION_RE.search(line["query"]).group(1)).strip() + return Doc( + task_name=task_name, + query=f"질문: {question}\n정답:", + choices=[f" {option}" for option in options], + gold_index=_LABELS.index(line["answer"]), + ) + + +def _task(name: str, prompt_function, subset: str, metrics: list) -> LightevalTaskConfig: + return LightevalTaskConfig( + name=name, + prompt_function=prompt_function, + hf_repo="HAERAE-HUB/HAE_RAE_BENCH_1.1", + hf_subset=subset, + hf_revision=HAERAE_REVISION, + hf_avail_splits=["test"], + evaluation_splits=["test"], + few_shots_split=None, + few_shots_select=None, + metrics=metrics, + version=0, + ) + + +HAERAE_TASKS = [ + _task(f"haerae:{subset}", haerae_prompt, subset, [Metrics.loglikelihood_acc]) for subset in HAERAE_SUBSETS +] + +HAERAE_CF_TASKS = [ + _task( + f"haerae_cf:{subset}", + haerae_cf_prompt, + subset, + [Metrics.loglikelihood_acc, LogLikelihoodAccMetric(normalization=LogProbCharNorm())], + ) + for subset in HAERAE_SUBSETS +] + +TASKS_TABLE = HAERAE_TASKS + HAERAE_CF_TASKS diff --git a/tests/unit/tasks/test_haerae.py b/tests/unit/tasks/test_haerae.py new file mode 100644 index 000000000..937de1bb9 --- /dev/null +++ b/tests/unit/tasks/test_haerae.py @@ -0,0 +1,92 @@ +# MIT License + +# Copyright (c) 2024 The HuggingFace Team + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import pytest + +from lighteval.tasks.tasks.haerae import ( + HAERAE_CF_TASKS, + HAERAE_SUBSETS, + HAERAE_TASKS, + haerae_cf_prompt, + haerae_prompt, +) + + +QUERY = ( + "다음 질문에 대한 정답을 선택지에서 고르시오.\n" + "### 질문: 우리나라의 수도는 어디인가?\n" + "### 선택지:\n(A) 부산\n(B) 서울\n(C) 대구\n(D) 인천\n(E) 광주\n" + "### 정답:" +) + + +def make_line(answer: str) -> dict: + return { + "query": QUERY, + "options": str(["부산", "서울", "대구", "인천", "광주"]), + "answer": answer, + } + + +@pytest.mark.parametrize("answer, index", [("(A)", 0), ("(B)", 1), ("(C)", 2), ("(D)", 3), ("(E)", 4)]) +def test_gold_index(answer, index): + assert haerae_prompt(make_line(answer), "haerae:history").gold_index == index + assert haerae_cf_prompt(make_line(answer), "haerae_cf:history").gold_index == index + + +def test_label_prompt(): + doc = haerae_prompt(make_line("(B)"), "haerae:history") + assert doc.query == QUERY + assert doc.choices == [" (A)", " (B)", " (C)", " (D)", " (E)"] + + +def test_closed_form_prompt(): + doc = haerae_cf_prompt(make_line("(B)"), "haerae_cf:history") + assert doc.query == "질문: 우리나라의 수도는 어디인가?\n정답:" + assert doc.choices == [" 부산", " 서울", " 대구", " 인천", " 광주"] + assert doc.choices[doc.gold_index] == " 서울" + + +def make_note_line(note: str) -> dict: + line = make_line("(B)") + line["query"] = QUERY.replace("\n### 선택지:", f"\n### 참고:{note}\n\n### 선택지:") + return line + + +def test_empty_note_dropped_in_closed_form_only(): + line = make_note_line("nan") + assert "### 참고:nan" in haerae_prompt(line, "haerae:general_knowledge").query + doc = haerae_cf_prompt(line, "haerae_cf:general_knowledge") + assert doc.query == "질문: 우리나라의 수도는 어디인가?\n정답:" + + +def test_real_note_kept_in_closed_form(): + doc = haerae_cf_prompt(make_note_line("서울은 한강이 흐른다."), "haerae_cf:general_knowledge") + assert doc.query == "질문: 우리나라의 수도는 어디인가?\n### 참고:서울은 한강이 흐른다.\n정답:" + + +def test_task_table(): + assert len(HAERAE_TASKS) == len(HAERAE_CF_TASKS) == len(HAERAE_SUBSETS) == 5 + assert [t.name for t in HAERAE_TASKS] == [f"haerae:{s}" for s in HAERAE_SUBSETS] + assert [t.name for t in HAERAE_CF_TASKS] == [f"haerae_cf:{s}" for s in HAERAE_SUBSETS] + assert all(task.hf_revision for task in HAERAE_TASKS + HAERAE_CF_TASKS)