From 87398cf754f63b6ef920a6c30596e773bdcee373 Mon Sep 17 00:00:00 2001 From: inchang-ing <197932532+inchang-ing@users.noreply.github.com> Date: Tue, 3 Nov 2026 07:46:27 +0000 Subject: [PATCH] Add translation literals for Latvian, Lithuanian, Maltese and Slovenian The Belebele cf tasks for these four languages could not be built: prompt construction reads `question_word` and `answer`, and both were left unset on their TranslationLiterals entries, so the documents failed with AttributeError before any model was loaded. Fill in the literal set described in docs/source/contributing-to-multilingual-evaluations.mdx. The punctuation defaults already match these languages, so only the words are set. A parametrized test checks the cf prompt labels for each of the four languages. Fixes #1471 --- .../templates/utils/translation_literals.py | 64 +++++++++++++++++-- .../unit/tasks/templates/test_multichoice.py | 36 +++++++++++ 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/src/lighteval/tasks/templates/utils/translation_literals.py b/src/lighteval/tasks/templates/utils/translation_literals.py index 96cd8fc2c..6263789cd 100644 --- a/src/lighteval/tasks/templates/utils/translation_literals.py +++ b/src/lighteval/tasks/templates/utils/translation_literals.py @@ -769,11 +769,39 @@ def __getattribute__(self, name: str) -> str: Language.LAO: TranslationLiterals(language=Language.LAO), Language.LATGALIAN: TranslationLiterals(language=Language.LATGALIAN), Language.LATIN: TranslationLiterals(language=Language.LATIN), - Language.LATVIAN: TranslationLiterals(language=Language.LATVIAN), + Language.LATVIAN: TranslationLiterals( + language=Language.LATVIAN, + question_word="jautājums", + answer="atbilde", + confirmation_word="vai ne", + yes="jā", + no="nē", + also="arī", + cause_word="jo", + effect_word="tāpēc", + or_word="vai", + true="patiess", + false="aplams", + neither="neviens no abiem", + ), Language.LIGURIAN: TranslationLiterals(language=Language.LIGURIAN), Language.LIMBURGISH: TranslationLiterals(language=Language.LIMBURGISH), Language.LINGALA: TranslationLiterals(language=Language.LINGALA), - Language.LITHUANIAN: TranslationLiterals(language=Language.LITHUANIAN), + Language.LITHUANIAN: TranslationLiterals( + language=Language.LITHUANIAN, + question_word="klausimas", + answer="atsakymas", + confirmation_word="ar ne", + yes="taip", + no="ne", + also="taip pat", + cause_word="nes", + effect_word="todėl", + or_word="arba", + true="teisingas", + false="neteisingas", + neither="nė vienas iš jų", + ), Language.LOMBARD: TranslationLiterals(language=Language.LOMBARD), Language.LUBA_KASAI: TranslationLiterals(language=Language.LUBA_KASAI), Language.LUO: TranslationLiterals(language=Language.LUO), @@ -806,7 +834,21 @@ def __getattribute__(self, name: str) -> str: Language.MALAGASY: TranslationLiterals(language=Language.MALAGASY), Language.MALAY: TranslationLiterals(language=Language.MALAY), Language.MALAYALAM: TranslationLiterals(language=Language.MALAYALAM), - Language.MALTESE: TranslationLiterals(language=Language.MALTESE), + Language.MALTESE: TranslationLiterals( + language=Language.MALTESE, + question_word="mistoqsija", + answer="tweġiba", + confirmation_word="hux hekk", + yes="iva", + no="le", + also="ukoll", + cause_word="għax", + effect_word="għalhekk", + or_word="jew", + true="veru", + false="falz", + neither="lanqas waħda", + ), Language.MAORI: TranslationLiterals(language=Language.MAORI), Language.MARATHI: TranslationLiterals(language=Language.MARATHI), Language.MEITEI: TranslationLiterals(language=Language.MEITEI), @@ -1070,7 +1112,21 @@ def __getattribute__(self, name: str) -> str: colon=":", semicolon=";", ), - Language.SLOVENIAN: TranslationLiterals(language=Language.SLOVENIAN), + Language.SLOVENIAN: TranslationLiterals( + language=Language.SLOVENIAN, + question_word="vprašanje", + answer="odgovor", + confirmation_word="kajne", + yes="da", + no="ne", + also="tudi", + cause_word="ker", + effect_word="zato", + or_word="ali", + true="resnično", + false="napačno", + neither="nobeno od naštetih", + ), Language.SOMALI: TranslationLiterals(language=Language.SOMALI), Language.SORANI: TranslationLiterals(language=Language.SORANI), Language.SOUTHERN_PASHTO: TranslationLiterals(language=Language.SOUTHERN_PASHTO), diff --git a/tests/unit/tasks/templates/test_multichoice.py b/tests/unit/tasks/templates/test_multichoice.py index 9b3614ed2..c4fb64160 100644 --- a/tests/unit/tasks/templates/test_multichoice.py +++ b/tests/unit/tasks/templates/test_multichoice.py @@ -20,6 +20,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import pytest + from lighteval.tasks.templates.multichoice import get_mcq_prompt_function from lighteval.tasks.templates.utils.formulation import CFFormulation, MCFFormulation from lighteval.utils.language import Language @@ -186,3 +188,37 @@ def test_multichoice_optional_keys(): Answer:\ """ ) + + +@pytest.mark.parametrize( + ("language", "question_word", "answer_word"), + [ + (Language.LATVIAN, "Jautājums", "Atbilde"), + (Language.LITHUANIAN, "Klausimas", "Atsakymas"), + (Language.MALTESE, "Mistoqsija", "Tweġiba"), + (Language.SLOVENIAN, "Vprašanje", "Odgovor"), + ], +) +def test_multichoice_prompt_cf_uses_language_literals(language, question_word, answer_word): + """Test that the CF prompt labels the question and the answer in the task's own language.""" + test_input = { + "question": "What is the capital of France?", + "choices": ["London", "Paris", "Berlin", "Madrid"], + "gold_idx": 1, + } + + prompt_fn = get_mcq_prompt_function( + language, + { + "question": "question", + "choices": "choices", + "gold_idx": "gold_idx", + }, + CFFormulation(), + ) + + doc = prompt_fn(test_input, "test_task") + + assert doc.query == f"{question_word}: What is the capital of France?\n{answer_word}:" + assert doc.unconditioned_query == f"{answer_word}:" + assert doc.choices == [" London", " Paris", " Berlin", " Madrid"]