Fill in the translation literals for Latvian, Lithuanian, Maltese and Slovenian

src/lighteval/tasks/templates/utils/translation_literals.py

@@ -769,11 +769,39 @@ def __getattribute__(self, name: str) -> str:
     Language.LAO: TranslationLiterals(language=Language.LAO),
     Language.LATGALIAN: TranslationLiterals(language=Language.LATGALIAN),
     Language.LATIN: TranslationLiterals(language=Language.LATIN),
-    Language.LATVIAN: TranslationLiterals(language=Language.LATVIAN),
+    Language.LATVIAN: TranslationLiterals(
+        language=Language.LATVIAN,
+        question_word="jautājums",
+        answer="atbilde",
+        confirmation_word="vai ne",
+        yes="jā",
+        no="nē",
+        also="arī",
+        cause_word="jo",
+        effect_word="tāpēc",
+        or_word="vai",
+        true="patiess",
+        false="aplams",
+        neither="neviens no abiem",
+    ),
     Language.LIGURIAN: TranslationLiterals(language=Language.LIGURIAN),
     Language.LIMBURGISH: TranslationLiterals(language=Language.LIMBURGISH),
     Language.LINGALA: TranslationLiterals(language=Language.LINGALA),
-    Language.LITHUANIAN: TranslationLiterals(language=Language.LITHUANIAN),
+    Language.LITHUANIAN: TranslationLiterals(
+        language=Language.LITHUANIAN,
+        question_word="klausimas",
+        answer="atsakymas",
+        confirmation_word="ar ne",
+        yes="taip",
+        no="ne",
+        also="taip pat",
+        cause_word="nes",
+        effect_word="todėl",
+        or_word="arba",
+        true="teisingas",
+        false="neteisingas",
+        neither="nė vienas iš jų",
+    ),
     Language.LOMBARD: TranslationLiterals(language=Language.LOMBARD),
     Language.LUBA_KASAI: TranslationLiterals(language=Language.LUBA_KASAI),
     Language.LUO: TranslationLiterals(language=Language.LUO),
@@ -806,7 +834,21 @@ def __getattribute__(self, name: str) -> str:
     Language.MALAGASY: TranslationLiterals(language=Language.MALAGASY),
     Language.MALAY: TranslationLiterals(language=Language.MALAY),
     Language.MALAYALAM: TranslationLiterals(language=Language.MALAYALAM),
-    Language.MALTESE: TranslationLiterals(language=Language.MALTESE),
+    Language.MALTESE: TranslationLiterals(
+        language=Language.MALTESE,
+        question_word="mistoqsija",
+        answer="tweġiba",
+        confirmation_word="hux hekk",
+        yes="iva",
+        no="le",
+        also="ukoll",
+        cause_word="għax",
+        effect_word="għalhekk",
+        or_word="jew",
+        true="veru",
+        false="falz",
+        neither="lanqas waħda",
+    ),
     Language.MAORI: TranslationLiterals(language=Language.MAORI),
     Language.MARATHI: TranslationLiterals(language=Language.MARATHI),
     Language.MEITEI: TranslationLiterals(language=Language.MEITEI),
@@ -1070,7 +1112,21 @@ def __getattribute__(self, name: str) -> str:
         colon=":",
         semicolon=";",
     ),
-    Language.SLOVENIAN: TranslationLiterals(language=Language.SLOVENIAN),
+    Language.SLOVENIAN: TranslationLiterals(
+        language=Language.SLOVENIAN,
+        question_word="vprašanje",
+        answer="odgovor",
+        confirmation_word="kajne",
+        yes="da",
+        no="ne",
+        also="tudi",
+        cause_word="ker",
+        effect_word="zato",
+        or_word="ali",
+        true="resnično",
+        false="napačno",
+        neither="nobeno od naštetih",
+    ),
     Language.SOMALI: TranslationLiterals(language=Language.SOMALI),
     Language.SORANI: TranslationLiterals(language=Language.SORANI),
     Language.SOUTHERN_PASHTO: TranslationLiterals(language=Language.SOUTHERN_PASHTO),

tests/unit/tasks/templates/test_multichoice.py

@@ -20,6 +20,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+import pytest
+
 from lighteval.tasks.templates.multichoice import get_mcq_prompt_function
 from lighteval.tasks.templates.utils.formulation import CFFormulation, MCFFormulation
 from lighteval.utils.language import Language
@@ -186,3 +188,37 @@ def test_multichoice_optional_keys():
 Answer:\
 """
     )
+
+
+@pytest.mark.parametrize(
+    ("language", "question_word", "answer_word"),
+    [
+        (Language.LATVIAN, "Jautājums", "Atbilde"),
+        (Language.LITHUANIAN, "Klausimas", "Atsakymas"),
+        (Language.MALTESE, "Mistoqsija", "Tweġiba"),
+        (Language.SLOVENIAN, "Vprašanje", "Odgovor"),
+    ],
+)
+def test_multichoice_prompt_cf_uses_language_literals(language, question_word, answer_word):
+    """Test that the CF prompt labels the question and the answer in the task's own language."""
+    test_input = {
+        "question": "What is the capital of France?",
+        "choices": ["London", "Paris", "Berlin", "Madrid"],
+        "gold_idx": 1,
+    }
+
+    prompt_fn = get_mcq_prompt_function(
+        language,
+        {
+            "question": "question",
+            "choices": "choices",
+            "gold_idx": "gold_idx",
+        },
+        CFFormulation(),
+    )
+
+    doc = prompt_fn(test_input, "test_task")
+
+    assert doc.query == f"{question_word}: What is the capital of France?\n{answer_word}:"
+    assert doc.unconditioned_query == f"{answer_word}:"
+    assert doc.choices == [" London", " Paris", " Berlin", " Madrid"]