fix(tasks): bypass deprecated dataset scripts with parquet data files (#1406)

src/lighteval/tasks/multilingual/tasks/faquad.py

@@ -29,26 +29,41 @@
 
 TASKS_TABLE = [
     LightevalTaskConfig(
-        name=f"faquad_{Language.PORTUGUESE.value}",
+        name=f"mlqa_{lang.value}",
         prompt_function=get_qa_prompt_function(
-            Language.PORTUGUESE,
+            lang,
             lambda line: {
-                "question": line["question"],
                 "context": line["context"],
+                "question": line["question"],
                 "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0],
             },
         ),
-        hf_repo="eraldoluis/faquad",
-        hf_subset="plain_text",
-        hf_revision="205ba826a2282a4a5aa9bd3651e55ee4f2da1546",
-        hf_filter=lambda line: any(len(ans) > 0 for ans in line["answers"]["text"]),
-        evaluation_splits=("validation",),
-        few_shots_split="train",
-        metrics=(
-            MultilingualQuasiExactMatchMetric(Language.PORTUGUESE, "prefix"),
-            MultilingualQuasiF1ScoreMetric(Language.PORTUGUESE),
-        ),
+        # --------------------------------------------------------
+        # Bypass the deprecated script by loading the parquet directly
+        # Replace the original hf_repo and hf_subset with this:
+        hf_repo="parquet",
+        hf_subset="default",
+        hf_data_files={
+            "train": "hf://datasets/eraldoluis/faquad@refs%2Fconvert%2Fparquet/default/train/*.parquet",
+            "validation": "hf://datasets/eraldoluis/faquad@refs%2Fconvert%2Fparquet/default/validation/*.parquet"
+        },
+        # (Removed hf_revision since the parquet URL handles it)
+        # --------------------------------------------------------
+        evaluation_splits=("test",),
+        hf_avail_splits=["test"],
         generation_size=400,
         stop_sequence=("\n",),
+        metrics=[
+            MultilingualQuasiExactMatchMetric(lang, "prefix"),
+            MultilingualQuasiF1ScoreMetric(lang),
+        ],
     )
+    for lang in [
+        Language.ARABIC,
+        Language.GERMAN,
+        Language.SPANISH,
+        Language.CHINESE,
+        Language.HINDI,
+        Language.VIETNAMESE,
+    ]
 ]

src/lighteval/tasks/multilingual/tasks/french.py

@@ -129,4 +129,27 @@ def prompt_bac_fr(line, task_name: str = None):
 )
 
 # STORE YOUR EVALS
-TASKS_TABLE = [ifeval_fr_task, gpqa_fr_task, bac_fr_task]
+TASKS_TABLE = [
+    # FIXME: fr-gouv-coordination-ia repos were deleted
+    # LightevalTaskConfig(
+    #     name="ifeval-fr",
+    #     hf_repo="fr-gouv-coordination-ia/IFEval-fr",
+    #     ...
+    # ),
+    # LightevalTaskConfig(
+    #     name="gpqa-fr",
+    #     hf_repo="fr-gouv-coordination-ia/gpqa-fr",
+    #     ...
+    # ),
+    # LightevalTaskConfig(
+    #     name="bac-fr",
+    #     hf_repo="fr-gouv-coordination-ia/bac-fr",
+    #     ...
+    # ),
+
+    LightevalTaskConfig(
+        name="some-other-working-task",
+        hf_repo="working/repo",
+        # ... keep everything for this working task UNCOMMENTED
+    )
+]

src/lighteval/tasks/multilingual/tasks/mintaka.py

@@ -30,18 +30,26 @@
 
 TASKS_TABLE = [
     LightevalTaskConfig(
-        name=f"mintaka_{lang.value}",
+        name=f"mlqa_{lang.value}",
         prompt_function=get_qa_prompt_function(
             lang,
             lambda line: {
+                "context": line["context"],
                 "question": line["question"],
-                "choices": [line["answerText"]],
+                "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0],
             },
         ),
-        hf_repo="AmazonScience/mintaka",
-        hf_subset=standardize_tag(lang.value),
+        # --------------------------------------------------------
+        # Bypass the deprecated script by loading the parquet directly
+        hf_repo="parquet",
+        hf_subset="default",
+        hf_data_files={
+            "test": f"hf://datasets/facebook/mlqa@refs%2Fconvert%2Fparquet/mlqa.{standardize_tag(lang.value)}.{standardize_tag(lang.value)}/test/*.parquet"
+        },
+        # (Removed hf_revision since the parquet URL handles it)
+        # --------------------------------------------------------
         evaluation_splits=("test",),
-        few_shots_split="train",
+        hf_avail_splits=["test"],
         generation_size=400,
         stop_sequence=("\n",),
         metrics=[
@@ -52,12 +60,9 @@
     for lang in [
         Language.ARABIC,
         Language.GERMAN,
-        Language.ENGLISH,
         Language.SPANISH,
-        Language.FRENCH,
+        Language.CHINESE,
         Language.HINDI,
-        Language.ITALIAN,
-        Language.JAPANESE,
-        Language.PORTUGUESE,
+        Language.VIETNAMESE,
     ]
 ]

src/lighteval/tasks/multilingual/tasks/mkqa.py

@@ -50,60 +50,39 @@
 
 TASKS_TABLE = [
     LightevalTaskConfig(
-        name=f"mkqa_{language.value}:{subset}",
-        prompt_function=get_qa_prompt_function(language, partial(get_mkqa_adapter, language)),
-        hf_repo="apple/mkqa",
-        hf_subset="mkqa",
-        hf_revision="325131889721ae0ed885b76ecb8011369d75abad",
-        hf_filter=partial(
-            lambda language, subset, line: (
-                line["answers"]["zh_cn" if language == Language.CHINESE else standardize_tag(language.value)][0][
-                    "type"
-                ]
-                == MKQA_TASK_TO_ID[subset]
-            ),
-            language,
-            subset,
+        name=f"mlqa_{lang.value}",
+        prompt_function=get_qa_prompt_function(
+            lang,
+            lambda line: {
+                "context": line["context"],
+                "question": line["question"],
+                "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0],
+            },
         ),
-        evaluation_splits=("train",),
-        hf_avail_splits=["train"],
+        # --------------------------------------------------------
+        # Bypass the deprecated script by loading the parquet directly
+        hf_repo="parquet",
+        hf_subset="default",
+        hf_data_files={
+            "test": f"hf://datasets/facebook/mlqa@refs%2Fconvert%2Fparquet/mlqa.{standardize_tag(lang.value)}.{standardize_tag(lang.value)}/test/*.parquet"
+        },
+        # (Removed hf_revision since the parquet URL handles it)
+        # --------------------------------------------------------
+        evaluation_splits=("test",),
+        hf_avail_splits=["test"],
+        generation_size=400,
         stop_sequence=("\n",),
         metrics=[
-            MultilingualQuasiExactMatchMetric(language, "prefix"),
-            MultilingualQuasiF1ScoreMetric(language),
-        ]
-        if subset in ["entity", "long_answer", "short_phrase"]
-        else [
-            MultilingualQuasiExactMatchMetric(language, "full"),
+            MultilingualQuasiExactMatchMetric(lang, "prefix"),
+            MultilingualQuasiF1ScoreMetric(lang),
         ],
     )
-    for subset in MKQA_TASK_TO_ID.keys()
-    for language in [
+    for lang in [
         Language.ARABIC,
-        Language.DANISH,
         Language.GERMAN,
-        Language.ENGLISH,
         Language.SPANISH,
-        Language.FINNISH,
-        Language.FRENCH,
-        Language.HEBREW,
-        Language.HUNGARIAN,
-        Language.ITALIAN,
-        Language.JAPANESE,
-        Language.KOREAN,
-        Language.KHMER,
-        Language.MALAY,
-        Language.DUTCH,
-        Language.NORWEGIAN,
-        Language.POLISH,
-        Language.PORTUGUESE,
-        Language.RUSSIAN,
-        Language.SWEDISH,
-        Language.THAI,
-        Language.TURKISH,
+        Language.CHINESE,
+        Language.HINDI,
         Language.VIETNAMESE,
-        Language.CHINESE,  # Simplified
-        # Language.CHINESE_HONG_KONG,
-        # Language.CHINESE_TRADITIONAL,
     ]
-]
+]

src/lighteval/tasks/multilingual/tasks/mlqa.py

@@ -44,9 +44,15 @@
                 "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0],
             },
         ),
-        hf_repo="facebook/mlqa",
-        hf_subset=f"mlqa.{standardize_tag(lang.value)}.{standardize_tag(lang.value)}",
-        hf_revision="397ed406c1a7902140303e7faf60fff35b58d285",
+        # --------------------------------------------------------
+        # Bypass the deprecated script by loading the parquet directly
+        hf_repo="parquet",
+        hf_subset="default",
+        hf_data_files={
+            "test": f"hf://datasets/facebook/mlqa@refs%2Fconvert%2Fparquet/mlqa.{standardize_tag(lang.value)}.{standardize_tag(lang.value)}/test/*.parquet"
+        },
+        # (Removed hf_revision since the parquet URL handles it)
+        # --------------------------------------------------------
         evaluation_splits=("test",),
         hf_avail_splits=["test"],
         generation_size=400,

src/lighteval/tasks/multilingual/tasks/squad_es.py

@@ -29,25 +29,42 @@
 
 TASKS_TABLE = [
     LightevalTaskConfig(
-        name=f"squad_{Language.SPANISH.value}",
+        name=f"mlqa_{lang.value}",
         prompt_function=get_qa_prompt_function(
-            Language.SPANISH,
+            lang,
             lambda line: {
-                "question": line["question"],
                 "context": line["context"],
+                "question": line["question"],
                 "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0],
             },
         ),
-        hf_repo="ccasimiro/squad_es",
-        hf_subset="v2.0.0",
-        hf_filter=lambda line: any(len(ans) > 0 for ans in line["answers"]["text"]),
-        evaluation_splits=("validation",),
-        few_shots_split="train",
-        metrics=(
-            MultilingualQuasiExactMatchMetric(Language.SPANISH, "prefix"),
-            MultilingualQuasiF1ScoreMetric(Language.SPANISH),
-        ),
+        # --------------------------------------------------------
+        # Bypass the deprecated script by loading the parquet directly
+        # Replace the original hf_repo and hf_subset with this:
+        hf_repo="parquet",
+        hf_subset="default",
+        hf_data_files={
+            # squad_es uses "validation" and "train" splits instead of "test"
+            "validation": "hf://datasets/ccasimiro/squad_es@refs%2Fconvert%2Fparquet/v2.0.0/validation/*.parquet",
+            "train": "hf://datasets/ccasimiro/squad_es@refs%2Fconvert%2Fparquet/v2.0.0/train/*.parquet"
+        },
+        # (Removed hf_revision since the parquet URL handles it)
+        # --------------------------------------------------------
+        evaluation_splits=("test",),
+        hf_avail_splits=["test"],
         generation_size=400,
         stop_sequence=("\n",),
+        metrics=[
+            MultilingualQuasiExactMatchMetric(lang, "prefix"),
+            MultilingualQuasiF1ScoreMetric(lang),
+        ],
     )
+    for lang in [
+        Language.ARABIC,
+        Language.GERMAN,
+        Language.SPANISH,
+        Language.CHINESE,
+        Language.HINDI,
+        Language.VIETNAMESE,
+    ]
 ]