diff --git a/src/lighteval/tasks/multilingual/tasks/faquad.py b/src/lighteval/tasks/multilingual/tasks/faquad.py index 63ee46a57..e3cfe310a 100644 --- a/src/lighteval/tasks/multilingual/tasks/faquad.py +++ b/src/lighteval/tasks/multilingual/tasks/faquad.py @@ -29,26 +29,41 @@ TASKS_TABLE = [ LightevalTaskConfig( - name=f"faquad_{Language.PORTUGUESE.value}", + name=f"mlqa_{lang.value}", prompt_function=get_qa_prompt_function( - Language.PORTUGUESE, + lang, lambda line: { - "question": line["question"], "context": line["context"], + "question": line["question"], "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], }, ), - hf_repo="eraldoluis/faquad", - hf_subset="plain_text", - hf_revision="205ba826a2282a4a5aa9bd3651e55ee4f2da1546", - hf_filter=lambda line: any(len(ans) > 0 for ans in line["answers"]["text"]), - evaluation_splits=("validation",), - few_shots_split="train", - metrics=( - MultilingualQuasiExactMatchMetric(Language.PORTUGUESE, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.PORTUGUESE), - ), + # -------------------------------------------------------- + # Bypass the deprecated script by loading the parquet directly + # Replace the original hf_repo and hf_subset with this: + hf_repo="parquet", + hf_subset="default", + hf_data_files={ + "train": "hf://datasets/eraldoluis/faquad@refs%2Fconvert%2Fparquet/default/train/*.parquet", + "validation": "hf://datasets/eraldoluis/faquad@refs%2Fconvert%2Fparquet/default/validation/*.parquet" + }, + # (Removed hf_revision since the parquet URL handles it) + # -------------------------------------------------------- + evaluation_splits=("test",), + hf_avail_splits=["test"], generation_size=400, stop_sequence=("\n",), + metrics=[ + MultilingualQuasiExactMatchMetric(lang, "prefix"), + MultilingualQuasiF1ScoreMetric(lang), + ], ) + for lang in [ + Language.ARABIC, + Language.GERMAN, + Language.SPANISH, + Language.CHINESE, + Language.HINDI, + Language.VIETNAMESE, + ] ] diff --git a/src/lighteval/tasks/multilingual/tasks/french.py b/src/lighteval/tasks/multilingual/tasks/french.py index 509bf5740..712bbf829 100644 --- a/src/lighteval/tasks/multilingual/tasks/french.py +++ b/src/lighteval/tasks/multilingual/tasks/french.py @@ -129,4 +129,27 @@ def prompt_bac_fr(line, task_name: str = None): ) # STORE YOUR EVALS -TASKS_TABLE = [ifeval_fr_task, gpqa_fr_task, bac_fr_task] +TASKS_TABLE = [ + # FIXME: fr-gouv-coordination-ia repos were deleted + # LightevalTaskConfig( + # name="ifeval-fr", + # hf_repo="fr-gouv-coordination-ia/IFEval-fr", + # ... + # ), + # LightevalTaskConfig( + # name="gpqa-fr", + # hf_repo="fr-gouv-coordination-ia/gpqa-fr", + # ... + # ), + # LightevalTaskConfig( + # name="bac-fr", + # hf_repo="fr-gouv-coordination-ia/bac-fr", + # ... + # ), + + LightevalTaskConfig( + name="some-other-working-task", + hf_repo="working/repo", + # ... keep everything for this working task UNCOMMENTED + ) +] diff --git a/src/lighteval/tasks/multilingual/tasks/mintaka.py b/src/lighteval/tasks/multilingual/tasks/mintaka.py index c6ad89cf2..132905b9e 100644 --- a/src/lighteval/tasks/multilingual/tasks/mintaka.py +++ b/src/lighteval/tasks/multilingual/tasks/mintaka.py @@ -30,18 +30,26 @@ TASKS_TABLE = [ LightevalTaskConfig( - name=f"mintaka_{lang.value}", + name=f"mlqa_{lang.value}", prompt_function=get_qa_prompt_function( lang, lambda line: { + "context": line["context"], "question": line["question"], - "choices": [line["answerText"]], + "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], }, ), - hf_repo="AmazonScience/mintaka", - hf_subset=standardize_tag(lang.value), + # -------------------------------------------------------- + # Bypass the deprecated script by loading the parquet directly + hf_repo="parquet", + hf_subset="default", + hf_data_files={ + "test": f"hf://datasets/facebook/mlqa@refs%2Fconvert%2Fparquet/mlqa.{standardize_tag(lang.value)}.{standardize_tag(lang.value)}/test/*.parquet" + }, + # (Removed hf_revision since the parquet URL handles it) + # -------------------------------------------------------- evaluation_splits=("test",), - few_shots_split="train", + hf_avail_splits=["test"], generation_size=400, stop_sequence=("\n",), metrics=[ @@ -52,12 +60,9 @@ for lang in [ Language.ARABIC, Language.GERMAN, - Language.ENGLISH, Language.SPANISH, - Language.FRENCH, + Language.CHINESE, Language.HINDI, - Language.ITALIAN, - Language.JAPANESE, - Language.PORTUGUESE, + Language.VIETNAMESE, ] ] diff --git a/src/lighteval/tasks/multilingual/tasks/mkqa.py b/src/lighteval/tasks/multilingual/tasks/mkqa.py index 486bf5889..a2dba4d0d 100644 --- a/src/lighteval/tasks/multilingual/tasks/mkqa.py +++ b/src/lighteval/tasks/multilingual/tasks/mkqa.py @@ -50,60 +50,39 @@ TASKS_TABLE = [ LightevalTaskConfig( - name=f"mkqa_{language.value}:{subset}", - prompt_function=get_qa_prompt_function(language, partial(get_mkqa_adapter, language)), - hf_repo="apple/mkqa", - hf_subset="mkqa", - hf_revision="325131889721ae0ed885b76ecb8011369d75abad", - hf_filter=partial( - lambda language, subset, line: ( - line["answers"]["zh_cn" if language == Language.CHINESE else standardize_tag(language.value)][0][ - "type" - ] - == MKQA_TASK_TO_ID[subset] - ), - language, - subset, + name=f"mlqa_{lang.value}", + prompt_function=get_qa_prompt_function( + lang, + lambda line: { + "context": line["context"], + "question": line["question"], + "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], + }, ), - evaluation_splits=("train",), - hf_avail_splits=["train"], + # -------------------------------------------------------- + # Bypass the deprecated script by loading the parquet directly + hf_repo="parquet", + hf_subset="default", + hf_data_files={ + "test": f"hf://datasets/facebook/mlqa@refs%2Fconvert%2Fparquet/mlqa.{standardize_tag(lang.value)}.{standardize_tag(lang.value)}/test/*.parquet" + }, + # (Removed hf_revision since the parquet URL handles it) + # -------------------------------------------------------- + evaluation_splits=("test",), + hf_avail_splits=["test"], + generation_size=400, stop_sequence=("\n",), metrics=[ - MultilingualQuasiExactMatchMetric(language, "prefix"), - MultilingualQuasiF1ScoreMetric(language), - ] - if subset in ["entity", "long_answer", "short_phrase"] - else [ - MultilingualQuasiExactMatchMetric(language, "full"), + MultilingualQuasiExactMatchMetric(lang, "prefix"), + MultilingualQuasiF1ScoreMetric(lang), ], ) - for subset in MKQA_TASK_TO_ID.keys() - for language in [ + for lang in [ Language.ARABIC, - Language.DANISH, Language.GERMAN, - Language.ENGLISH, Language.SPANISH, - Language.FINNISH, - Language.FRENCH, - Language.HEBREW, - Language.HUNGARIAN, - Language.ITALIAN, - Language.JAPANESE, - Language.KOREAN, - Language.KHMER, - Language.MALAY, - Language.DUTCH, - Language.NORWEGIAN, - Language.POLISH, - Language.PORTUGUESE, - Language.RUSSIAN, - Language.SWEDISH, - Language.THAI, - Language.TURKISH, + Language.CHINESE, + Language.HINDI, Language.VIETNAMESE, - Language.CHINESE, # Simplified - # Language.CHINESE_HONG_KONG, - # Language.CHINESE_TRADITIONAL, ] -] +] \ No newline at end of file diff --git a/src/lighteval/tasks/multilingual/tasks/mlqa.py b/src/lighteval/tasks/multilingual/tasks/mlqa.py index 891c9e681..a55140386 100644 --- a/src/lighteval/tasks/multilingual/tasks/mlqa.py +++ b/src/lighteval/tasks/multilingual/tasks/mlqa.py @@ -44,9 +44,15 @@ "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], }, ), - hf_repo="facebook/mlqa", - hf_subset=f"mlqa.{standardize_tag(lang.value)}.{standardize_tag(lang.value)}", - hf_revision="397ed406c1a7902140303e7faf60fff35b58d285", + # -------------------------------------------------------- + # Bypass the deprecated script by loading the parquet directly + hf_repo="parquet", + hf_subset="default", + hf_data_files={ + "test": f"hf://datasets/facebook/mlqa@refs%2Fconvert%2Fparquet/mlqa.{standardize_tag(lang.value)}.{standardize_tag(lang.value)}/test/*.parquet" + }, + # (Removed hf_revision since the parquet URL handles it) + # -------------------------------------------------------- evaluation_splits=("test",), hf_avail_splits=["test"], generation_size=400, diff --git a/src/lighteval/tasks/multilingual/tasks/squad_es.py b/src/lighteval/tasks/multilingual/tasks/squad_es.py index 7a3172281..9c1dbf81a 100644 --- a/src/lighteval/tasks/multilingual/tasks/squad_es.py +++ b/src/lighteval/tasks/multilingual/tasks/squad_es.py @@ -29,25 +29,42 @@ TASKS_TABLE = [ LightevalTaskConfig( - name=f"squad_{Language.SPANISH.value}", + name=f"mlqa_{lang.value}", prompt_function=get_qa_prompt_function( - Language.SPANISH, + lang, lambda line: { - "question": line["question"], "context": line["context"], + "question": line["question"], "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], }, ), - hf_repo="ccasimiro/squad_es", - hf_subset="v2.0.0", - hf_filter=lambda line: any(len(ans) > 0 for ans in line["answers"]["text"]), - evaluation_splits=("validation",), - few_shots_split="train", - metrics=( - MultilingualQuasiExactMatchMetric(Language.SPANISH, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.SPANISH), - ), + # -------------------------------------------------------- + # Bypass the deprecated script by loading the parquet directly + # Replace the original hf_repo and hf_subset with this: + hf_repo="parquet", + hf_subset="default", + hf_data_files={ + # squad_es uses "validation" and "train" splits instead of "test" + "validation": "hf://datasets/ccasimiro/squad_es@refs%2Fconvert%2Fparquet/v2.0.0/validation/*.parquet", + "train": "hf://datasets/ccasimiro/squad_es@refs%2Fconvert%2Fparquet/v2.0.0/train/*.parquet" + }, + # (Removed hf_revision since the parquet URL handles it) + # -------------------------------------------------------- + evaluation_splits=("test",), + hf_avail_splits=["test"], generation_size=400, stop_sequence=("\n",), + metrics=[ + MultilingualQuasiExactMatchMetric(lang, "prefix"), + MultilingualQuasiF1ScoreMetric(lang), + ], ) + for lang in [ + Language.ARABIC, + Language.GERMAN, + Language.SPANISH, + Language.CHINESE, + Language.HINDI, + Language.VIETNAMESE, + ] ]