Merge pull request #601 from nntoan209/fix-split-data

Fix "idx" bug in split_data_by_length.py of BGE-M3

Author: staoxiao

FlagEmbedding/BGE_M3/split_data_by_length.py

@@ -130,7 +130,7 @@ def _process_file(self, file_path: str, output_path: str):
                 continue
 
             idxs = mapped_dataset.filter(lambda x: length_l <= x['max_length'] < length_r, num_proc=self.num_proc)
-            split_dataset = dataset.select(idxs['idx'])
+            split_dataset = dataset.select(list(idxs._indices.to_pandas()['indices'].values))
 
             split_info_dict[f'len-{length_l}-{length_r}'] = len(split_dataset)