update custom eval

Author: 545999961

FlagEmbedding/evaluation/custom/data_loader.py

@@ -10,7 +10,7 @@
 logger = logging.getLogger(__name__)
 
 
-class MLDREvalDataLoader(AbsEvalDataLoader):
+class CustomEvalDataLoader(AbsEvalDataLoader):
     def available_dataset_names(self) -> List[str]:
         return []
 

FlagEmbedding/evaluation/custom/runner.py

@@ -1,11 +1,11 @@
 from FlagEmbedding.abc.evaluation import AbsEvalRunner
 
-from .data_loader import MLDREvalDataLoader
+from .data_loader import CustomEvalDataLoader
 
 
-class MLDREvalRunner(AbsEvalRunner):
-    def load_data_loader(self) -> MLDREvalDataLoader:
-        data_loader = MLDREvalDataLoader(
+class CustomEvalRunner(AbsEvalRunner):
+    def load_data_loader(self) -> CustomEvalDataLoader:
+        data_loader = CustomEvalDataLoader(
             eval_name=self.eval_args.eval_name,
             dataset_dir=self.eval_args.dataset_dir,
             cache_dir=self.eval_args.cache_path,

examples/evaluation/README.md

@@ -338,3 +338,28 @@ python -m FlagEmbedding.evaluation.custom \
     --reranker_query_max_length 512 \
     --reranker_max_length 1024 
 ```
+
+Please put the above file in `dataset_dir`, and then you can use the following code:
+
+```shell
+python -m FlagEmbedding.evaluation.custom \
+	--eval_name your_data_name \
+    --dataset_dir /share/chaofan/code/FlagEmbedding_update/data/beir/cqadupstack/android \
+    --splits test \
+    --corpus_embd_save_dir ./your_data_name/corpus_embd \
+    --output_dir ./your_data_name/search_results \
+    --search_top_k 1000 \
+    --rerank_top_k 100 \
+    --cache_path ./cache/data \
+    --overwrite False \
+    --k_values 10 100 \
+    --eval_output_method markdown \
+    --eval_output_path ./your_data_name/eval_results.md \
+    --eval_metrics ndcg_at_10 recall_at_100 \
+    --embedder_name_or_path BAAI/bge-m3 \
+    --reranker_name_or_path BAAI/bge-reranker-v2-m3 \
+    --devices cuda:0 cuda:1 \
+    --cache_dir /share/shared_models \
+    --reranker_query_max_length 512 \
+    --reranker_max_length 1024 
+```