update model_class parameters in evaluation

- delete 'auto' option
- support import `EmbedderModelClass` and `RerankerModelClass`

Author: hanhainebula

FlagEmbedding/abc/evaluation/arguments.py

@@ -81,7 +81,7 @@ class AbsEvalModelArgs:
         metadata={"help": "The embedder name or path.", "required": True}
     )
     embedder_model_class: Optional[str] = field(
-        default="auto", metadata={"help": "The embedder model class. Available classes: ['auto', 'encoder-only-base', 'encoder-only-m3', 'decoder-only-base', 'decoder-only-icl']. Default: auto.", "choices": ["auto", "encoder-only-base", "encoder-only-m3", "decoder-only-base", "decoder-only-icl"]}
+        default=None, metadata={"help": "The embedder model class. Available classes: ['encoder-only-base', 'encoder-only-m3', 'decoder-only-base', 'decoder-only-icl']. Default: None. For the custom model, you need to specifiy the model class.", "choices": ["encoder-only-base", "encoder-only-m3", "decoder-only-base", "decoder-only-icl"]}
     )
     normalize_embeddings: bool = field(
         default=True, metadata={"help": "whether to normalize the embeddings"}
@@ -114,7 +114,7 @@ class AbsEvalModelArgs:
         default=None, metadata={"help": "The reranker name or path."}
     )
     reranker_model_class: Optional[str] = field(
-        default="auto", metadata={"help": "The reranker model class. Available classes: ['auto', 'encoder-only-base', 'decoder-only-base', 'decoder-only-layerwise', 'decoder-only-lightweight']. Default: auto.", "choices": ["auto", "encoder-only-base", "decoder-only-base", "decoder-only-layerwise", "decoder-only-lightweight"]}
+        default=None, metadata={"help": "The reranker model class. Available classes: ['encoder-only-base', 'decoder-only-base', 'decoder-only-layerwise', 'decoder-only-lightweight']. Default: None. For the custom model, you need to specify the model class.", "choices": ["encoder-only-base", "decoder-only-base", "decoder-only-layerwise", "decoder-only-lightweight"]}
     )
     reranker_peft_path: Optional[str] = field(
         default=None, metadata={"help": "The reranker peft path."}

FlagEmbedding/inference/__init__.py

@@ -2,17 +2,21 @@
 from .auto_reranker import FlagAutoReranker
 from .embedder import (
     FlagModel, BGEM3FlagModel,
-    FlagICLModel, FlagLLMModel
+    FlagICLModel, FlagLLMModel,
+    EmbedderModelClass
 )
 from .reranker import (
     FlagReranker,
-    FlagLLMReranker, LayerWiseFlagLLMReranker, LightWeightFlagLLMReranker
+    FlagLLMReranker, LayerWiseFlagLLMReranker, LightWeightFlagLLMReranker,
+    RerankerModelClass
 )
 
 
 __all__ = [
     "FlagAutoModel",
     "FlagAutoReranker",
+    "EmbedderModelClass",
+    "RerankerModelClass",
     "FlagModel",
     "BGEM3FlagModel",
     "FlagICLModel",

FlagEmbedding/inference/auto_embedder.py

@@ -34,7 +34,7 @@ def from_finetuned(
         if model_name.startswith("checkpoint-"):
             model_name = os.path.basename(os.path.dirname(model_name_or_path))
 
-        if model_class is not None and model_class != 'auto':
+        if model_class is not None:
             _model_class = EMBEDDER_CLASS_MAPPING[EmbedderModelClass(model_class)]
             if pooling_method is None:
                 pooling_method = _model_class.DEFAULT_POOLING_METHOD

FlagEmbedding/inference/auto_reranker.py

@@ -30,7 +30,7 @@ def from_finetuned(
         if model_name.startswith("checkpoint-"):
             model_name = os.path.basename(os.path.dirname(model_name_or_path))
 
-        if model_class is not None and model_class != 'auto':
+        if model_class is not None:
             _model_class = RERANKER_CLASS_MAPPING[RerankerModelClass(model_class)]
             if trust_remote_code is None:
                 trust_remote_code = False

FlagEmbedding/inference/embedder/__init__.py

@@ -1,9 +1,11 @@
 from .encoder_only import FlagModel, BGEM3FlagModel
 from .decoder_only import FlagICLModel, FlagLLMModel
+from .model_mapping import EmbedderModelClass
 
 __all__ = [
     "FlagModel",
     "BGEM3FlagModel",
     "FlagICLModel",
     "FlagLLMModel",
+    "EmbedderModelClass",
 ]

FlagEmbedding/inference/reranker/__init__.py

@@ -1,9 +1,11 @@
 from .decoder_only import FlagLLMReranker, LayerWiseFlagLLMReranker, LightWeightFlagLLMReranker
 from .encoder_only import FlagReranker
+from .model_mapping import RerankerModelClass
 
 __all__ = [
     "FlagReranker",
     "FlagLLMReranker",
     "LayerWiseFlagLLMReranker",
     "LightWeightFlagLLMReranker",
+    "RerankerModelClass",
 ]