Merge branch 'FlagOpen:master' into master

Author: ZiyiXia

FlagEmbedding/abc/finetune/embedder/AbsDataset.py

@@ -416,8 +416,9 @@ def _create_batch_data(self, batch_raw_data):
 
             passages.extend(tmp_passages)
 
-            if len(teacher_scores) > 0 and len(passages) > 0:
-                assert len(teacher_scores) == len(passages)
+            if teacher_scores is not None:
+                if len(teacher_scores) > 0 and len(passages) > 0:
+                    assert len(teacher_scores) == len(passages)
 
         return queries, passages, teacher_scores
 

FlagEmbedding/finetune/reranker/decoder_only/layerwise/load_model.py

@@ -8,6 +8,7 @@
 from FlagEmbedding.finetune.reranker.decoder_only.layerwise.arguments import RerankerModelArguments
 
 from .modeling_minicpm_reranker import LayerWiseMiniCPMForCausalLM, LayerWiseHead
+from .configuration_minicpm_reranker import LayerWiseMiniCPMConfig
 
 logger = logging.getLogger(__name__)
 
@@ -41,7 +42,7 @@ def get_model(model_args: RerankerModelArguments, only_for_one_logit):
         config = AutoConfig.from_pretrained(
             model_args.model_name_or_path,
             trust_remote_code=model_args.trust_remote_code,
-            token=model_args,
+            token=model_args.token,
             cache_dir=model_args.cache_dir
         )
     else:
@@ -61,7 +62,7 @@ def get_model(model_args: RerankerModelArguments, only_for_one_logit):
             trust_remote_code=model_args.trust_remote_code,
             # torch_dtype=torch.float16 if training_args.fp16 else torch.bfloat16,
             use_flash_attention_2=True if model_args.use_flash_attn else False,
-            token=model_args,
+            token=model_args.token,
             cache_dir=model_args.cache_dir,
             from_tf=bool(".ckpt" in model_args.model_name_or_path),
             config=config,
@@ -115,7 +116,7 @@ def get_model(model_args: RerankerModelArguments, only_for_one_logit):
             model_args.model_name_or_path,
             # torch_dtype=torch.float16 if training_args.fp16 else torch.bfloat16,
             use_flash_attention_2=True if model_args.use_flash_attn else False,
-            token=model_args,
+            token=model_args.token,
             cache_dir=model_args.cache_dir,
             from_tf=bool(".ckpt" in model_args.model_name_or_path),
             config=config,
@@ -155,14 +156,14 @@ def save_merged_model(model_args: RerankerModelArguments, output_dir: str):
         config = AutoConfig.from_pretrained(
             model_args.config_name,
             trust_remote_code=model_args.trust_remote_code,
-            token=model_args,
+            token=model_args.token,
             cache_dir=model_args.cache_dir
         )
     elif model_args.model_name_or_path:
         config = AutoConfig.from_pretrained(
             model_args.model_name_or_path,
             trust_remote_code=model_args.trust_remote_code,
-            token=model_args,
+            token=model_args.token,
             cache_dir=model_args.cache_dir
         )
     else:
@@ -172,19 +173,19 @@ def save_merged_model(model_args: RerankerModelArguments, output_dir: str):
     config.use_cache = False
 
     if model_args.model_type == 'from_raw_model':
-        config = AutoConfig.from_pretrained('BAAI/bge-reranker-v2-minicpm-layerwise',
-                                            cache_dir=model_args.cache_dir,
-                                            token=model_args,
-                                            trust_remote_code=model_args.trust_remote_code)
+        config = LayerWiseMiniCPMConfig.from_pretrained('BAAI/bge-reranker-v2-minicpm-layerwise',
+                                                        cache_dir=model_args.cache_dir,
+                                                        token=model_args.token,
+                                                        trust_remote_code=model_args.trust_remote_code)
         config.start_layer = model_args.start_layer
         config.head_multi = model_args.head_multi
         config.head_type = model_args.head_type
 
-    model = AutoModelForCausalLM.from_pretrained(model_args.model_name_or_path,
-                                                 config=config,
-                                                 cache_dir=model_args.cache_dir,
-                                                 token=model_args,
-                                                 trust_remote_code=model_args.trust_remote_code)
+    model = LayerWiseMiniCPMForCausalLM.from_pretrained(model_args.model_name_or_path,
+                                                        config=config,
+                                                        cache_dir=model_args.cache_dir,
+                                                        token=model_args.token,
+                                                        trust_remote_code=model_args.trust_remote_code)
 
     if model_args.raw_peft is not None:
         for peft_path in model_args.raw_peft:

FlagEmbedding/inference/reranker/decoder_only/models/__init__.py

@@ -8,7 +8,7 @@
 
 
 def sigmoid(x):
-    return 1 / (1 + np.exp(-x))
+    return float(1 / (1 + np.exp(-x)))
 
 
 class BaseReranker(AbsReranker):

FlagEmbedding/inference/reranker/encoder_only/base.py

@@ -92,20 +92,31 @@ It is the first embedding model which supports all three retrieval methods, achi
 
 ## Installation
 ### Using pip:
+If you do not want to finetune the models, you can install the package without the finetune dependency:
 ```
 pip install -U FlagEmbedding
 ```
+If you want to finetune the models, you can install the package with the finetune dependency:
+```
+pip install -U FlagEmbedding[finetune]
+```
 ### Install from sources:
 
 Clone the repository and install
 ```
 git clone https://github.com/FlagOpen/FlagEmbedding.git
 cd FlagEmbedding
+# If you do not want to finetune the models, you can install the package without the finetune dependency:
 pip install  .
+# If you want to finetune the models, you can install the package with the finetune dependency:
+# pip install  .[finetune]
 ```
 For development in editable mode:
 ```
+# If you do not want to finetune the models, you can install the package without the finetune dependency:
 pip install -e .
+# If you want to finetune the models, you can install the package with the finetune dependency:
+# pip install -e .[finetune]
 ```
 
 ## Quick Start

README.md

@@ -33,7 +33,8 @@
         <a href="#license">License</a> 
     <p>
 </h4>
-[English](README.md) | [中文](README_zh.md)
+
+[English](https://github.com/FlagOpen/FlagEmbedding/blob/master/README.md) | [中文](https://github.com/FlagOpen/FlagEmbedding/blob/master/README_zh.md)
 
 BGE (BAAI General Embedding) 专注于检索增强llm领域，目前包括以下项目:
 
@@ -85,20 +86,31 @@ BGE (BAAI General Embedding) 专注于检索增强llm领域，目前包括以下
 
 ## 安装
 ### 使用pip:
+如果你不想微调模型，你可以直接安装包，不用finetune依赖：
 ```
 pip install -U FlagEmbedding
 ```
+如果你想微调模型，你可以用finetune依赖安装：
+```
+pip install -U FlagEmbedding[finetune]
+```
 ### 从源文件安装部署:
 
 克隆并安装FlagEmbedding：
 ```
 git clone https://github.com/FlagOpen/FlagEmbedding.git
 cd FlagEmbedding
+# 如果你不想微调模型，你可以直接安装包，不用finetune依赖：
 pip install  .
+# 如果你想微调模型，你可以用finetune依赖安装：
+# pip install  .[finetune]
 ```
 在可编辑模式下安装:
 ```
+# 如果你不想微调模型，你可以直接安装包，不用finetune依赖：
 pip install -e .
+# 如果你想微调模型，你可以用finetune依赖安装：
+# pip install -e .[finetune]
 ```
 
 ## 快速开始

README_zh.md

@@ -1,30 +1,42 @@
-# 1. Introduction
+# Examples
+
+- [1. Introduction](#1-Introduction)
+- [2. Installation](#2-Installation)
+- [3. Inference](#3-Inference)
+- [4. Finetune](#4-Finetune)
+- [5. Evaluation](#5-Evaluation)
+
+## 1. Introduction
 
 In this example, we show how to **inference**, **finetune** and **evaluate** the baai-general-embedding.
 
-# 2. Installation
+## 2. Installation
 
 * **with pip**
+
 ```shell
 pip install -U FlagEmbedding
 ```
 
 * **from source**
+
 ```shell
 git clone https://github.com/FlagOpen/FlagEmbedding.git
 cd FlagEmbedding
 pip install  .
 ```
+
 For development, install as editable:
+
 ```shell
 pip install -e .
 ```
 
-# 3. Inference
+## 3. Inference
 
 We have provided the inference code for two types of models: the **embedder** and the **reranker**. These can be loaded using `FlagAutoModel` and `FlagAutoReranker`, respectively. For more detailed instructions on their use, please refer to the documentation for the [embedder](https://github.com/FlagOpen/FlagEmbedding/tree/master/examples/inference/embedder) and [reranker](https://github.com/FlagOpen/FlagEmbedding/tree/master/examples/inference/reranker).
 
-## 1. Embedder
+### 1. Embedder
 
 ```python
 from FlagEmbedding import FlagAutoModel
@@ -49,7 +61,7 @@ scores = q_embeddings @ p_embeddings.T
 print(scores)
 ```
 
-## 2. Reranker
+### 2. Reranker
 
 ```python
 from FlagEmbedding import FlagAutoReranker
@@ -65,11 +77,17 @@ scores = model.compute_score(pairs)
 print(scores)
 ```
 
-# 4. Finetune
+## 4. Finetune
 
 We support fine-tuning a variety of BGE series models, including `bge-large-en-v1.5`, `bge-m3`, `bge-en-icl`, `bge-multilingual-gemma2`, `bge-reranker-v2-m3`, `bge-reranker-v2-gemma`, and `bge-reranker-v2-minicpm-layerwise`, among others. As examples, we use the basic models `bge-large-en-v1.5` and `bge-reranker-large`. For more details, please refer to the [embedder](https://github.com/FlagOpen/FlagEmbedding/tree/master/examples/finetune/embedder) and [reranker](https://github.com/FlagOpen/FlagEmbedding/tree/master/examples/finetune/reranker) sections.
 
-## 1. Embedder
+If you do not have the `deepspeed` and `flash-attn` packages installed, you can install them with the following commands:
+```shell
+pip install deepspeed
+pip install flash-attn --no-build-isolation
+```
+
+### 1. Embedder
 
 ```shell
 torchrun --nproc_per_node 2 \
@@ -104,7 +122,7 @@ torchrun --nproc_per_node 2 \
     --kd_loss_type kl_div
 ```
 
-## 2. Reranker
+### 2. Reranker
 
 ```shell
 torchrun --nproc_per_node 2 \
@@ -134,16 +152,13 @@ torchrun --nproc_per_node 2 \
     --save_steps 1000
 ```
 
-# 5. Evaluation
+## 5. Evaluation
 
 We support evaluations on [MTEB](https://github.com/embeddings-benchmark/mteb), [BEIR](https://github.com/beir-cellar/beir), [MSMARCO](https://microsoft.github.io/msmarco/), [MIRACL](https://github.com/project-miracl/miracl), [MLDR](https://huggingface.co/datasets/Shitao/MLDR), [MKQA](https://github.com/apple/ml-mkqa), [AIR-Bench](https://github.com/AIR-Bench/AIR-Bench), and custom datasets. Below is an example of evaluating MSMARCO passages. For more details, please refer to the [evaluation examples](https://github.com/FlagOpen/FlagEmbedding/tree/master/examples/evaluation).
 
 ```shell
 pip install pytrec_eval
 pip install https://github.com/kyamagu/faiss-wheels/releases/download/v1.7.3/faiss_gpu-1.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
-```
-
-```shell
 python -m FlagEmbedding.evaluation.msmarco \
     --eval_name msmarco \
     --dataset_dir ./data/msmarco \