Merge branch 'FlagOpen:master' into master

Author: muazhari

C_MTEB/MKQA/dense_retrieval/step0-generate_embedding.py

@@ -5,7 +5,6 @@
 --max_passage_length 512 \
 --batch_size 256 \
 --fp16 \
---add_instruction False \
 --pooling_method cls \
 --normalize_embeddings True
 """
@@ -35,14 +34,6 @@ class ModelArgs:
         default=True,
         metadata={'help': 'Use fp16 in inference?'}
     )
-    add_instruction: bool = field(
-        default=False,
-        metadata={'help': 'Add instruction?'}
-    )
-    passage_instruction_for_retrieval: str = field(
-        default=None,
-        metadata={'help': 'passage instruction for retrieval'}
-    )
     pooling_method: str = field(
         default='cls',
         metadata={'help': "Pooling method. Avaliable methods: 'cls', 'mean'"}
@@ -78,8 +69,6 @@ def get_model(model_args: ModelArgs):
         model_args.encoder, 
         pooling_method=model_args.pooling_method,
         normalize_embeddings=model_args.normalize_embeddings,
-        # query_instruction_for_retrieval=model_args.query_instruction_for_retrieval if model_args.add_instruction else None,
-        passage_instruction_for_retrieval=model_args.passage_instruction_for_retrieval if model_args.add_instruction else None,
         use_fp16=model_args.fp16
     )
     return model

C_MTEB/MLDR/dense_retrieval/step0-generate_embedding.py

@@ -6,7 +6,6 @@
 --max_passage_length 8192 \
 --batch_size 4 \
 --fp16 \
---add_instruction False \
 --pooling_method cls \
 --normalize_embeddings True
 """
@@ -30,14 +29,6 @@ class ModelArgs:
         default=True,
         metadata={'help': 'Use fp16 in inference?'}
     )
-    add_instruction: bool = field(
-        default=False,
-        metadata={'help': 'Add instruction?'}
-    )
-    passage_instruction_for_retrieval: str = field(
-        default=None,
-        metadata={'help': 'passage instruction for retrieval'}
-    )
     pooling_method: str = field(
         default='cls',
         metadata={'help': "Pooling method. Avaliable methods: 'cls', 'mean'"}
@@ -78,8 +69,6 @@ def get_model(model_args: ModelArgs):
         model_args.encoder, 
         pooling_method=model_args.pooling_method,
         normalize_embeddings=model_args.normalize_embeddings,
-        # query_instruction_for_retrieval=model_args.query_instruction_for_retrieval if model_args.add_instruction else None,
-        passage_instruction_for_retrieval=model_args.passage_instruction_for_retrieval if model_args.add_instruction else None,
         use_fp16=model_args.fp16
     )
     return model

C_MTEB/setup.py

@@ -5,14 +5,14 @@
 
 setup(
     name='C_MTEB',
-    version='1.1.0',
+    version='1.1.1',
     description='Chinese Massive Text Embedding Benchmark',
     long_description=readme,
     long_description_content_type="text/markdown",
     author_email='2906698981@qq.com',
     url='https://github.com/FlagOpen/FlagEmbedding/tree/master/C_MTEB',
     packages=find_packages(),
     install_requires=[
-        'mteb[beir]',
+        'mteb[beir]==1.1.1',
     ],
 )

FlagEmbedding/BGE_M3/modeling.py

@@ -252,7 +252,7 @@ def forward(self, query: Dict[str, Tensor] = None, passage: Dict[str, Tensor] =
 
             else:
                 idxs = torch.arange(q_dense_vecs.size(0), device=q_dense_vecs.device, dtype=torch.long)
-                targets = idxs * (p_sparse_vecs.size(0) // q_sparse_vecs.size(0))
+                targets = idxs * (p_dense_vecs.size(0) // q_dense_vecs.size(0))
 
                 # dense loss
                 dense_scores = self.dense_score(q_dense_vecs, p_dense_vecs)  # B, B * N
@@ -325,8 +325,11 @@ def _trans_state_dict(state_dict):
 
         self.model.save_pretrained(output_dir, state_dict=_trans_state_dict(self.model.state_dict()))
 
-        torch.save(_trans_state_dict(self.colbert_linear.state_dict()), os.path.join(output_dir, 'colbert_linear.pt'))
-        torch.save(_trans_state_dict(self.sparse_linear.state_dict()), os.path.join(output_dir, 'sparse_linear.pt'))
+        if self.unified_finetuning:
+            torch.save(_trans_state_dict(self.colbert_linear.state_dict()),
+                       os.path.join(output_dir, 'colbert_linear.pt'))
+            torch.save(_trans_state_dict(self.sparse_linear.state_dict()),
+                       os.path.join(output_dir, 'sparse_linear.pt'))
 
     def load_pooler(self, model_dir):
         colbert_state_dict = torch.load(os.path.join(model_dir, 'colbert_linear.pt'), map_location='cpu')

FlagEmbedding/baai_general_embedding/finetune/hn_mine.py

@@ -100,7 +100,9 @@ def find_knn_neg(model, input_file, candidate_pool, output_file, sample_range, n
     with open(output_file, 'w') as f:
         for data in train_data:
             if len(data['neg']) < negative_number:
-                data['neg'].extend(random.sample(corpus, negative_number - len(data['neg'])))
+                samples = random.sample(corpus, negative_number - len(data['neg']) + len(data['pos']))
+                samples = [sent for sent in samples if sent not in data['pos']]
+                data['neg'].extend(samples[: negative_number - len(data['neg'])])
             f.write(json.dumps(data, ensure_ascii=False) + '\n')
 
 

FlagEmbedding/baai_general_embedding/finetune/modeling.py

@@ -43,6 +43,9 @@ def __init__(self,
         if not normlized:
             self.temperature = 1.0
             logger.info("reset temperature = 1.0 due to using inner product to compute similarity")
+        if normlized:
+            if self.temperature > 0.5:
+                raise ValueError("Temperature should be smaller than 1.0 when use cosine similarity (i.e., normlized=True). Recommend to set it 0.01-0.1")
 
         self.negatives_cross_device = negatives_cross_device
         if self.negatives_cross_device:

FlagEmbedding/flag_reranker.py

@@ -406,6 +406,7 @@ def __init__(
         self.model = AutoModelForCausalLM.from_pretrained(model_name_or_path,
                                                           cache_dir=cache_dir,
                                                           trust_remote_code=True,
+                                                          local_files_only=True,
                                                           torch_dtype=torch.bfloat16 if use_bf16 else torch.float32)
 
         self.model_name_or_path = model_name_or_path

FlagEmbedding/llm_reranker/README.md

@@ -314,7 +314,8 @@ torchrun --nproc_per_node {number of gpus} \
 --start_layer 8 \
 --head_multi True \
 --head_type simple \
---lora_extra_parameters linear_head
+--lora_extra_parameters linear_head \
+--finetune_type from_raw_model # should be one of ['from_raw_model', 'from_finetuned_model']
 ```
 
 Our rerankers are initialized from [google/gemma-2b](https://huggingface.co/google/gemma-2b) (for llm-based reranker) and [openbmb/MiniCPM-2B-dpo-bf16](https://huggingface.co/openbmb/MiniCPM-2B-dpo-bf16) (for llm-based layerwise reranker), and we train it on a mixture of multilingual datasets:
@@ -323,6 +324,33 @@ Our rerankers are initialized from [google/gemma-2b](https://huggingface.co/goog
 - [quora train data](https://huggingface.co/datasets/quora)
 - [fever train data](https://fever.ai/dataset/fever.html)
 
+### Merge Model
+
+After finetune, you need to merge the model
+
+**For llm-based reranker**
+
+```python
+from FlagEmbedding.llm_reranker.merge import merge_llm
+merge_llm('google/gemma-2b', 'lora_llm_output_path', 'merged_model_output_paths')
+```
+
+**For llm-based layerwise reranker**
+
+If you finetune the raw model (openbmb/MiniCPM-2B-dpo-bf16)
+
+```shell
+from FlagEmbedding.llm_reranker.merge import merge_layerwise_raw_llm
+merge_layerwise_raw_llm('openbmb/MiniCPM-2B-dpo-bf16', 'lora_llm_output_path', 'merged_model_output_paths')
+```
+
+If you finetune the finetuned model (BAAI/bge-reranker-v2-minicpm-layerwise)
+
+```shell
+from FlagEmbedding.llm_reranker.merge import merge_layerwise_finetuned_llm
+merge_layerwise_finetuned_llm('BAAI/bge-reranker-v2-minicpm-layerwise', 'lora_llm_output_path', 'merged_model_output_paths')
+```
+
 ## Evaluation
 
 - llama-index.
@@ -351,7 +379,6 @@ It rereank the top 100 results from bge-m3.
 ![image-20240317173117639](./evaluation/miracl-bge-m3.png)
 
 
-
 ## Citation
 
 If you find this repository useful, please consider giving a star :star: and citation

FlagEmbedding/llm_reranker/finetune_for_instruction/modeling.py

@@ -85,4 +85,6 @@ def save(self, output_dir: str):
         self.model.save_pretrained(output_dir, state_dict=state_dict)
 
     def save_pretrained(self, **kwargs):
+        self.tokenizer.save_pretrained(**kwargs)
         return self.model.save_pretrained(**kwargs)
+

FlagEmbedding/llm_reranker/finetune_for_layerwise/arguments.py

@@ -86,6 +86,11 @@ class ModelArguments:
         default='simple',
         metadata={"help": "the type of the classifier"}
     )
+    finetune_type: str = field(
+        default='from_raw_model'  # should be one of ['from_raw_model', 'from_finetuned_model']
+        # from_raw_model -- openbmb/MiniCPM-2B-dpo-bf16
+        # from_finetuned_model -- BAAI/bge-reranker-v2-minicpm-layerwise
+    )
 
 
 @dataclass