Merge branch 'master' into master

Author: staoxiao

FlagEmbedding/llm_embedder/src/lm/modeling_lm.py

@@ -38,7 +38,7 @@ def __init__(self, model_name_or_path=None, padding_side="left", dtype="bf16", c
         except ValueError:
             self.model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path, cache_dir=cache_dir, torch_dtype=dtype, trust_remote_code=True, device_map=device_map)
 
-        # if device_map is specified, we don't need to move the model to any specifc gpu
+        # if device_map is specified, we don't need to move the model to any specific gpu
         if device_map is None:
             if accelerator is not None:
                 device = accelerator.device

Long_LLM/activation_beacon/new/README.md

@@ -21,6 +21,9 @@ wget https://huggingface.co/datasets/namespace-Pt/projects/resolve/main/activati
 
 cd /data
 tar -xzvf activation-beacon.tar.gz
+
+# you must download the new longalpaca dataset that was organized into single-turn conversation
+wget https://huggingface.co/datasets/namespace-Pt/projects/resolve/main/longalpaca.json?download=true -O /data/activation-beacon/finetune/longalpaca.new.json
 ```
 
 **IMPORTANT NOTE**
@@ -36,7 +39,7 @@ cd new
 torchrun --nproc_per_node 8 -m main.train \
 --output_dir data/outputs/activation-beacon-llama2-chat-7b \
 --model_name_or_path meta-llama/Llama-2-7b-chat-hf \
---train_data activation-beacon:pretrain/redpajama-sample.json activation-beacon:finetune/longalpaca.json \
+--train_data activation-beacon:pretrain/redpajama-sample.json activation-beacon:finetune/longalpaca.new.json \
 --max_length 8192 \
 --min_length 1200 \
 --max_train_num_per_data 200000 \