Fix double BOS when using chat templates (vllm, transformers, sglang backends)

src/lighteval/models/sglang/sglang_model.py

@@ -83,7 +83,8 @@ class SGLangModelConfig(ModelConfig):
         kv_cache_dtype (str):
             Data type for key-value cache. Defaults to "auto".
         add_special_tokens (bool):
-            Whether to add special tokens during tokenization. Defaults to True.
+            Whether to add special tokens (e.g. BOS) when tokenizing plain-text prompts. Chat-template prompts
+            are tokenized as rendered, since the template writes its own special tokens. Defaults to True.
         pairwise_tokenization (bool):
             Whether to tokenize context and continuation separately for loglikelihood evals. Defaults to False.
         sampling_backend (str | None):
@@ -180,7 +181,8 @@ def cleanup(self):
 
     @property
     def add_special_tokens(self):
-        return self._add_special_tokens
+        # a rendered chat template already holds its special tokens (e.g. BOS)
+        return self._add_special_tokens and not self.use_chat_template
 
     @property
     def max_length(self) -> int:

src/lighteval/models/transformers/transformers_model.py

@@ -94,7 +94,8 @@ class TransformersModelConfig(ModelConfig):
         model_loading_kwargs (dict):
             Additional keyword arguments passed to `from_pretrained`. Defaults to empty dict.
         add_special_tokens (bool):
-            Whether to add special tokens during tokenization. Defaults to True.
+            Whether to add special tokens (e.g. BOS) when tokenizing plain-text prompts. Chat-template prompts
+            are tokenized as rendered, since the template writes its own special tokens. Defaults to True.
         skip_special_tokens (bool):
             Whether the tokenizer should output special tokens back during generation. Needed for reasoning models. Defaults to True
         model_parallel (bool | None):
@@ -317,7 +318,8 @@ def tokenizer(self):
 
     @property
     def add_special_tokens(self):
-        return self._add_special_tokens
+        # a rendered chat template already holds its special tokens (e.g. BOS)
+        return self._add_special_tokens and not self.use_chat_template
 
     @property
     def max_length(self) -> int:

src/lighteval/models/vllm/vllm_model.py

@@ -128,7 +128,8 @@ class VLLMModelConfig(ModelConfig):
         trust_remote_code (bool):
             Whether to trust remote code when loading models. Defaults to False.
         add_special_tokens (bool):
-            Whether to add special tokens during tokenization. Defaults to True.
+            Whether to add special tokens (e.g. BOS) when tokenizing plain-text prompts. Chat-template prompts
+            are tokenized as rendered, since the template writes its own special tokens. Defaults to True.
         multichoice_continuations_start_space (bool):
             Whether to add a space before multiple choice continuations. Defaults to True.
         pairwise_tokenization (bool):
@@ -247,7 +248,8 @@ def cleanup(self):
 
     @property
     def add_special_tokens(self):
-        return self._add_special_tokens
+        # a rendered chat template already holds its special tokens (e.g. BOS)
+        return self._add_special_tokens and not self.use_chat_template
 
     @property
     def max_length(self) -> int:

tests/unit/models/test_base_model.py

@@ -45,7 +45,10 @@ def test_loglikelihood_rolling_matches_manual_scoring():
     """Rolling perplexity on a `choices=None` doc (the_pile/wikitext style) should not
     crash and should match a plain autoregressive logits[:-1] vs ids[1:] scoring."""
     model_config = TransformersModelConfig(
-        model_name="hf-internal-testing/tiny-random-LlamaForCausalLM", model_parallel=False, revision="main"
+        model_name="hf-internal-testing/tiny-random-LlamaForCausalLM",
+        model_parallel=False,
+        revision="main",
+        override_chat_template=False,
     )
     model: TransformersModel = load_model(config=model_config)
 

tests/unit/models/test_transformers_model.py

@@ -426,5 +426,38 @@ def test_transformers_model_use_chat_template_with_different_model_names(
                 self.assertEqual(model.use_chat_template, model._tokenizer.chat_template is not None)
 
 
+class TestTransformersModelPromptTokens(unittest.TestCase):
+    @patch("lighteval.models.transformers.transformers_model.Accelerator")
+    @patch("lighteval.models.transformers.transformers_model.TransformersModel._create_auto_model")
+    @patch("lighteval.models.transformers.transformers_model.TransformersModel._generate")
+    def test_prompt_starts_with_one_bos(self, mock_generate, mock_create_model, mock_accelerator):
+        """The input ids passed to generation start with exactly one BOS: chat templates that write BOS (Llama 3) do
+        not get a second one from the tokenizer, plain-text prompts keep the tokenizer's BOS, and tokenizers that add
+        none (Qwen) are unchanged."""
+        mock_accelerator_instance = Mock()
+        mock_accelerator_instance.device = torch.device("cpu")
+        mock_accelerator_instance.is_main_process = True
+        mock_accelerator_instance.prepare = lambda x: x
+        mock_accelerator.return_value = mock_accelerator_instance
+        mock_generate.side_effect = lambda batch, **kwargs: [ModelResponse(text=[""]) for _ in batch.input_ids]
+        doc = Doc(query="What is 2 + 2?", choices=[""], gold_index=0, generation_size=1, stop_sequences=[])
+        test_cases = [
+            ("unsloth/Llama-3.2-1B-Instruct", {}, [128000, 128006]),  # chat: <|begin_of_text|><|start_header_id|>
+            ("unsloth/Llama-3.2-1B-Instruct", {"override_chat_template": False}, [128000, 3923]),  # plain text
+            ("Qwen/Qwen3-0.6B", {}, [151644, 872]),  # chat, no BOS: <|im_start|>user
+        ]
+
+        for model_name, extra_args, expected_start in test_cases:
+            with self.subTest(model_name=model_name, **extra_args):
+                config = TransformersModelConfig(
+                    model_name=model_name, batch_size=1, model_parallel=True, compile=False, **extra_args
+                )
+                model = TransformersModel(config)
+
+                model._padded_greedy_until([doc])
+                input_ids = mock_generate.call_args.kwargs["batch"].input_ids[0].tolist()
+                self.assertEqual(input_ids[:2], expected_start)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/unit/models/vllm/test_vllm_model.py

@@ -28,6 +28,7 @@
 from transformers import AutoTokenizer
 
 from lighteval.models.vllm.vllm_model import VLLMModel, VLLMModelConfig, build_vllm_token_prompts
+from lighteval.tasks.requests import Doc
 
 
 class TestVLLMPromptConstruction(unittest.TestCase):
@@ -84,3 +85,32 @@ def test_vllm_model_use_chat_template_with_different_model_names(self, mock_crea
 
                 self.assertEqual(model.use_chat_template, expected_result)
                 self.assertEqual(model.use_chat_template, model._tokenizer.chat_template is not None)
+
+
+class TestVLLMModelPromptTokens(unittest.TestCase):
+    @patch("lighteval.models.vllm.vllm_model.VLLMModel._create_auto_model")
+    def test_prompt_starts_with_one_bos(self, mock_create_model):
+        """The token ids sent to vLLM start with exactly one BOS: chat templates that write BOS (Llama 3) do not get
+        a second one from the tokenizer, plain-text prompts keep the tokenizer's BOS, and tokenizers that add none
+        (Qwen) are unchanged."""
+
+        def generate(prompts, sampling_params, use_tqdm):
+            return [
+                Mock(outputs=[Mock(token_ids=[], text="")], prompt_token_ids=p["prompt_token_ids"]) for p in prompts
+            ]
+
+        mock_create_model.return_value = Mock(generate=generate)
+        doc = Doc(query="What is 2 + 2?", choices=[""], gold_index=0, generation_size=1, stop_sequences=[])
+        test_cases = [
+            ("unsloth/Llama-3.2-1B-Instruct", {}, [128000, 128006]),  # chat: <|begin_of_text|><|start_header_id|>
+            ("unsloth/Llama-3.2-1B-Instruct", {"override_chat_template": False}, [128000, 3923]),  # plain text
+            ("Qwen/Qwen3-0.6B", {}, [151644, 872]),  # chat, no BOS: <|im_start|>user
+        ]
+
+        for model_name, extra_args, expected_start in test_cases:
+            with self.subTest(model_name=model_name, **extra_args):
+                config = VLLMModelConfig(model_name=model_name, **extra_args)
+                model = VLLMModel(config)
+
+                (response,) = model._greedy_until([doc])
+                self.assertEqual(response.input_tokens[:2], expected_start)