Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 26 additions & 4 deletions deepspeed/runtime/rollout/hybrid_engine_rollout.py
Original file line number Diff line number Diff line change
Expand Up @@ -1461,8 +1461,8 @@ def _continuous_prefill_cache(self,
logits = prefill_output.logits[-1]
else:
prompt_len = prompt_ids.shape[1]
prefill_cache = self._create_static_cache(static_cache_type, module.config, prompt_ids.shape[0],
prompt_len, device, model_dtype)
prefill_cache = self._create_prefill_cache(static_cache_type, module.config, prompt_ids.shape[0],
prompt_len, device, model_dtype)
prefill_output = self._call_model(
module,
prompt_ids,
Expand Down Expand Up @@ -1580,6 +1580,27 @@ def _call_model(module, input_ids, **kwargs):
kwargs = {name: value for name, value in kwargs.items() if name in parameters}
return module(input_ids, **kwargs)

@classmethod
def _create_prefill_cache(cls, static_cache_type, config, batch_size, max_cache_len, device, dtype):
"""Cache for a single prefill forward whose KV is then copied into the decode cache.

Under autocast, keys and values can come out in different dtypes (rotary embeddings promote
bf16 key projections to fp32), which HF StaticCache rejects. DeepSpeedStaticCache accepts them,
but models every layer as full attention, so sliding-window and linear-attention models keep
HF StaticCache.
"""
text_config = getattr(config, "text_config", config)
layer_types = getattr(text_config, "layer_types", None) or []
full_attention_only = (getattr(text_config, "sliding_window", None) is None
and all(layer_type == "full_attention" for layer_type in layer_types))
if not full_attention_only:
return cls._create_static_cache(static_cache_type, config, batch_size, max_cache_len, device, dtype)

from deepspeed.utils.static_cache import DeepSpeedStaticCache

# Sized lazily from the prefill keys, as AutoTP shards KV heads.
return DeepSpeedStaticCache(config, max_cache_len=max_cache_len)

@staticmethod
def _create_static_cache(static_cache_type, config, batch_size, max_cache_len, device, dtype):
"""Construct StaticCache across Transformers' batch-size API variants."""
Expand Down Expand Up @@ -1809,8 +1830,9 @@ def next_tokens(logits):
prompt_attn)
return select_next_tokens(logits)

# --- Prefill with HF StaticCache (correct attention semantics) ---
prefill_cache = self._create_static_cache(StaticCache, module.config, batch_size, max_len, device, model_dtype)
# --- Prefill with a static cache (correct attention semantics) ---
prefill_cache = self._create_prefill_cache(StaticCache, module.config, batch_size, max_len, device,
model_dtype)
# Hybrid models take a per-type mask dict (GDN must see None; a 2D
# mask corrupts its conv-state init); standard models take a tensor.
is_hybrid = "linear_attention" in (getattr(getattr(module.config, "text_config", module.config), "layer_types",
Expand Down
4 changes: 4 additions & 0 deletions deepspeed/utils/static_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,10 @@ def update(
) -> tuple[torch.Tensor, torch.Tensor]:
if not self.is_initialized:
self.lazy_initialization(key_states, value_states)
# Under autocast, keys and values can arrive in different dtypes: rotary embeddings
# promote bf16 key projections to fp32 while value projections stay bf16.
key_states = key_states.to(self.dtype)
value_states = value_states.to(self.dtype)

kv_length = key_states.shape[-2]

Expand Down
21 changes: 21 additions & 0 deletions tests/unit/runtime/rollout/test_hybrid_engine_rollout.py
Original file line number Diff line number Diff line change
Expand Up @@ -410,6 +410,27 @@ def __init__(self):
assert capture.call_count == 6


@pytest.mark.parametrize("layer_types, sliding_window, expect_deepspeed_cache", [
(None, None, True),
(["full_attention", "full_attention"], None, True),
(None, 4, False),
(["full_attention", "sliding_attention"], 4, False),
(["full_attention", "linear_attention"], None, False),
])
def test_prefill_cache_keeps_hf_static_cache_unless_full_attention_only(layer_types, sliding_window,
expect_deepspeed_cache):
# DeepSpeedStaticCache models every layer as full attention. Prefilling a sliding-window model into it
# would silently attend outside the window, and hybrid models bind their GDN slots to the HF cache.
from transformers import LlamaConfig, StaticCache

config = LlamaConfig(hidden_size=8, num_hidden_layers=2, num_attention_heads=2, num_key_value_heads=1)
config.layer_types = layer_types
config.sliding_window = sliding_window
cache = HybridEngineRollout._create_prefill_cache(StaticCache, config, 1, 4, torch.device("cpu"), torch.float32)

assert isinstance(cache, DeepSpeedStaticCache) == expect_deepspeed_cache


def test_static_cache_constructor_supports_max_batch_keyword():

class MaxBatchStaticCache:
Expand Down
23 changes: 23 additions & 0 deletions tests/unit/utils/test_static_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -243,3 +243,26 @@ def test_static_cache_memory_skips_layers_without_kv():

# Only layer 0 holds KV: 4 rows x 3 heads x 4 positions x (5 key + 7 value dims) x 2 bytes.
assert cache.memory_bytes() == 4 * 3 * 4 * (5 + 7) * 2


@pytest.mark.parametrize("per_row_decode", [False, True])
def test_static_layer_accepts_values_in_a_different_dtype_than_keys(per_row_decode):
# Under autocast, rotary embeddings promote bf16 key projections to fp32 while the value
# projections stay bf16, so one update carries keys and values of different dtypes (#9102).
layer = DeepSpeedStaticLayer(max_cache_len=4)
seq_len = 1 if per_row_decode else 3
keys = torch.randn((2, 1, seq_len, 2), dtype=torch.float32)
values = torch.randn((2, 1, seq_len, 2), dtype=torch.bfloat16)
if per_row_decode:
layer.lazy_initialization(keys, values)
layer.set_write_position(torch.tensor([0, 2]))

cached_keys, cached_values = layer.update(keys, values)

assert cached_values.dtype == torch.float32
if per_row_decode:
written = cached_values[torch.arange(2), :, torch.tensor([0, 2]), :]
assert torch.equal(written, values[:, :, 0, :].float())
else:
assert torch.equal(cached_values[:, :, :seq_len, :], values.float())
assert torch.equal(cached_keys[:, :, :seq_len, :], keys)
33 changes: 33 additions & 0 deletions tests/unit/v1/test_hybrid_engine_rollout_v1.py
Original file line number Diff line number Diff line change
Expand Up @@ -571,6 +571,39 @@ def _random_request(batch_size, prompt_len, device):
return RolloutRequest(prompt_ids.to(device), torch.ones_like(prompt_ids).to(device))


@pytest.mark.parametrize("use_graph_capture, continuous_batch_size", [(True, None), (True, 2), (False, 2)])
@pytest.mark.parametrize("supports_cache_class", [None, True])
def test_rollout_under_autocast_matches_hf_generate(use_graph_capture, continuous_batch_size, supports_cache_class):
# Under autocast an fp32 model emits fp32 keys (rotary embeddings promote them) but bf16 values,
# which the pre-allocated KV caches used to reject with a dtype mismatch (#9102).
# Continuous prefill goes through generate() unless the model sets _supports_cache_class.
accelerator = get_accelerator()
if not accelerator.is_bf16_supported():
pytest.skip("bf16 autocast is not supported")
if use_graph_capture and not accelerator.is_graph_capture_supported():
pytest.skip("graph capture is not supported")

device = torch.device(accelerator.device_name())
model = _tiny_llama(device)
if supports_cache_class is not None:
model._supports_cache_class = supports_cache_class
new_tokens = 8
request = _random_request(batch_size=4, prompt_len=8, device=device)
sampling = SamplingConfig(max_new_tokens=new_tokens, temperature=0, continuous_batch_size=continuous_batch_size)

with torch.autocast(device_type=device.type, dtype=torch.bfloat16):
output = _rollout(model, use_graph_capture).generate(request, sampling)
with torch.no_grad():
expected = model.generate(request.prompt_ids,
attention_mask=request.prompt_attention_mask,
max_new_tokens=new_tokens,
do_sample=False,
eos_token_id=None,
pad_token_id=0)

assert output.input_ids.tolist() == expected.tolist()


@pytest.mark.parametrize("use_graph_capture, continuous_batch_size", [(True, None), (True, 8), (False, 8)])
@pytest.mark.parametrize("temperature, top_k, top_p", [(1.0, 0, 1.0), (0.7, 5, 1.0), (1.0, 0, 0.8)])
def test_decode_sampling_matches_hf_generate(temperature, top_k, top_p, use_graph_capture, continuous_batch_size):
Expand Down
Loading