class TritonMLAImpl(MLACommonImpl[MLACommonMetadata]):
can_return_lse_for_decode: bool = True
supports_dcp: bool = True
def __init__(
self,
num_heads: int,
head_size: int,
scale: float,
num_kv_heads: int,
alibi_slopes: list[float] | None,
sliding_window: int | None,
kv_cache_dtype: str,
logits_soft_cap: float | None,
attn_type: str,
kv_sharing_target_layer_name: str | None,
# MLA Specific Arguments
**mla_args,
) -> None:
super().__init__(
num_heads,
head_size,
scale,
num_kv_heads,
alibi_slopes,
sliding_window,
kv_cache_dtype,
logits_soft_cap,
attn_type,
kv_sharing_target_layer_name,
**mla_args,
)
unsupported_features = [alibi_slopes, logits_soft_cap]
if any(unsupported_features):
raise NotImplementedError(
"TritonMLAImpl does not support one of the following: "
"alibi_slopes, logits_soft_cap"
)
# Each decode row attends to the last `decode_sliding_window` keys.
self.decode_sliding_window = sliding_window
assert sliding_window is None or self.dcp_world_size == 1, (
"Sliding-window Triton MLA does not support decode context parallelism"
)
if attn_type != AttentionType.DECODER:
raise NotImplementedError(
"Encoder self-attention and "
"encoder/decoder cross-attention "
"are not implemented for "
"TritonMLAImpl"
)
if current_platform.is_cuda():
cap = current_platform.get_device_capability()
cap_str = cap.as_version_str() if cap is not None else "unknown"
dev = current_platform.get_device_name()
if self.kv_cache_dtype.startswith("fp8") and not (
current_platform.has_device_capability(89)
):
suggested = (
"float16" if (cap is None or cap.to_int() < 80) else "bfloat16"
)
raise ValueError(
f"FP8 KV cache is not supported by the Triton MLA backend "
f"on {dev} (compute capability {cap_str}); native FP8 "
f"(fp8e4nv) requires SM89+. Re-run with "
f"--kv-cache-dtype {suggested}."
)
if self.kv_cache_dtype == "bfloat16" and not (
current_platform.has_device_capability(80)
):
raise ValueError(
f"bfloat16 KV cache is not supported by the Triton MLA "
f"backend on {dev} (compute capability {cap_str}); "
f"bfloat16 requires SM80+. Re-run with "
f"--kv-cache-dtype float16."
)
# For FP8 KV cache, we dequantize to BF16 on load inside the
# Triton kernel. Tell the common layer not to quantize queries
# to FP8 — we handle FP8 KV cache with BF16 queries (Mode 1).
if is_quantized_kv_cache(self.kv_cache_dtype):
self.supports_quant_query_input = False
self._sm_count = current_platform.num_compute_units()
self._vllm_config = get_current_vllm_config()
def reserve_workspace(self) -> None:
# The builder reserves after memory profiling; reserve its upper bound
# here so the KV cache is sized around it.
vllm_config = self._vllm_config
_reserve_attn_logits_workspace(
vllm_config,
vllm_config.scheduler_config.max_num_seqs
* max_decode_query_len(vllm_config),
self.num_heads * self.dcp_world_size,
self.kv_lora_rank,
self.decode_sliding_window,
)
def forward_mha(self, *args, **kwargs) -> None:
if self.decode_sliding_window is not None:
# Windowed DSpark drafts only run non-causal decode blocks.
raise NotImplementedError("Sliding-window Triton MLA prefill")
return super().forward_mha(*args, **kwargs)
def register_decode_warmup(
self, kv_cache: torch.Tensor, kv_cache_dtype: str
) -> None:
"""Register the split-KV decode kernels for a bound KV cache.
Mirrors forward_mqa's launch: MLAAttention views a uint8 fp8 cache as
fp8, num_kv_splits is any value `_compute_num_kv_splits` can select
up to max_model_len (clamped to the sliding window), and the query
keeps the model dtype.
"""
if is_quantized_kv_cache(kv_cache_dtype) and kv_cache_dtype not in (
"fp8_ds_mla",
"nvfp4_ds_mla",
):
kv_cache = kv_cache.view(current_platform.fp8_dtype())
if envs.VLLM_BATCH_INVARIANT:
num_kv_splits: tuple[int, ...] = (1,)
else:
max_kv_len = self._vllm_config.model_config.max_model_len
if self.decode_sliding_window is not None:
max_kv_len = min(max_kv_len, self.decode_sliding_window)
max_splits = _compute_num_kv_splits(max_kv_len, self._sm_count)
num_kv_splits = tuple(
sorted({1 << i for i in range(max_splits.bit_length())} | {max_splits})
)
num_heads = self.num_heads * self.dcp_world_size
dtype = self._vllm_config.model_config.dtype
sliding_window = self.decode_sliding_window or 0
_decode_grouped_att_m_fwd.register_warmup(
q_dtype=dtype,
kv_dtype=kv_cache.dtype,
num_heads=num_heads,
head_size=kv_cache.shape[-1],
kv_lora_rank=self.kv_lora_rank,
page_size=kv_cache.shape[1],
kv_block_stride=kv_cache.stride(0),
kv_token_stride=kv_cache.stride(1),
num_kv_splits=num_kv_splits,
sliding_window=sliding_window,
)
_decode_softmax_reducev_fwd.register_warmup(
dtype=dtype,
num_heads=num_heads,
kv_lora_rank=self.kv_lora_rank,
num_kv_splits=num_kv_splits,
sliding_window=sliding_window,
)
def forward_mqa(
self,
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: MLACommonMetadata,
layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]:
assert kv_c_and_k_pe_cache.numel() > 0
assert attn_metadata.decode is not None
if type(q) is tuple:
q = torch.cat(q, dim=-1)
assert isinstance(q, torch.Tensor)
B = q.shape[0]
q_num_heads = q.shape[1]
o = torch.zeros(
B, q_num_heads, self.kv_lora_rank, dtype=q.dtype, device=q.device
)
lse = torch.zeros(B, q_num_heads, dtype=q.dtype, device=q.device)
# For batch invariance, use only 1 split to ensure deterministic reduction
if envs.VLLM_BATCH_INVARIANT:
num_kv_splits = 1
else:
max_kv_len = attn_metadata.max_seq_len
if self.decode_sliding_window is not None:
max_kv_len = min(max_kv_len, self.decode_sliding_window)
num_kv_splits = _compute_num_kv_splits(max_kv_len, self._sm_count)
# NOTE: the +1 stores the LogSumExp (LSE) that the stage2 kernel uses to
# merge partial attention outputs across splits. The scratch is served
# from the shared workspace (reserved at max in the metadata builder), so
# there is no per-call allocation on the decode hot path. Fall back to a
# direct allocation when the workspace manager is not initialized (e.g.
# unit tests without a GPUModelRunner).
logits_shape = (B, q_num_heads, num_kv_splits, self.kv_lora_rank + 1)
if is_workspace_manager_initialized():
(attn_logits,) = current_workspace_manager().get_simultaneous(
(logits_shape, torch.float32),
)
else:
attn_logits = torch.empty(
logits_shape, dtype=torch.float32, device=q.device
)
# Add a head dim of 1
kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.unsqueeze(2)
kv_c_cache = kv_c_and_k_pe_cache[..., : self.kv_lora_rank]
PAGE_SIZE = kv_c_and_k_pe_cache.size(1)
block_table = attn_metadata.decode.block_table
seq_lens = attn_metadata.decode.seq_lens
# decode_attention_fwd has no causal flag: it launches one program per
# row of q and reads that row's KV extent from seq_lens, so intra-block
# causality is expressed as per-row extents. Deriving query_len from the
# tensors the kernel indexes keeps the three row counts in step.
num_decodes = seq_lens.shape[0]
query_len, remainder = divmod(B, num_decodes) if num_decodes else (1, 0)
assert remainder == 0, (
f"non-uniform decode block: {B} query rows over {num_decodes} requests"
)
if query_len > 1:
block_table = block_table.repeat_interleave(query_len, dim=0)
if attn_metadata.causal:
# Per-row extents are offsets off the global sequence length;
# under DCP seq_lens is this rank's local slice instead.
assert self.dcp_world_size == 1, (
"causal multi-token decode is not supported with DCP"
)
# Row t attends the prefix plus block tokens 0..t. Clamp holds
# padding rows at extent 0; the kernel skips them either way
# (split_kv_end == split_kv_start).
offsets = torch.arange(
1 - query_len, 1, device=seq_lens.device, dtype=seq_lens.dtype
)
seq_lens = (seq_lens.unsqueeze(1) + offsets).flatten().clamp(min=0)
else:
# Non-causal draft block: every row sees the same prefix.
seq_lens = seq_lens.repeat_interleave(query_len)
# Run MQA — always pass layer scales. When KV cache is
# BF16 the kernel's `if dtype.is_fp8()` check is a no-op.
decode_attention_fwd(
q,
kv_c_and_k_pe_cache,
kv_c_cache,
o,
lse,
block_table,
seq_lens,
attn_logits,
num_kv_splits,
self.scale,
PAGE_SIZE,
k_scale=layer._k_scale,
v_scale=layer._k_scale,
is_mla=True,
sliding_window=self.decode_sliding_window,
)
return o, lse