vllm.model_executor.models.qwen2_5_omni_thinker

Inference-only Qwen2.5-Omni model (thinker part).

Classes:

• Qwen2_5OmniAudioFeatureInputs –

  Dimensions:

• Qwen2_5OmniThinkerForConditionalGeneration –
• Qwen2_5OmniThinkerMultiModalProcessor –
• Qwen2_5OmniThinkerProcessingInfo –

Functions:

• check_interleaved_audio_video –

  Check if video and audio positions are interleaved in any per-video span.

• merge_interleaved_embeddings –

  Merge embeddings for interleaved audio-in-video sequences.

• unpad_and_flat_audio_features –

  Unpad and flatten stacked or ragged per-audio features.

Qwen2_5OmniAudioFeatureInputs

Bases: TensorSchema

Dimensions: - na: Number of audios - nmb: Number of mel bins - msl: Maximum sequence length - tsl: Total sequence length

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

class Qwen2_5OmniAudioFeatureInputs(TensorSchema):
    """Dimensions:
    - na: Number of audios
    - nmb: Number of mel bins
    - msl: Maximum sequence length
    - tsl: Total sequence length
    """

    type: Literal["audio_features"]
    input_features: Annotated[
        torch.Tensor | list[torch.Tensor],
        TensorShape("nmb", "tsl", dynamic_dims={"tsl"}),
    ]

    audio_feature_lengths: Annotated[torch.Tensor, TensorShape("na")]

    feature_attention_mask: Annotated[
        torch.Tensor | list[torch.Tensor],
        TensorShape("na", "msl", dynamic_dims={"msl"}),
    ]

Qwen2_5OmniThinkerForConditionalGeneration

Bases: Module, SupportsMultiModal, SupportsPP, SupportsLoRA, SupportsMRoPE, Qwen2_5OmniConditionalGenerationMixin

Methods:

• get_mm_mapping –

  Get the module prefix in multimodal models

• get_mrope_input_positions –

  Compute M-RoPE input positions using mm_features directly.

• iter_mm_features –

  Iterate over multimodal features sorted by position offset.

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

@MULTIMODAL_REGISTRY.register_processor(
    Qwen2_5OmniThinkerMultiModalProcessor,
    info=Qwen2_5OmniThinkerProcessingInfo,
    dummy_inputs=Qwen2_5OmniThinkerDummyInputsBuilder,
)
class Qwen2_5OmniThinkerForConditionalGeneration(
    nn.Module,
    SupportsMultiModal,
    SupportsPP,
    SupportsLoRA,
    SupportsMRoPE,
    Qwen2_5OmniConditionalGenerationMixin,
):
    supports_mm_device_do_normalize = True

    hf_to_vllm_mapper = WeightsMapper(
        orig_to_new_prefix={
            "thinker.lm_head.": "language_model.lm_head.",
            "thinker.model.": "language_model.model.",
            "thinker.": "",
            "talker.": None,
            "token2wav.": None,
        }
    )
    packed_modules_mapping = {
        "qkv_proj": [
            "q_proj",
            "k_proj",
            "v_proj",
        ],
        "attn.qkv": [
            "attn.q",
            "attn.k",
            "attn.v",
        ],
        "gate_up_proj": [
            "gate_proj",
            "up_proj",
        ],
    }

    @classmethod
    def get_placeholder_str(cls, modality: str, i: int) -> str | None:
        if modality.startswith("image"):
            return "<|vision_start|><|IMAGE|><|vision_end|>"
        if modality.startswith("video"):
            return "<|vision_start|><|VIDEO|><|vision_end|>"
        if modality.startswith("audio"):
            return f"Audio {i}: <|audio_bos|><|AUDIO|><|audio_eos|>"

        raise ValueError("Only image, video or audio modality is supported")

    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
        super().__init__()
        self.vllm_config = vllm_config
        thinker_config: Qwen2_5OmniThinkerConfig = (
            vllm_config.model_config.hf_config.thinker_config
        )
        quant_config = vllm_config.quant_config
        multimodal_config = vllm_config.model_config.multimodal_config
        self.config = thinker_config
        self.multimodal_config = multimodal_config
        self.quant_config = quant_config

        # force "use_flash_attention_2=True" to audio tower to align
        # the results.
        if flash_attn is not None:
            audio_config = thinker_config.audio_config
            audio_config._attn_implementation_autoset = True
            audio_config._attn_implementation = "flash_attention_2"
        else:
            logger.warning(
                "flash_attn is not available, the model may not yield the "
                "exactly same result as the transformers implementation "
                "in the audio tower part."
            )

        with self._mark_tower_model(vllm_config, "audio"):
            self.audio_tower = Qwen2_5OmniAudioEncoder(thinker_config.audio_config)

        with self._mark_tower_model(vllm_config, {"image", "video"}):
            self.visual = Qwen2_5_VisionTransformer(
                vision_config=thinker_config.vision_config,
                norm_eps=getattr(thinker_config.text_config, "rms_norm_eps", 1e-6),
                quant_config=quant_config,
                input_norm=build_mm_input_norm(vllm_config.model_config),
                prefix=maybe_prefix(prefix, "visual"),
            )

        with self._mark_language_model(vllm_config):
            self.language_model = init_vllm_registered_model(
                vllm_config=vllm_config,
                prefix=maybe_prefix(prefix, "language_model"),
                hf_config=thinker_config.text_config,
                architectures=["Qwen2ForCausalLM"],
            )

        self.make_empty_intermediate_tensors = (
            self.language_model.make_empty_intermediate_tensors
        )

    def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
        mm_input_by_modality: dict[
            str,
            Qwen2_5_VLImageInputs
            | Qwen2_5_VLVideoInputs
            | Qwen2_5OmniAudioFeatureInputs
            | None,
        ] = {}

        # Preserve the order of modalities if there are multiple of them
        # from the order of kwargs.
        for input_key in kwargs:
            if (
                input_key in ("pixel_values", "image_embeds")
                and "image" not in mm_input_by_modality
            ):
                mm_input_by_modality["image"] = self._parse_and_validate_image_input(
                    **kwargs
                )
            if (
                input_key in ("pixel_values_videos", "video_embeds")
                and "video" not in mm_input_by_modality
            ):
                mm_input_by_modality["video"] = self._parse_and_validate_video_input(
                    **kwargs
                )
            if (
                input_key in ("input_audio_features")
                and "audio" not in mm_input_by_modality
            ):
                mm_input_by_modality["audio"] = self._parse_and_validate_audio_input(
                    **kwargs
                )
        return mm_input_by_modality

    def _get_audio_for_video_mapping(
        self, mm_features: list[MultiModalFeatureSpec]
    ) -> tuple[dict[int, int], set[int]]:
        """Map video offset -> paired audio_feature_length for use_audio_in_video.

        When use_audio_in_video=True, audio is interleaved within video chunks.
        The pairing is based on feature order in mm_features.

        Returns:
            Tuple of (video_offset -> audio_feature_length mapping,
                      set of paired audio offsets to skip)

        """
        videos_with_audio = [
            f
            for f in mm_features
            if f.modality == "video"
            and f.data is not None
            and f.data.get("use_audio_in_video")
            and _get_feature_tensor(f, "use_audio_in_video").item()
        ]
        audios = [f for f in mm_features if f.modality == "audio"]

        # Pair videos with audio features (assumes matching order)
        mapping: dict[int, int] = {}
        paired_audio_offsets: set[int] = set()
        for i, video_f in enumerate(videos_with_audio):
            if i < len(audios):
                audio_len = int(
                    _get_feature_tensor(audios[i], "audio_feature_lengths").item()
                )
                mapping[video_f.mm_position.offset] = audio_len
                paired_audio_offsets.add(audios[i].mm_position.offset)
        return mapping, paired_audio_offsets

    def _compute_audio_token_count(self, audio_feature_length: int) -> int:
        """Compute audio tokens from feature length."""
        return ((audio_feature_length - 1) // 2 + 1 - 2) // 2 + 1

    def iter_mm_features(
        self, mm_features: list[MultiModalFeatureSpec]
    ) -> Iterator[tuple[int, str, dict[str, Any]]]:
        """Iterate over multimodal features sorted by position offset.

        Yields: (offset, modality, feature_data) where feature_data contains:
        - image: {"grid_t", "grid_h", "grid_w", "t_factor"}
        - video: {"grid_t", "grid_h", "grid_w", "t_factor",
                  "use_audio_in_video", "audio_feature_length"}
        - audio: {"audio_feature_length"}
        """
        thinker_config = self.config
        spatial_merge_size = thinker_config.vision_config.spatial_merge_size
        tokens_per_second = getattr(
            thinker_config.vision_config, "tokens_per_second", 25
        )

        # Sort features by offset first, then pair audio with video
        sorted_features = sorted(mm_features, key=lambda f: f.mm_position.offset)
        audio_for_video, paired_audio_offsets = self._get_audio_for_video_mapping(
            sorted_features
        )

        for mm_feature in sorted_features:
            offset = mm_feature.mm_position.offset
            modality = mm_feature.modality
            assert mm_feature.data is not None

            if modality == "image":
                t, h, w = _get_feature_tensor(mm_feature, "image_grid_thw").tolist()
                yield (
                    offset,
                    "image",
                    {
                        "grid_t": t,
                        "grid_h": h // spatial_merge_size,
                        "grid_w": w // spatial_merge_size,
                        "t_factor": 1.0 * tokens_per_second,
                    },
                )
            elif modality == "video":
                t, h, w = _get_feature_tensor(mm_feature, "video_grid_thw").tolist()
                second_per_grid_ts = 1.0
                if mm_feature.data.get("second_per_grid_ts"):
                    second_per_grid_ts = _get_feature_tensor(
                        mm_feature, "second_per_grid_ts"
                    ).item()
                use_audio_in_video = False
                if mm_feature.data.get("use_audio_in_video"):
                    use_audio_in_video = bool(
                        _get_feature_tensor(mm_feature, "use_audio_in_video").item()
                    )

                yield (
                    offset,
                    "video",
                    {
                        "grid_t": t,
                        "grid_h": h // spatial_merge_size,
                        "grid_w": w // spatial_merge_size,
                        "t_factor": second_per_grid_ts * tokens_per_second,
                        "use_audio_in_video": use_audio_in_video,
                        "audio_feature_length": audio_for_video.get(offset),
                    },
                )
            elif modality == "audio":
                # Skip audio that's paired with video (handled in video case)
                if offset not in paired_audio_offsets:
                    audio_len = _get_feature_tensor(
                        mm_feature, "audio_feature_lengths"
                    ).item()
                    yield offset, "audio", {"audio_feature_length": audio_len}

    def _compute_interleaved_positions(
        self, start_idx: int, data: dict[str, Any]
    ) -> tuple[np.ndarray, int]:
        """Compute positions for interleaved video+audio chunks.

        Returns: (position_ids, total_token_count)
        """
        grid_t = data["grid_t"]
        grid_h = data["grid_h"]
        grid_w = data["grid_w"]
        t_factor = data["t_factor"]
        audio_len = data["audio_feature_length"]

        thinker_config = self.config
        tokens_per_second = getattr(
            thinker_config.vision_config, "tokens_per_second", 25
        )
        seconds_per_chunk = thinker_config.seconds_per_chunk
        t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk)

        # Temporal indices with scaling
        t_index = (np.arange(grid_t) * t_factor).astype(np.int64)

        # Split temporal indices into chunks
        t_index_split_chunk: list[list[int]] = [
            [] for _ in range((int(t_index.max()) // t_ntoken_per_chunk) + 1)
        ]
        for t_val in t_index:
            idx = int(t_val) // t_ntoken_per_chunk
            t_index_split_chunk[idx].append(int(t_val))

        pure_audio_len = self._compute_audio_token_count(audio_len)
        added_audio_len = 0
        pos_ids_list: list[np.ndarray] = []
        audio_start_idx = start_idx

        for t_chunk in t_index_split_chunk:
            if not t_chunk:
                continue

            chunk_t = len(t_chunk)

            # Build vision positions for this chunk
            h_indices = np.tile(
                np.arange(grid_h).reshape(1, -1, 1), (chunk_t, 1, grid_w)
            ).flatten()
            w_indices = np.tile(
                np.arange(grid_w).reshape(1, 1, -1), (chunk_t, grid_h, 1)
            ).flatten()
            t_indices = np.repeat(np.array(t_chunk), grid_h * grid_w)

            vision_pos = np.stack([t_indices, h_indices, w_indices]) + start_idx
            pos_ids_list.append(vision_pos)

            # Audio tokens for this chunk
            audio_chunk_size = min(t_ntoken_per_chunk, pure_audio_len - added_audio_len)
            if audio_chunk_size > 0:
                audio_pos = (
                    np.broadcast_to(np.arange(audio_chunk_size), (3, audio_chunk_size))
                    + audio_start_idx
                )
                pos_ids_list.append(audio_pos)
                audio_start_idx = audio_start_idx + audio_chunk_size
                added_audio_len += audio_chunk_size

        # Handle remaining audio that doesn't fit in chunks
        if added_audio_len < pure_audio_len:
            remaining = pure_audio_len - added_audio_len
            remaining_audio_pos = (
                np.broadcast_to(np.arange(remaining), (3, remaining)) + audio_start_idx
            )
            pos_ids_list.append(remaining_audio_pos)

        # Calculate total token count
        vision_tokens = grid_t * grid_h * grid_w
        total_tokens = vision_tokens + pure_audio_len

        return np.concatenate(pos_ids_list, axis=1), total_tokens

    def get_mrope_input_positions(
        self,
        input_tokens: list[int],
        mm_features: list[MultiModalFeatureSpec],
    ) -> tuple[torch.Tensor, int]:
        """Compute M-RoPE input positions using mm_features directly.

        Example for use_audio_in_video case:
            (V_i are vision position ids, A_i are audio position ids)

            |V_1 ...    V_n|A_1 ...   A_n|V_n+1 ... V_2n|A_n+1 ... A_2n|...
            |vision chunk 1|audio chunk 1|vision chunk 2|audio chunk 2 |...
        """
        llm_pos_ids_list: list[np.ndarray] = []
        st = 0

        for offset, modality, data in self.iter_mm_features(mm_features):
            # Add text segment before this feature
            text_len = offset - st
            st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0
            if text_len > 0:
                llm_pos_ids_list.append(
                    np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
                )
                st_idx += text_len

            if modality == "audio":
                # Standalone audio positions
                audio_tokens = self._compute_audio_token_count(
                    data["audio_feature_length"]
                )
                llm_pos_ids_list.append(
                    np.broadcast_to(np.arange(audio_tokens), (3, audio_tokens)) + st_idx
                )
                st = offset + audio_tokens

            elif modality == "image":
                # Image uses np.indices like Qwen2-VL
                grid_t = data["grid_t"]
                grid_h = data["grid_h"]
                grid_w = data["grid_w"]
                t_factor = data["t_factor"]

                grid_indices = np.indices((grid_t, grid_h, grid_w))
                if t_factor != 1.0:
                    grid_indices[0] = (grid_indices[0] * t_factor).astype(np.int64)
                llm_pos_ids_list.append(grid_indices.reshape(3, -1) + st_idx)
                st = offset + grid_t * grid_h * grid_w

            elif modality == "video":
                grid_t = data["grid_t"]
                grid_h = data["grid_h"]
                grid_w = data["grid_w"]
                t_factor = data["t_factor"]

                if not data["use_audio_in_video"]:
                    # Simple video (same as Qwen2-VL)
                    grid_indices = np.indices((grid_t, grid_h, grid_w))
                    if t_factor != 1.0:
                        grid_indices[0] = (grid_indices[0] * t_factor).astype(np.int64)
                    llm_pos_ids_list.append(grid_indices.reshape(3, -1) + st_idx)
                    st = offset + grid_t * grid_h * grid_w
                else:
                    # Interleaved video+audio
                    pos_ids, token_count = self._compute_interleaved_positions(
                        st_idx, data
                    )
                    llm_pos_ids_list.append(pos_ids)
                    st = offset + token_count

        # Add trailing text
        if st < len(input_tokens):
            st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0
            text_len = len(input_tokens) - st
            llm_pos_ids_list.append(
                np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
            )

        llm_positions = np.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1)
        mrope_position_delta = int(llm_positions.max()) + 1 - len(input_tokens)

        return torch.from_numpy(llm_positions), mrope_position_delta

    def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
        mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
        if not mm_input_by_modality:
            return []

        # The result multimodal_embeddings is tuple of tensors, with each
        # tensor corresponding to a multimodal data item (image or video).
        multimodal_embeddings: tuple[torch.Tensor, ...] = ()

        # NOTE: It is important to iterate over the keys in this dictionary
        # to preserve the order of the modalities.
        for modality in mm_input_by_modality:
            multimodal_input = mm_input_by_modality[modality]
            if modality == "image":
                image_embeddings = self._process_image_input(multimodal_input)
                multimodal_embeddings += tuple(image_embeddings)
            if modality == "video":
                video_embeddings = self._process_video_input(multimodal_input)
                multimodal_embeddings += tuple(video_embeddings)
            if modality == "audio":
                audio_embeddings = self._process_audio_input(multimodal_input)
                multimodal_embeddings += tuple(audio_embeddings)
        return multimodal_embeddings

    def embed_input_ids(
        self,
        input_ids: torch.Tensor,
        multimodal_embeddings: MultiModalEmbeddings | None = None,
        *,
        is_multimodal: torch.Tensor | None = None,
    ) -> torch.Tensor:
        if multimodal_embeddings is None or is_multimodal is None:
            return super().embed_input_ids(input_ids)

        if len(multimodal_embeddings) == 0:
            return super().embed_input_ids(
                input_ids,
                multimodal_embeddings=multimodal_embeddings,
                is_multimodal=is_multimodal,
            )

        # Check for audio-in-video: interleaved video and audio tokens
        # in the multimodal region. Only use the interleaved path when
        # needed; otherwise fall back to the default parent implementation.
        video_token_id = self.config.video_token_index
        audio_token_id = self.config.audio_token_index

        with gpu_sync_allowed():
            input_ids_cpu = input_ids.cpu()
            is_video = is_multimodal & (input_ids_cpu == video_token_id)
            is_audio = is_multimodal & (input_ids_cpu == audio_token_id)

            num_video = is_video.sum().item()
            num_audio = is_audio.sum().item()

        if check_interleaved_audio_video(is_video, is_audio, num_video, num_audio):
            inputs_embeds = self._embed_text_input_ids(
                input_ids,
                self.get_language_model().embed_input_ids,
                is_multimodal=is_multimodal,
            )
            return merge_interleaved_embeddings(
                inputs_embeds,
                multimodal_embeddings,
                is_video,
                is_audio,
                is_multimodal,
            )

        # Default: standard merge (no interleaving), same as parent class
        return super().embed_input_ids(
            input_ids,
            multimodal_embeddings=multimodal_embeddings,
            is_multimodal=is_multimodal,
        )

    def forward(
        self,
        input_ids: torch.Tensor | None,
        positions: torch.Tensor,
        intermediate_tensors: IntermediateTensors | None = None,
        inputs_embeds: torch.Tensor | None = None,
        **kwargs: object,
    ) -> torch.Tensor | IntermediateTensors:
        if intermediate_tensors is not None:
            inputs_embeds = None

        hidden_states = self.language_model.model(
            input_ids, positions, intermediate_tensors, inputs_embeds=inputs_embeds
        )
        return hidden_states

    def compute_logits(
        self,
        hidden_states: torch.Tensor,
    ) -> torch.Tensor | None:
        return self.language_model.compute_logits(hidden_states)

    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
        loader = AutoWeightsLoader(self)
        return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)

    def get_mm_mapping(self) -> MultiModelKeys:
        """Get the module prefix in multimodal models"""
        return MultiModelKeys.from_string_field(
            language_model="language_model",
            connector="merger.",
            tower_model=["visual.", "audio_tower."],
        )

_compute_audio_token_count(audio_feature_length)

Compute audio tokens from feature length.

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

def _compute_audio_token_count(self, audio_feature_length: int) -> int:
    """Compute audio tokens from feature length."""
    return ((audio_feature_length - 1) // 2 + 1 - 2) // 2 + 1

_compute_interleaved_positions(start_idx, data)

Compute positions for interleaved video+audio chunks.

Returns: (position_ids, total_token_count)

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

def _compute_interleaved_positions(
    self, start_idx: int, data: dict[str, Any]
) -> tuple[np.ndarray, int]:
    """Compute positions for interleaved video+audio chunks.

    Returns: (position_ids, total_token_count)
    """
    grid_t = data["grid_t"]
    grid_h = data["grid_h"]
    grid_w = data["grid_w"]
    t_factor = data["t_factor"]
    audio_len = data["audio_feature_length"]

    thinker_config = self.config
    tokens_per_second = getattr(
        thinker_config.vision_config, "tokens_per_second", 25
    )
    seconds_per_chunk = thinker_config.seconds_per_chunk
    t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk)

    # Temporal indices with scaling
    t_index = (np.arange(grid_t) * t_factor).astype(np.int64)

    # Split temporal indices into chunks
    t_index_split_chunk: list[list[int]] = [
        [] for _ in range((int(t_index.max()) // t_ntoken_per_chunk) + 1)
    ]
    for t_val in t_index:
        idx = int(t_val) // t_ntoken_per_chunk
        t_index_split_chunk[idx].append(int(t_val))

    pure_audio_len = self._compute_audio_token_count(audio_len)
    added_audio_len = 0
    pos_ids_list: list[np.ndarray] = []
    audio_start_idx = start_idx

    for t_chunk in t_index_split_chunk:
        if not t_chunk:
            continue

        chunk_t = len(t_chunk)

        # Build vision positions for this chunk
        h_indices = np.tile(
            np.arange(grid_h).reshape(1, -1, 1), (chunk_t, 1, grid_w)
        ).flatten()
        w_indices = np.tile(
            np.arange(grid_w).reshape(1, 1, -1), (chunk_t, grid_h, 1)
        ).flatten()
        t_indices = np.repeat(np.array(t_chunk), grid_h * grid_w)

        vision_pos = np.stack([t_indices, h_indices, w_indices]) + start_idx
        pos_ids_list.append(vision_pos)

        # Audio tokens for this chunk
        audio_chunk_size = min(t_ntoken_per_chunk, pure_audio_len - added_audio_len)
        if audio_chunk_size > 0:
            audio_pos = (
                np.broadcast_to(np.arange(audio_chunk_size), (3, audio_chunk_size))
                + audio_start_idx
            )
            pos_ids_list.append(audio_pos)
            audio_start_idx = audio_start_idx + audio_chunk_size
            added_audio_len += audio_chunk_size

    # Handle remaining audio that doesn't fit in chunks
    if added_audio_len < pure_audio_len:
        remaining = pure_audio_len - added_audio_len
        remaining_audio_pos = (
            np.broadcast_to(np.arange(remaining), (3, remaining)) + audio_start_idx
        )
        pos_ids_list.append(remaining_audio_pos)

    # Calculate total token count
    vision_tokens = grid_t * grid_h * grid_w
    total_tokens = vision_tokens + pure_audio_len

    return np.concatenate(pos_ids_list, axis=1), total_tokens

_get_audio_for_video_mapping(mm_features)

Map video offset -> paired audio_feature_length for use_audio_in_video.

When use_audio_in_video=True, audio is interleaved within video chunks. The pairing is based on feature order in mm_features.

Returns:

• tuple[dict[int, int], set[int]] –

  Tuple of (video_offset -> audio_feature_length mapping, set of paired audio offsets to skip)

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

def _get_audio_for_video_mapping(
    self, mm_features: list[MultiModalFeatureSpec]
) -> tuple[dict[int, int], set[int]]:
    """Map video offset -> paired audio_feature_length for use_audio_in_video.

    When use_audio_in_video=True, audio is interleaved within video chunks.
    The pairing is based on feature order in mm_features.

    Returns:
        Tuple of (video_offset -> audio_feature_length mapping,
                  set of paired audio offsets to skip)

    """
    videos_with_audio = [
        f
        for f in mm_features
        if f.modality == "video"
        and f.data is not None
        and f.data.get("use_audio_in_video")
        and _get_feature_tensor(f, "use_audio_in_video").item()
    ]
    audios = [f for f in mm_features if f.modality == "audio"]

    # Pair videos with audio features (assumes matching order)
    mapping: dict[int, int] = {}
    paired_audio_offsets: set[int] = set()
    for i, video_f in enumerate(videos_with_audio):
        if i < len(audios):
            audio_len = int(
                _get_feature_tensor(audios[i], "audio_feature_lengths").item()
            )
            mapping[video_f.mm_position.offset] = audio_len
            paired_audio_offsets.add(audios[i].mm_position.offset)
    return mapping, paired_audio_offsets

get_mm_mapping()

Get the module prefix in multimodal models

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

def get_mm_mapping(self) -> MultiModelKeys:
    """Get the module prefix in multimodal models"""
    return MultiModelKeys.from_string_field(
        language_model="language_model",
        connector="merger.",
        tower_model=["visual.", "audio_tower."],
    )

get_mrope_input_positions(input_tokens, mm_features)

Compute M-RoPE input positions using mm_features directly.

Example for use_audio_in_video case

(V_i are vision position ids, A_i are audio position ids)

|V_1 ... V_n|A_1 ... A_n|V_n+1 ... V_2n|A_n+1 ... A_2n|... |vision chunk 1|audio chunk 1|vision chunk 2|audio chunk 2 |...

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

def get_mrope_input_positions(
    self,
    input_tokens: list[int],
    mm_features: list[MultiModalFeatureSpec],
) -> tuple[torch.Tensor, int]:
    """Compute M-RoPE input positions using mm_features directly.

    Example for use_audio_in_video case:
        (V_i are vision position ids, A_i are audio position ids)

        |V_1 ...    V_n|A_1 ...   A_n|V_n+1 ... V_2n|A_n+1 ... A_2n|...
        |vision chunk 1|audio chunk 1|vision chunk 2|audio chunk 2 |...
    """
    llm_pos_ids_list: list[np.ndarray] = []
    st = 0

    for offset, modality, data in self.iter_mm_features(mm_features):
        # Add text segment before this feature
        text_len = offset - st
        st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0
        if text_len > 0:
            llm_pos_ids_list.append(
                np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
            )
            st_idx += text_len

        if modality == "audio":
            # Standalone audio positions
            audio_tokens = self._compute_audio_token_count(
                data["audio_feature_length"]
            )
            llm_pos_ids_list.append(
                np.broadcast_to(np.arange(audio_tokens), (3, audio_tokens)) + st_idx
            )
            st = offset + audio_tokens

        elif modality == "image":
            # Image uses np.indices like Qwen2-VL
            grid_t = data["grid_t"]
            grid_h = data["grid_h"]
            grid_w = data["grid_w"]
            t_factor = data["t_factor"]

            grid_indices = np.indices((grid_t, grid_h, grid_w))
            if t_factor != 1.0:
                grid_indices[0] = (grid_indices[0] * t_factor).astype(np.int64)
            llm_pos_ids_list.append(grid_indices.reshape(3, -1) + st_idx)
            st = offset + grid_t * grid_h * grid_w

        elif modality == "video":
            grid_t = data["grid_t"]
            grid_h = data["grid_h"]
            grid_w = data["grid_w"]
            t_factor = data["t_factor"]

            if not data["use_audio_in_video"]:
                # Simple video (same as Qwen2-VL)
                grid_indices = np.indices((grid_t, grid_h, grid_w))
                if t_factor != 1.0:
                    grid_indices[0] = (grid_indices[0] * t_factor).astype(np.int64)
                llm_pos_ids_list.append(grid_indices.reshape(3, -1) + st_idx)
                st = offset + grid_t * grid_h * grid_w
            else:
                # Interleaved video+audio
                pos_ids, token_count = self._compute_interleaved_positions(
                    st_idx, data
                )
                llm_pos_ids_list.append(pos_ids)
                st = offset + token_count

    # Add trailing text
    if st < len(input_tokens):
        st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0
        text_len = len(input_tokens) - st
        llm_pos_ids_list.append(
            np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
        )

    llm_positions = np.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1)
    mrope_position_delta = int(llm_positions.max()) + 1 - len(input_tokens)

    return torch.from_numpy(llm_positions), mrope_position_delta

iter_mm_features(mm_features)

Iterate over multimodal features sorted by position offset.

Yields: (offset, modality, feature_data) where feature_data contains: - image: {"grid_t", "grid_h", "grid_w", "t_factor"} - video: {"grid_t", "grid_h", "grid_w", "t_factor", "use_audio_in_video", "audio_feature_length"} - audio: {"audio_feature_length"}

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

def iter_mm_features(
    self, mm_features: list[MultiModalFeatureSpec]
) -> Iterator[tuple[int, str, dict[str, Any]]]:
    """Iterate over multimodal features sorted by position offset.

    Yields: (offset, modality, feature_data) where feature_data contains:
    - image: {"grid_t", "grid_h", "grid_w", "t_factor"}
    - video: {"grid_t", "grid_h", "grid_w", "t_factor",
              "use_audio_in_video", "audio_feature_length"}
    - audio: {"audio_feature_length"}
    """
    thinker_config = self.config
    spatial_merge_size = thinker_config.vision_config.spatial_merge_size
    tokens_per_second = getattr(
        thinker_config.vision_config, "tokens_per_second", 25
    )

    # Sort features by offset first, then pair audio with video
    sorted_features = sorted(mm_features, key=lambda f: f.mm_position.offset)
    audio_for_video, paired_audio_offsets = self._get_audio_for_video_mapping(
        sorted_features
    )

    for mm_feature in sorted_features:
        offset = mm_feature.mm_position.offset
        modality = mm_feature.modality
        assert mm_feature.data is not None

        if modality == "image":
            t, h, w = _get_feature_tensor(mm_feature, "image_grid_thw").tolist()
            yield (
                offset,
                "image",
                {
                    "grid_t": t,
                    "grid_h": h // spatial_merge_size,
                    "grid_w": w // spatial_merge_size,
                    "t_factor": 1.0 * tokens_per_second,
                },
            )
        elif modality == "video":
            t, h, w = _get_feature_tensor(mm_feature, "video_grid_thw").tolist()
            second_per_grid_ts = 1.0
            if mm_feature.data.get("second_per_grid_ts"):
                second_per_grid_ts = _get_feature_tensor(
                    mm_feature, "second_per_grid_ts"
                ).item()
            use_audio_in_video = False
            if mm_feature.data.get("use_audio_in_video"):
                use_audio_in_video = bool(
                    _get_feature_tensor(mm_feature, "use_audio_in_video").item()
                )

            yield (
                offset,
                "video",
                {
                    "grid_t": t,
                    "grid_h": h // spatial_merge_size,
                    "grid_w": w // spatial_merge_size,
                    "t_factor": second_per_grid_ts * tokens_per_second,
                    "use_audio_in_video": use_audio_in_video,
                    "audio_feature_length": audio_for_video.get(offset),
                },
            )
        elif modality == "audio":
            # Skip audio that's paired with video (handled in video case)
            if offset not in paired_audio_offsets:
                audio_len = _get_feature_tensor(
                    mm_feature, "audio_feature_lengths"
                ).item()
                yield offset, "audio", {"audio_feature_length": audio_len}

Qwen2_5OmniThinkerMultiModalProcessor

Bases: BaseMultiModalProcessor[_I]

Methods:

• omni_get_updates_use_audio_in_video –

  Get video prompt updates when use_audio_in_video is True.

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

class Qwen2_5OmniThinkerMultiModalProcessor(BaseMultiModalProcessor[_I]):
    def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
        return self.dummy_inputs.get_dummy_text(mm_counts)

    def _get_hf_mm_inputs(
        self,
        mm_items: MultiModalDataItems,
        hf_kwargs: Mapping[str, object],
    ) -> HFMultiModalInputs:
        hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

        mm_counts = mm_items.get_all_counts()
        if hf_kwargs.get("use_audio_in_video"):
            num_videos = mm_counts.get("video", 0)
            if mm_counts.get("audio", 0) < num_videos:
                raise ValueError(
                    "Video doesn't have audio track with `audio_in_video=True`"
                )

            if num_videos and "text" in hf_inputs.hf_data:
                standalone_mm_counts = dict(mm_counts)
                standalone_mm_counts["audio"] -= num_videos
                hf_inputs.hf_data["text"] = self._get_hf_mm_text(standalone_mm_counts)

        hf_data = hf_inputs.hf_data
        normalized_kwargs = hf_inputs.hf_kwargs
        audios = hf_data.pop("audio", [])

        # NOTE: WhisperFeatureExtractor cannot handle empty list of audios
        if audios:
            hf_data["audio"] = audios
        elif (
            normalized_kwargs.get("use_audio_in_video")
            and "audio" not in hf_data
            and hf_data.get("videos")
        ):
            raise ValueError(
                "Video doesn't have audio track with `audio_in_video=True`"
            )

        merged = self.info.ctx.get_merged_mm_kwargs(normalized_kwargs)
        if hf_data.get("videos") and (
            merged.keys() & {"size", "min_pixels", "max_pixels"}
        ):
            normalized_kwargs = dict(normalized_kwargs)
            video_size = dict(self.info.get_hf_processor().video_processor.size)
            size_override = merged.get("size")
            if size_override is not None:
                video_size = video_size | size_override
            min_pixels = merged.get("min_pixels")
            if min_pixels is not None:
                video_size["shortest_edge"] = min_pixels
            max_pixels = merged.get("max_pixels")
            if max_pixels is not None:
                video_size["longest_edge"] = max_pixels
            normalized_kwargs["size"] = video_size

        return hf_inputs._replace(hf_kwargs=normalized_kwargs)

    def _get_mm_fields_config(
        self,
        hf_inputs: BatchFeature,
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        return create_qwen2_5_omni_thinker_field_factory(
            self.info.get_hf_config().vision_config.spatial_merge_size
        )(hf_inputs)

    def _derive_audio_from_video_placeholders(
        self,
        placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
        mm_prompt_updates: MultiModalPromptUpdates,
    ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
        """Helper to derive audio placeholders from video placeholders when
        use_audio_in_video=True.
        """
        if "video" not in placeholders:
            return placeholders

        # Validate audio and video counts match
        num_videos = len(placeholders["video"])
        num_audios = len(mm_prompt_updates.get("audio", []))
        if num_audios != num_videos:
            raise ValueError(
                f"use_audio_in_video requires equal number of audio and video "
                f"items, got {num_audios=}, {num_videos=}"
            )

        tokenizer = self.info.get_tokenizer()
        processor = self.info.get_hf_processor()
        audio_token_id = tokenizer.get_vocab()[processor.audio_token]
        video_token_id = tokenizer.get_vocab()[processor.video_token]

        result_placeholders = dict(placeholders)
        audio_placeholders = []
        video_placeholders = []

        # Each video is paired with one audio
        for video_idx, video_placeholder in enumerate(placeholders["video"]):
            # Create is_embed mask selecting only audio tokens
            audio_is_embed = torch.tensor(video_placeholder.tokens) == audio_token_id

            # Create is_embed mask selecting only video tokens
            video_is_embed = torch.tensor(video_placeholder.tokens) == video_token_id

            audio_placeholder = PlaceholderFeaturesInfo(
                modality="audio",
                item_idx=video_idx,
                start_idx=video_placeholder.start_idx,
                tokens=video_placeholder.tokens,
                is_embed=audio_is_embed,
            )
            audio_placeholders.append(audio_placeholder)

            # Update video placeholder with is_embed mask
            video_placeholder_with_mask = PlaceholderFeaturesInfo(
                modality="video",
                item_idx=video_idx,
                start_idx=video_placeholder.start_idx,
                tokens=video_placeholder.tokens,
                is_embed=video_is_embed,
            )
            video_placeholders.append(video_placeholder_with_mask)

        result_placeholders["audio"] = audio_placeholders
        result_placeholders["video"] = video_placeholders
        return result_placeholders

    def _maybe_apply_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        mm_res: MultiModalProcessingResult,
    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
        """Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`."""
        mm_item_counts = mm_items.get_all_counts()
        prompt_ids = mm_res.prompt_ids
        mm_kwargs = mm_res.kwargs
        mm_prompt_updates = mm_res.prompt_updates

        self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
        self._validate_mm_updates(mm_prompt_updates, mm_item_counts)

        # Detect use_audio_in_video from mm_kwargs
        use_audio_in_video = False
        if "video" in mm_kwargs:
            for item in mm_kwargs["video"]:
                if item and item.get("use_audio_in_video"):
                    use_audio_in_video_tensor = item["use_audio_in_video"].data
                    assert isinstance(use_audio_in_video_tensor, torch.Tensor)
                    if use_audio_in_video_tensor.numel() > 0:
                        use_audio_in_video = bool(use_audio_in_video_tensor.item())
                        break
            # for mutilmodality cache
            if any(item is None for item in mm_kwargs["video"]):
                video_token_id = self.info.get_hf_config().video_token_id
                audio_token_id = self.info.get_hf_config().audio_token_id
                video_audio_item_num = sum(
                    id in (video_token_id, audio_token_id) for id in prompt_ids
                )
                audio_updates_num = len(mm_prompt_updates.get("audio", []))
                video_updates_num = len(mm_prompt_updates.get("video", []))
                if video_audio_item_num != video_updates_num + audio_updates_num:
                    use_audio_in_video = True

        if use_audio_in_video and "audio" in mm_prompt_updates:
            # Filter out audio updates - they are embedded in video
            filtered_updates = {
                k: v for k, v in mm_prompt_updates.items() if k != "audio"
            }
            prompt_ids, mm_placeholders = self._apply_prompt_updates(
                prompt_ids,
                filtered_updates,
            )
            # Derive audio placeholders from video placeholders
            mm_placeholders = self._derive_audio_from_video_placeholders(
                mm_placeholders, mm_prompt_updates
            )
        else:
            prompt_ids, mm_placeholders = self._apply_prompt_updates(
                prompt_ids,
                mm_prompt_updates,
            )

        self._validate_mm_placeholders(mm_placeholders, mm_item_counts)

        return prompt_ids, mm_placeholders

    @classmethod
    def omni_get_updates_use_audio_in_video(
        cls,
        thinker_config: PreTrainedConfig,
        audio_len: int,
        video_grid_thw: list[int] | torch.Tensor,
        video_second_per_grid_t: float,
    ) -> list[int]:
        """Get video prompt updates when `use_audio_in_video` is True.

        In this case, audio and vision update ids will be split into
        chunks and interleaved (details in `_omni_get_input_positions_tensor`).

        <|video_bos|><|VIDEO|><|video_eos|> =>
        <|video_bos|><|audio_bos|>(... chunks ...)<|audio_eos|><|video_eos|>
        """
        audio_token_id = thinker_config.audio_token_index
        video_token_id = thinker_config.video_token_index
        audio_start_token_id = thinker_config.audio_start_token_id
        audio_end_token_id = thinker_config.audio_end_token_id
        seconds_per_chunk = thinker_config.seconds_per_chunk
        spatial_merge_size = thinker_config.vision_config.spatial_merge_size
        tokens_per_second = getattr(
            thinker_config.vision_config, "tokens_per_second", 25
        )

        grid_t = video_grid_thw[0]
        grid_h = video_grid_thw[1]
        grid_w = video_grid_thw[2]
        t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk)
        t_index = (
            torch.arange(grid_t) * video_second_per_grid_t * tokens_per_second
        ).long()
        t_index_split_chunk = split_list_into_ranges(t_index, t_ntoken_per_chunk)

        updates = [audio_start_token_id]
        added_audio_len = 0
        for t_chunk in t_index_split_chunk:
            vision_ntoken_per_chunk = (
                len(t_chunk) * grid_h * grid_w // (spatial_merge_size**2)
            )
            updates.extend([video_token_id] * vision_ntoken_per_chunk)

            audio_chunk_size = min(t_ntoken_per_chunk, audio_len - added_audio_len)
            updates.extend(audio_chunk_size * [audio_token_id])
            added_audio_len += audio_chunk_size
        if added_audio_len < audio_len:
            updates.extend((audio_len - added_audio_len) * [audio_token_id])
        updates.extend([audio_end_token_id])

        return updates

    def _get_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, Any],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> Sequence[PromptUpdate]:
        processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
        tokenizer = self.info.get_tokenizer()
        assert isinstance(self.info, Qwen2VLProcessingInfo)
        image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
        vocab = tokenizer.get_vocab()

        audio_token = processor.audio_token
        image_token = processor.image_token
        video_token = processor.video_token
        audio_token_id = vocab[audio_token]
        image_token_id = vocab[image_token]
        video_token_id = vocab[video_token]

        out_mm_data = out_mm_kwargs.get_data()
        audio_feature_lengths = out_mm_data.get("audio_feature_lengths")
        feature_attention_mask = out_mm_data.get("feature_attention_mask")
        if audio_feature_lengths is None and feature_attention_mask is None:
            audio_output_lengths = []
        elif audio_feature_lengths is not None:
            _, audio_output_lens = _get_feat_extract_output_lengths(
                audio_feature_lengths
            )
            audio_output_lengths = audio_output_lens.tolist()
        elif feature_attention_mask is not None:
            assert isinstance(feature_attention_mask, torch.Tensor)
            _, audio_output_lens = _get_feat_extract_output_lengths(
                feature_attention_mask.sum(-1)
            )
            audio_output_lengths = audio_output_lens.tolist()

        # number of audios read from video.
        audio_in_video_item_idx = 0

        def get_replacement_qwen2_audio(item_idx: int):
            item_idx += audio_in_video_item_idx

            num_features = audio_output_lengths[item_idx]
            if num_features == 0:
                audios = mm_items.get_items("audio", AudioProcessorItems)
                audio = audios.get(item_idx)
                assert audio is not None
                raise ValueError(
                    f"The audio {audio} (len={len(audio)}) is too short "
                    "to be represented inside the model"
                )

            return [audio_token_id] * num_features

        def get_replacement_qwen2_vision(item_idx: int, modality: str):
            grid_thw = out_mm_data[f"{modality}_grid_thw"][item_idx]
            assert isinstance(grid_thw, torch.Tensor)
            merge_length = image_processor.merge_size**2

            token_id = image_token_id if modality == "image" else video_token_id
            return [token_id] * (int(grid_thw.prod()) // merge_length)

        use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)
        thinker_config = self.info.get_hf_config()

        def get_replacement_qwen2_use_audio_in_video(item_idx: int):
            nonlocal audio_in_video_item_idx

            audio_num_features = audio_output_lengths[audio_in_video_item_idx]
            video_grid_thw = out_mm_data["video_grid_thw"][item_idx]

            audio_in_video_item_idx += 1

            second_per_grid_ts = hf_processor_mm_kwargs.get("second_per_grid_ts", None)
            processed_second_per_grid_ts = out_mm_data.get("second_per_grid_ts")
            if isinstance(processed_second_per_grid_ts, torch.Tensor):
                video_second_per_grid_t = processed_second_per_grid_ts[item_idx].item()
            elif second_per_grid_ts:
                video_second_per_grid_t = second_per_grid_ts[item_idx]
            else:
                video_second_per_grid_t = 1.0

            updates = self.omni_get_updates_use_audio_in_video(
                thinker_config=thinker_config,
                audio_len=audio_num_features,
                video_grid_thw=video_grid_thw,
                video_second_per_grid_t=video_second_per_grid_t,
            )

            # Only video tokens should receive video embeddings
            return PromptUpdateDetails.select_token_id(
                seq=updates,
                embed_token_id=video_token_id,
            )

        video_replacement_fn = (
            get_replacement_qwen2_use_audio_in_video
            if use_audio_in_video
            else partial(get_replacement_qwen2_vision, modality="video")
        )

        return [
            PromptReplacement(
                modality="audio",
                target=[audio_token_id],
                replacement=get_replacement_qwen2_audio,
            ),
            PromptReplacement(
                modality="image",
                target=[image_token_id],
                replacement=partial(get_replacement_qwen2_vision, modality="image"),
            ),
            PromptReplacement(
                modality="video",
                target=[video_token_id],
                replacement=video_replacement_fn,
            ),
        ]

    def _postprocess_hf_mm_data(
        self,
        hf_data: Mapping[str, object],
        hf_kwargs: Mapping[str, object],
        processed_data: BatchFeature,
    ) -> BatchFeature:
        input_features = processed_data.pop("input_features", None)
        feature_attention_mask = processed_data.get("feature_attention_mask", None)
        if "input_audio_features" not in processed_data and input_features is not None:
            if feature_attention_mask is not None:
                input_features = input_features.permute(0, 2, 1)[
                    feature_attention_mask.bool()
                ].permute(1, 0)
            processed_data["input_audio_features"] = input_features
        if (
            "audio_feature_lengths" not in processed_data
            and feature_attention_mask is not None
        ):
            processed_data["audio_feature_lengths"] = feature_attention_mask.sum(-1)

        video_second_per_grid = processed_data.get("video_second_per_grid", None)
        if video_second_per_grid is not None:
            processed_data["second_per_grid_ts"] = video_second_per_grid

        use_audio_in_video = hf_kwargs.get("use_audio_in_video", False)
        processed_data["use_audio_in_video"] = torch.tensor(use_audio_in_video)
        processed_data.pop("input_ids", None)

        return processed_data

_derive_audio_from_video_placeholders(placeholders, mm_prompt_updates)

Helper to derive audio placeholders from video placeholders when use_audio_in_video=True.

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

def _derive_audio_from_video_placeholders(
    self,
    placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
    mm_prompt_updates: MultiModalPromptUpdates,
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
    """Helper to derive audio placeholders from video placeholders when
    use_audio_in_video=True.
    """
    if "video" not in placeholders:
        return placeholders

    # Validate audio and video counts match
    num_videos = len(placeholders["video"])
    num_audios = len(mm_prompt_updates.get("audio", []))
    if num_audios != num_videos:
        raise ValueError(
            f"use_audio_in_video requires equal number of audio and video "
            f"items, got {num_audios=}, {num_videos=}"
        )

    tokenizer = self.info.get_tokenizer()
    processor = self.info.get_hf_processor()
    audio_token_id = tokenizer.get_vocab()[processor.audio_token]
    video_token_id = tokenizer.get_vocab()[processor.video_token]

    result_placeholders = dict(placeholders)
    audio_placeholders = []
    video_placeholders = []

    # Each video is paired with one audio
    for video_idx, video_placeholder in enumerate(placeholders["video"]):
        # Create is_embed mask selecting only audio tokens
        audio_is_embed = torch.tensor(video_placeholder.tokens) == audio_token_id

        # Create is_embed mask selecting only video tokens
        video_is_embed = torch.tensor(video_placeholder.tokens) == video_token_id

        audio_placeholder = PlaceholderFeaturesInfo(
            modality="audio",
            item_idx=video_idx,
            start_idx=video_placeholder.start_idx,
            tokens=video_placeholder.tokens,
            is_embed=audio_is_embed,
        )
        audio_placeholders.append(audio_placeholder)

        # Update video placeholder with is_embed mask
        video_placeholder_with_mask = PlaceholderFeaturesInfo(
            modality="video",
            item_idx=video_idx,
            start_idx=video_placeholder.start_idx,
            tokens=video_placeholder.tokens,
            is_embed=video_is_embed,
        )
        video_placeholders.append(video_placeholder_with_mask)

    result_placeholders["audio"] = audio_placeholders
    result_placeholders["video"] = video_placeholders
    return result_placeholders

_maybe_apply_prompt_updates(mm_items, mm_res)

Qwen2.5-Omni reimplements this function to handle use_audio_in_video.

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

def _maybe_apply_prompt_updates(
    self,
    mm_items: MultiModalDataItems,
    mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
    """Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`."""
    mm_item_counts = mm_items.get_all_counts()
    prompt_ids = mm_res.prompt_ids
    mm_kwargs = mm_res.kwargs
    mm_prompt_updates = mm_res.prompt_updates

    self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
    self._validate_mm_updates(mm_prompt_updates, mm_item_counts)

    # Detect use_audio_in_video from mm_kwargs
    use_audio_in_video = False
    if "video" in mm_kwargs:
        for item in mm_kwargs["video"]:
            if item and item.get("use_audio_in_video"):
                use_audio_in_video_tensor = item["use_audio_in_video"].data
                assert isinstance(use_audio_in_video_tensor, torch.Tensor)
                if use_audio_in_video_tensor.numel() > 0:
                    use_audio_in_video = bool(use_audio_in_video_tensor.item())
                    break
        # for mutilmodality cache
        if any(item is None for item in mm_kwargs["video"]):
            video_token_id = self.info.get_hf_config().video_token_id
            audio_token_id = self.info.get_hf_config().audio_token_id
            video_audio_item_num = sum(
                id in (video_token_id, audio_token_id) for id in prompt_ids
            )
            audio_updates_num = len(mm_prompt_updates.get("audio", []))
            video_updates_num = len(mm_prompt_updates.get("video", []))
            if video_audio_item_num != video_updates_num + audio_updates_num:
                use_audio_in_video = True

    if use_audio_in_video and "audio" in mm_prompt_updates:
        # Filter out audio updates - they are embedded in video
        filtered_updates = {
            k: v for k, v in mm_prompt_updates.items() if k != "audio"
        }
        prompt_ids, mm_placeholders = self._apply_prompt_updates(
            prompt_ids,
            filtered_updates,
        )
        # Derive audio placeholders from video placeholders
        mm_placeholders = self._derive_audio_from_video_placeholders(
            mm_placeholders, mm_prompt_updates
        )
    else:
        prompt_ids, mm_placeholders = self._apply_prompt_updates(
            prompt_ids,
            mm_prompt_updates,
        )

    self._validate_mm_placeholders(mm_placeholders, mm_item_counts)

    return prompt_ids, mm_placeholders

omni_get_updates_use_audio_in_video(thinker_config, audio_len, video_grid_thw, video_second_per_grid_t) classmethod

Get video prompt updates when use_audio_in_video is True.

In this case, audio and vision update ids will be split into chunks and interleaved (details in _omni_get_input_positions_tensor).

<|video_bos|><|VIDEO|><|video_eos|> => <|video_bos|><|audio_bos|>(... chunks ...)<|audio_eos|><|video_eos|>

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

@classmethod
def omni_get_updates_use_audio_in_video(
    cls,
    thinker_config: PreTrainedConfig,
    audio_len: int,
    video_grid_thw: list[int] | torch.Tensor,
    video_second_per_grid_t: float,
) -> list[int]:
    """Get video prompt updates when `use_audio_in_video` is True.

    In this case, audio and vision update ids will be split into
    chunks and interleaved (details in `_omni_get_input_positions_tensor`).

    <|video_bos|><|VIDEO|><|video_eos|> =>
    <|video_bos|><|audio_bos|>(... chunks ...)<|audio_eos|><|video_eos|>
    """
    audio_token_id = thinker_config.audio_token_index
    video_token_id = thinker_config.video_token_index
    audio_start_token_id = thinker_config.audio_start_token_id
    audio_end_token_id = thinker_config.audio_end_token_id
    seconds_per_chunk = thinker_config.seconds_per_chunk
    spatial_merge_size = thinker_config.vision_config.spatial_merge_size
    tokens_per_second = getattr(
        thinker_config.vision_config, "tokens_per_second", 25
    )

    grid_t = video_grid_thw[0]
    grid_h = video_grid_thw[1]
    grid_w = video_grid_thw[2]
    t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk)
    t_index = (
        torch.arange(grid_t) * video_second_per_grid_t * tokens_per_second
    ).long()
    t_index_split_chunk = split_list_into_ranges(t_index, t_ntoken_per_chunk)

    updates = [audio_start_token_id]
    added_audio_len = 0
    for t_chunk in t_index_split_chunk:
        vision_ntoken_per_chunk = (
            len(t_chunk) * grid_h * grid_w // (spatial_merge_size**2)
        )
        updates.extend([video_token_id] * vision_ntoken_per_chunk)

        audio_chunk_size = min(t_ntoken_per_chunk, audio_len - added_audio_len)
        updates.extend(audio_chunk_size * [audio_token_id])
        added_audio_len += audio_chunk_size
    if added_audio_len < audio_len:
        updates.extend((audio_len - added_audio_len) * [audio_token_id])
    updates.extend([audio_end_token_id])

    return updates

Qwen2_5OmniThinkerProcessingInfo

Bases: Qwen2AudioProcessingInfo, Qwen2_5_VLProcessingInfo

Methods:

• get_target_channels –

  Return target audio channels for Qwen2.5 Omni models (mono).

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

class Qwen2_5OmniThinkerProcessingInfo(
    Qwen2AudioProcessingInfo, Qwen2_5_VLProcessingInfo
):
    processor_kwargs_cls: type = Qwen2_5OmniProcessorKwargs

    def get_hf_config(self):
        return self.ctx.get_hf_config(Qwen2_5OmniConfig).thinker_config

    def get_hf_processor(self, **kwargs: object) -> Qwen2_5OmniProcessor:
        return self.ctx.get_hf_processor(
            Qwen2_5OmniProcessor,
            **kwargs,
        )

    def get_feature_extractor(self, **kwargs: object):
        hf_processor = self.get_hf_processor(**kwargs)
        feature_extractor = hf_processor.feature_extractor  # type: ignore
        assert isinstance(feature_extractor, WhisperFeatureExtractor)
        return feature_extractor

    def get_data_parser(self):
        feature_extractor = self.get_feature_extractor()

        return Qwen2_5OmniThinkerMultiModalDataParser(
            spatial_merge_size=self.get_hf_config().vision_config.spatial_merge_size,
            target_sr=feature_extractor.sampling_rate,
            target_channels=self.get_target_channels(),
            expected_hidden_size=self._get_expected_hidden_size(),
            allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
        )

    def get_target_channels(self) -> int:
        """Return target audio channels for Qwen2.5 Omni models (mono)."""
        return 1

    def get_supported_mm_limits(self) -> Mapping[str, int | None]:
        return {"audio": None, "image": None, "video": None}

    def _get_videos_kwargs(self) -> dict[str, Any]:
        processor = self.get_hf_processor()
        return processor._merge_kwargs(
            self.processor_kwargs_cls,
            tokenizer_init_kwargs=processor.tokenizer.init_kwargs,
            **self.ctx.get_merged_mm_kwargs({}),
        )["videos_kwargs"]

    def _get_video_size(self) -> dict[str, int]:
        merged_kwargs = self.ctx.get_merged_mm_kwargs({})
        if merged_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
            return self._get_vision_size(
                merged_kwargs,
                default_size=self.get_hf_processor().video_processor.size,
            )
        return self._get_videos_kwargs()["size"]

    def get_video_size_with_most_features(self) -> ImageSize:
        return self.get_image_size_with_most_features(
            max_pixels=self._get_video_size()["longest_edge"]
        )

    def get_video_frame_size(self, width: int, height: int) -> ImageSize:
        video_processor = self.get_hf_processor().video_processor
        if not self._get_videos_kwargs().get("do_resize", video_processor.do_resize):
            return ImageSize(width=width, height=height)
        vision_config = self.get_hf_config().vision_config
        video_size = self._get_video_size()
        resized_height, resized_width = smart_resize(
            height=height,
            width=width,
            factor=vision_config.patch_size * vision_config.spatial_merge_size,
            min_pixels=video_size["shortest_edge"],
            max_pixels=video_size["longest_edge"],
        )
        return ImageSize(width=resized_width, height=resized_height)

    def get_num_frames_with_most_features(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
        max_frames_per_video: int | None = None,
    ) -> int:
        video_processor = self.get_hf_processor().video_processor
        temporal_patch_size = video_processor.temporal_patch_size
        target_size = self.get_video_size_with_most_features()
        tokens_per_step = self.get_num_video_tokens(
            image_width=target_size.width,
            image_height=target_size.height,
            num_frames=temporal_patch_size,
            image_processor=video_processor,
            mm_kwargs={},
        )
        num_videos = max(mm_counts.get("video", 0), 1)
        num_steps = max(seq_len // num_videos // tokens_per_step, 1)
        num_frames = num_steps * temporal_patch_size
        if max_frames_per_video is not None:
            num_frames = min(num_frames, max_frames_per_video)
        return num_frames

    def get_max_video_tokens(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> int:
        return seq_len

    def get_mm_max_tokens_per_item(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int] | None = None,
    ) -> Mapping[str, int]:
        mm_counts = mm_counts or {}
        requested_modalities = {m for m, c in mm_counts.items() if c > 0}
        mm_max_tokens: dict[str, int] = {}

        if requested_modalities & {"image", "video"}:
            vl_tokens = Qwen2_5_VLProcessingInfo.get_mm_max_tokens_per_item(
                self,
                seq_len=seq_len,
                mm_counts=mm_counts,
            )
            mm_max_tokens.update(
                {
                    m: vl_tokens[m]
                    for m in ["image", "video"]
                    if m in requested_modalities
                }
            )

        if "audio" in requested_modalities:
            audio_tokens = Qwen2AudioProcessingInfo.get_mm_max_tokens_per_item(
                self,
                seq_len=seq_len,
                mm_counts=mm_counts,
            )
            mm_max_tokens["audio"] = audio_tokens["audio"]

        return mm_max_tokens

get_target_channels()

Return target audio channels for Qwen2.5 Omni models (mono).

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

def get_target_channels(self) -> int:
    """Return target audio channels for Qwen2.5 Omni models (mono)."""
    return 1

check_interleaved_audio_video(is_video, is_audio, num_video, num_audio)

Check if video and audio positions are interleaved in any per-video span.

For use_audio_in_video=True, each video placeholder is expanded into one local span containing only video/audio pad tokens, bounded by non-pad tokens such as audio_start/audio_end. Check each contiguous V/A span independently instead of requiring all V/A tokens in the whole sequence to form one global dense range; otherwise multi-video requests can be misclassified as non-interleaved because of the boundary tokens between videos.

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

def check_interleaved_audio_video(
    is_video: torch.Tensor,
    is_audio: torch.Tensor,
    num_video: int,
    num_audio: int,
) -> bool:
    """Check if video and audio positions are interleaved in any per-video span.

    For use_audio_in_video=True, each video placeholder is expanded into one
    local span containing only video/audio pad tokens, bounded by non-pad
    tokens such as audio_start/audio_end. Check each contiguous V/A span
    independently instead of requiring all V/A tokens in the whole sequence to
    form one global dense range; otherwise multi-video requests can be
    misclassified as non-interleaved because of the boundary tokens between
    videos.
    """
    if num_video == 0 or num_audio == 0:
        return False

    va_pos = (is_video | is_audio).nonzero(as_tuple=True)[0]
    if len(va_pos) == 0:
        return False

    span_start = 0
    for span_end in range(1, len(va_pos) + 1):
        is_last = span_end == len(va_pos)
        if not is_last and va_pos[span_end].item() == va_pos[span_end - 1].item() + 1:
            continue

        span = va_pos[span_start:span_end]
        span_is_video = is_video[span]
        span_is_audio = is_audio[span]
        if span_is_video.any() and span_is_audio.any():
            video_offsets = span_is_video.nonzero(as_tuple=True)[0]
            audio_offsets = span_is_audio.nonzero(as_tuple=True)[0]
            if (
                video_offsets[0].item() < audio_offsets[-1].item()
                and audio_offsets[0].item() < video_offsets[-1].item()
            ):
                return True

        span_start = span_end

    return False

merge_interleaved_embeddings(inputs_embeds, multimodal_embeddings, is_video, is_audio, is_multimodal)

Merge embeddings for interleaved audio-in-video sequences.

When use_audio_in_video=True, video and audio tokens are interleaved in the token sequence, but embeddings are provided as separate contiguous tensors. This function scatters each modality by the modality attribute attached to each embedding tensor (set during encoder gather) and also supports image embeddings in the same interleaved request.

Parameters:

• inputs_embeds

  (Tensor) –

  The input embeddings tensor to merge into.

• multimodal_embeddings

  (MultiModalEmbeddings) –

  List of embedding tensors (video, audio, image).

• is_video

  (Tensor) –

  Boolean mask for video token positions.

• is_audio

  (Tensor) –

  Boolean mask for audio token positions.

• is_multimodal

  (Tensor) –

  Boolean mask for all multimodal token positions.

Returns:

• Tensor –

  The merged inputs_embeds tensor with multimodal embeddings scattered

• Tensor –

  to their correct positions.

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

def merge_interleaved_embeddings(
    inputs_embeds: torch.Tensor,
    multimodal_embeddings: "MultiModalEmbeddings",
    is_video: torch.Tensor,
    is_audio: torch.Tensor,
    is_multimodal: torch.Tensor,
) -> torch.Tensor:
    """Merge embeddings for interleaved audio-in-video sequences.

    When use_audio_in_video=True, video and audio tokens are interleaved in
    the token sequence, but embeddings are provided as separate contiguous
    tensors. This function scatters each modality by the `modality` attribute
    attached to each embedding tensor (set during encoder gather) and also
    supports image embeddings in the same interleaved request.

    Args:
        inputs_embeds: The input embeddings tensor to merge into.
        multimodal_embeddings: List of embedding tensors (video, audio, image).
        is_video: Boolean mask for video token positions.
        is_audio: Boolean mask for audio token positions.
        is_multimodal: Boolean mask for all multimodal token positions.

    Returns:
        The merged inputs_embeds tensor with multimodal embeddings scattered
        to their correct positions.

    """
    from vllm.multimodal.utils import get_mm_embedding_modalities

    def _merge_embedding_group(
        mask: torch.Tensor,
        embeddings: Sequence[torch.Tensor],
        modality: str,
    ) -> None:
        num_expected_tokens = mask.sum().item()
        num_actual_tokens = sum(embedding.shape[0] for embedding in embeddings)
        if num_actual_tokens != num_expected_tokens:
            raise ValueError(
                f"Attempted to assign {num_actual_tokens} {modality} tokens "
                f"to {num_expected_tokens} placeholders"
            )
        if embeddings:
            inputs_embeds[mask] = torch.cat(list(embeddings), dim=0)

    # Qwen Omni multimodal placeholders are image/video/audio; after excluding
    # video and audio positions, the remaining multimodal positions are images.
    is_image = is_multimodal & ~is_video & ~is_audio

    embedding_modalities = get_mm_embedding_modalities(multimodal_embeddings)

    video_embeds = [
        embedding
        for embedding, modality in zip(multimodal_embeddings, embedding_modalities)
        if modality == "video"
    ]
    audio_embeds = [
        embedding
        for embedding, modality in zip(multimodal_embeddings, embedding_modalities)
        if modality == "audio"
    ]
    image_embeds = [
        embedding
        for embedding, modality in zip(multimodal_embeddings, embedding_modalities)
        if modality == "image"
    ]

    _merge_embedding_group(is_video, video_embeds, "video")
    _merge_embedding_group(is_audio, audio_embeds, "audio")
    _merge_embedding_group(is_image, image_embeds, "image")

    return inputs_embeds

unpad_and_flat_audio_features(input_audio_features, audio_feature_lengths)

Unpad and flatten stacked or ragged per-audio features.

Source code in vllm/model_executor/models/qwen2_5_omni_thinker.py

def unpad_and_flat_audio_features(
    input_audio_features: torch.Tensor | Sequence[torch.Tensor],
    audio_feature_lengths: torch.Tensor,
) -> torch.Tensor:
    """Unpad and flatten stacked or ragged per-audio features."""
    if len(audio_feature_lengths) != len(input_audio_features):
        raise ValueError(
            "Length of audio_feature_lengths must match "
            "the batch size of input_audio_features."
        )
    unpadded_features = [
        audio_feature[..., :length]
        for audio_feature, length in zip(input_audio_features, audio_feature_lengths)
    ]
    return torch.cat(unpadded_features, dim=-1)