Audio from video in jinja

@@ -223,7 +223,7 @@ def load_audio(audio: str | np.ndarray, sampling_rate=16000, timeout=None, backe     if not isinstance(audio, str):         raise TypeError(             "Incorrect format used for `audio`. Should be a numpy array or a `str`: an `http(s)://` URL, "-            "a local file path, or a base64-encoded string (optionally wrapped as a `data:...` URI)."+            f"a local file path, or a base64-encoded string (optionally wrapped as a `data:...` URI) but got {type(audio)}."         )      # torchcodec handles audio/video; librosa only plain audio. `backend` lets callers pin one.

@@ -30,7 +30,7 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel-from ...processing_utils import ProcessingKwargs, ProcessorMixin, Unpack+from ...processing_utils import AudioKwargs, OmniModalProcessorMixin, Unpack, VideosKwargs from ...tokenization_utils_base import PreTokenizedInput, TextInput from ...utils import (     TransformersKwargs,@@ -765,11 +765,14 @@ class EmbeddingGemma2VideoProcessorKwargs(Gemma4VideoProcessorKwargs):         The strategy used to cut the total number of sampled frames down to fit into the budget.         Can be set only to "uniform" or "truncate". Applied after FPS-based sampling, and on its         own when FPS-based sampling is off or not applicable.+    load_audio_from_video (`bool`, *optional*):+        Whether to load the audio track of an input video or not.     """      add_timestamps: bool     max_frames: int | None     overflow_strategy: str | None+    load_audio_from_video: bool   class EmbeddingGemma2VideoProcessor(Gemma4VideoProcessor):@@ -842,18 +845,14 @@ def sample_frames(  class EmbeddingGemma2ProcessorKwargs(Gemma4ProcessorKwargs):     _defaults = {-        "text_kwargs": {-            "padding": True,-        },-        "images_kwargs": {-            "do_convert_rgb": True,-        },+        "text_kwargs": {"padding": True},+        "images_kwargs": {"do_convert_rgb": True},         "audio_kwargs": {},         "videos_kwargs": {"return_metadata": True},     }  -class EmbeddingGemma2Processor(Gemma4Processor):+class EmbeddingGemma2Processor(OmniModalProcessorMixin, Gemma4Processor):     valid_processor_kwargs = EmbeddingGemma2ProcessorKwargs      def model_input_names(self):@@ -868,7 +867,7 @@ def prepare_inputs_layout(         **kwargs,     ):         # When `text` is None, record per-sample counts for `audio` and `videos` before-        # `ProcessorMixin.prepare_inputs_layout` runs `make_list_of_audio` and `make_batched_videos`,+        # `OmniModalProcessorMixin.prepare_inputs_layout` runs `make_list_of_audio` and `make_batched_videos`,         # which flatten 2D nested per-sample lists into a 1D list of total items and discard sample         # boundaries. `images` does not need this because `make_nested_list_of_images` preserves the         # outer per-sample list structure.@@ -887,7 +886,7 @@ def prepare_inputs_layout(                 else None             ) -        images, text, videos, audio = ProcessorMixin.prepare_inputs_layout(+        images, text, videos, audio = OmniModalProcessorMixin.prepare_inputs_layout(             self, images=images, text=text, videos=videos, audio=audio, **kwargs         ) @@ -929,9 +928,9 @@ def validate_inputs(         text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] = None,         videos: VideoInput = None,         audio: AudioInput = None,-        **kwargs: Unpack[ProcessingKwargs],+        **kwargs: Unpack[EmbeddingGemma2ProcessorKwargs],     ):-        ProcessorMixin.validate_inputs(self, images=images, text=text, **kwargs)+        OmniModalProcessorMixin.validate_inputs(self, images=images, text=text, **kwargs)          # Unlike Gemma 4, any single modality on its own is a valid embedding input.         if text is None and images is None and videos is None and audio is None:@@ -985,32 +984,175 @@ def validate_inputs(                         f"Found {sum(n_audio_in_text)} {self.audio_token} tokens in the text but no audio inputs were passed."                     ) -    def replace_video_token(self, video_inputs: dict, video_idx: int, **kwargs) -> str:+    @staticmethod+    def timestamp_fmt(sec: int | float) -> str:+        # Round (not floor): at fractional fps (e.g. 29.97) sampled timestamps land just below the nominal second+        sec = round(sec)+        return f"{sec // 60:02d}:{sec % 60:02d}"++    def replace_video_with_audio_token(+        self,+        video_inputs: dict,+        video_idx: int,+        processed_audio: dict | None = None,+        video_to_audio_indices: list | None = None,+        **kwargs: VideosKwargs,+    ) -> str:+        # When `load_audio_from_video` -> format is `mm:ss, vision_tokens, audio_tokens, mm:ss, vision_tokens, audio_tokens [...]`+        metadata = video_inputs["video_metadata"][video_idx]+        num_soft_tokens = video_inputs["num_soft_tokens_per_video"][video_idx]++        # Round (not floor) so 29.97fps timestamps like 0.9676 / 1.9686 land on 1s / 2s+        frame_seconds = [round(s) for s in metadata.timestamps]+        num_frames = len(frame_seconds)++        # Mirrors `max(1, int(metadata.duration * fps))` in `sample_frames` for fps=1+        total_seconds = max(1, int(metadata.duration))++        num_audio_tokens_per_sec = self.audio_frame_boundary(seconds=1)++        mapped_audio_idx = video_to_audio_indices[video_idx]+        audio_mask = processed_audio["input_features_mask"][mapped_audio_idx]+        num_audio_tokens_total = self.compute_sscp(audio_mask)++        # Each 1s chunk of video/audio is wrapped in special tokens:+        # "00:00 <BOI> [frame image tokens] <EOI> 00:00 <BOA> [audio tokens] <EOA> ..."+        segments = []+        audio_secs = range(frame_seconds[0], total_seconds)+        counts = [+            min(num_audio_tokens_per_sec, max(0, num_audio_tokens_total - k * num_audio_tokens_per_sec))+            for k in range(len(audio_secs))+        ]+        # Token count per audio second; the last chunk absorbs rounding/sscp leftovers+        if counts:+            counts[-1] += num_audio_tokens_total - sum(counts)+        audio_tokens = dict(zip(audio_secs, counts))+        for i, frame_sec in enumerate(frame_seconds):+            segments.append(+                f"{self.timestamp_fmt(frame_sec)} {self.boi_token}{self.video_token * num_soft_tokens}{self.eoi_token}"+            )++            # 1s audio chunks that follow this frame before the next one+            end_sec = frame_seconds[i + 1] if i < num_frames - 1 else total_seconds+            for sec in range(frame_sec, end_sec):+                if audio_tokens.get(sec, 0) > 0:+                    segments.append(+                        f"{self.timestamp_fmt(sec)} {self.boa_token}{self.audio_token * audio_tokens[sec]}{self.eoa_token}"+                    )++        return " ".join(segments)++    def replace_video_token(+        self,+        video_inputs: dict,+        video_idx: int,+        processed_audio: dict | None = None,+        video_to_audio_indices: list | None = None,+        **kwargs: VideosKwargs,+    ) -> str:         num_soft_tokens = video_inputs["num_soft_tokens_per_video"][video_idx]         add_timestamps = kwargs.get("add_timestamps", self.video_processor.add_timestamps)          # Visual-only mode: one block per frame, no timestamps         if not add_timestamps:+            if kwargs.get("load_audio_from_video"):+                raise ValueError("You must allow adding timestamps when `load_audio_from_video=True`!")+             # `pixel_values_videos` is a flat frame sequence, so its leading axis indexes frames, not videos             num_frames = int(video_inputs["num_frames_per_video"][video_idx])             frame_str = f"{self.boi_token}{self.video_token * num_soft_tokens}{self.eoi_token}"             return "".join([frame_str] * num_frames)          metadata = video_inputs["video_metadata"][video_idx]-         if metadata.fps is None:             raise ValueError(                 "Asked to build a prompt with frame timestamps, but no `fps` was provided in video metadata. "                 "The capture rate of already-decoded frames cannot be inferred. Please pass a `VideoMetadata` "                 "object with a valid `fps`, or set `add_timestamps=False`."             ) -        # mm:ss format for timestamps-        timestamp_str = [f"{int(seconds // 60):02d}:{int(seconds % 60):02d}" for seconds in metadata.timestamps]+        # When loading audio from video and if current video has an associated audio, i.e. no silent+        if kwargs.get("load_audio_from_video") and video_to_audio_indices[video_idx] is not None:+            return self.replace_video_with_audio_token(+                video_inputs, video_idx, processed_audio, video_to_audio_indices, **kwargs+            )++        # General timestamp format is `mm:ss` per each video frame/tokens         return " ".join(-            [f"{t} {self.boi_token}{self.video_token * num_soft_tokens}{self.eoi_token}" for t in timestamp_str]+            [+                f"{self.timestamp_fmt(s)} {self.boi_token}{self.video_token * num_soft_tokens}{self.eoi_token}"+                for s in metadata.timestamps+            ]         ) +    def replace_audio_token(self, audio_inputs: dict, audio_idx: int, **kwargs: AudioKwargs) -> str:+        mask = audio_inputs["input_features_mask"][audio_idx]+        num_audio_tokens_total = self.compute_sscp(mask)+        return f"{self.boa_token}{self.audio_token * num_audio_tokens_total}{self.eoa_token}"++    def _compute_audio_num_tokens(self, audio_waveform, sampling_rate: int) -> int:+        """Number of audio soft tokens, replicating the encoder's seq-length arithmetic.++        Mirrors EmbeddingGemma2AudioFeatureExtractor mel framing + the two stride-2 Conv2d+        subsampling layers in EmbeddingGemma2AudioSubSampleConvProjection, capped at+        ``audio_seq_length``. Must match ``audio_mask.sum()`` from the audio tower or+        vLLM's ``_merge_multimodal_embeddings`` will raise on a length mismatch.++        Args:+            audio_waveform: A 1-D array or list containing the raw audio samples.+            sampling_rate: The sampling rate of the audio waveform in Hz.++        Returns:+            The number of audio soft tokens to insert as placeholders.+        """+        num_samples = len(audio_waveform)++        # Step 1: mel frames (matches feature_extraction_embedding_gemma2.py)+        frame_size_for_unfold = self.feature_extractor.frame_length + 1+        pad_left = self.feature_extractor.frame_length // 2  # semicausal time padding+        num_mel_frames = (num_samples + pad_left - frame_size_for_unfold) // self.feature_extractor.hop_length + 1+        if num_mel_frames <= 0:+            return 0++        # Step 2: SSCP subsampling. Two stride-2 Conv2d layers (kernel=3, pad=1)+        num_audio_tokens = self.compute_sscp(num_mel_frames)++        # Cap at the configured maximum+        return min(num_audio_tokens, self.audio_seq_length)++    def audio_frame_boundary(self, seconds: float) -> int:+        """Number of `audio_tokens` whose analysis window falls within [0, seconds)."""+        # early exit if audio truncated and current video frame has no corresponding track+        if seconds == 0:+            return 0++        input_raw_length = round(seconds * self.feature_extractor.sampling_rate)+        boundary = (+            input_raw_length - self.feature_extractor.frame_length // 2 - 1+        ) // self.feature_extractor.hop_length + 1+        mels_per_chunk = max(boundary, 0)+        return self.compute_sscp(mels_per_chunk)++    @staticmethod+    def compute_sscp(mask_or_len: int | np.ndarray):+        """+        SSCP subsampling. Two stride-2 Conv2d layers (kernel=3, pad=1)+        values same as in EmbeddingGemma2AudioSubSampleConvProjection(Layer)+        """+        sscp_num_layers = 2+        sscp_kernel, sscp_stride, sscp_padding = 3, 2, 1+        if isinstance(mask_or_len, (int, np.integer)):+            for _ in range(sscp_num_layers):+                # Conv1d/2d output-length along the time axis (dilation=1).+                mask_or_len = (mask_or_len + 2 * sscp_padding - sscp_kernel) // sscp_stride + 1+            return mask_or_len++        for _ in range(2):+            mask_or_len = mask_or_len[::sscp_stride][+                : (len(mask_or_len) + 2 * sscp_padding - sscp_kernel) // sscp_stride + 1+            ]+        return int(mask_or_len.sum())+  __all__ = [     "EmbeddingGemma2Config",

@@ -24,7 +24,14 @@  from ...audio_utils import AudioInput, is_valid_audio from ...image_utils import ImageInput, make_nested_list_of_images-from ...processing_utils import MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack+from ...processing_utils import (+    AudioKwargs,+    MultiModalData,+    OmniModalProcessorMixin,+    ProcessingKwargs,+    Unpack,+    VideosKwargs,+) from ...tokenization_utils_base import PreTokenizedInput, TextInput from ...utils import (     auto_docstring,@@ -34,12 +41,8 @@  class EmbeddingGemma2ProcessorKwargs(ProcessingKwargs, total=False):     _defaults = {-        "text_kwargs": {-            "padding": True,-        },-        "images_kwargs": {-            "do_convert_rgb": True,-        },+        "text_kwargs": {"padding": True},+        "images_kwargs": {"do_convert_rgb": True},         "audio_kwargs": {},         "videos_kwargs": {"return_metadata": True},     }@@ -100,7 +103,7 @@ def get_aspect_ratio_preserving_size(   @auto_docstring-class EmbeddingGemma2Processor(ProcessorMixin):+class EmbeddingGemma2Processor(OmniModalProcessorMixin):     valid_processor_kwargs = EmbeddingGemma2ProcessorKwargs      def __init__(@@ -168,7 +171,7 @@ def prepare_inputs_layout(         **kwargs,     ):         # When `text` is None, record per-sample counts for `audio` and `videos` before-        # `ProcessorMixin.prepare_inputs_layout` runs `make_list_of_audio` and `make_batched_videos`,+        # `OmniModalProcessorMixin.prepare_inputs_layout` runs `make_list_of_audio` and `make_batched_videos`,         # which flatten 2D nested per-sample lists into a 1D list of total items and discard sample         # boundaries. `images` does not need this because `make_nested_list_of_images` preserves the         # outer per-sample list structure.@@ -229,7 +232,7 @@ def validate_inputs(         text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] = None,         videos: VideoInput = None,         audio: AudioInput = None,-        **kwargs: Unpack[ProcessingKwargs],+        **kwargs: Unpack[EmbeddingGemma2ProcessorKwargs],     ):         super().validate_inputs(images=images, text=text, **kwargs) @@ -289,44 +292,53 @@ def replace_image_token(self, image_inputs: dict, image_idx: int, **kwargs) -> s         num_soft_tokens = image_inputs["num_soft_tokens_per_image"][image_idx]         return f"{self.boi_token}{self.image_token * num_soft_tokens}{self.eoi_token}" -    def replace_video_token(self, video_inputs: dict, video_idx: int, **kwargs) -> str:+    def replace_video_token(+        self,+        video_inputs: dict,+        video_idx: int,+        processed_audio: dict | None = None,+        video_to_audio_indices: list | None = None,+        **kwargs: VideosKwargs,+    ) -> str:         num_soft_tokens = video_inputs["num_soft_tokens_per_video"][video_idx]         add_timestamps = kwargs.get("add_timestamps", self.video_processor.add_timestamps)          # Visual-only mode: one block per frame, no timestamps         if not add_timestamps:+            if kwargs.get("load_audio_from_video"):+                raise ValueError("You must allow adding timestamps when `load_audio_from_video=True`!")+             # `pixel_values_videos` is a flat frame sequence, so its leading axis indexes frames, not videos             num_frames = int(video_inputs["num_frames_per_video"][video_idx])             frame_str = f"{self.boi_token}{self.video_token * num_soft_tokens}{self.eoi_token}"             return "".join([frame_str] * num_frames)          metadata = video_inputs["video_metadata"][video_idx]-         if metadata.fps is None:             raise ValueError(                 "Asked to build a prompt with frame timestamps, but no `fps` was provided in video metadata. "                 "The capture rate of already-decoded frames cannot be inferred. Please pass a `VideoMetadata` "                 "object with a valid `fps`, or set `add_timestamps=False`."             ) -        # mm:ss format for timestamps-        timestamp_str = [f"{int(seconds // 60):02d}:{int(seconds % 60):02d}" for seconds in metadata.timestamps]+        # When loading audio from video and if current video has an associated audio, i.e. no silent+        if kwargs.get("load_audio_from_video") and video_to_audio_indices[video_idx] is not None:+            return self.replace_video_with_audio_token(+                video_inputs, video_idx, processed_audio, video_to_audio_indices, **kwargs+            )++        # General timestamp format is `mm:ss` per each video frame/tokens         return " ".join(-            [f"{t} {self.boi_token}{self.video_token * num_soft_tokens}{self.eoi_token}" for t in timestamp_str]+            [+                f"{self.timestamp_fmt(s)} {self.boi_token}{self.video_token * num_soft_tokens}{self.eoi_token}"+                for s in metadata.timestamps+            ]         ) -    def replace_audio_token(self, audio_inputs: dict, audio_idx: int, **kwargs) -> str:-        # TODO: Add tests for audio-only processor inputs.+    def replace_audio_token(self, audio_inputs: dict, audio_idx: int, **kwargs: AudioKwargs) -> str:         mask = audio_inputs["input_features_mask"][audio_idx]--        # Simulate two stride-2 conv blocks on the mask-        t = len(mask)-        for _ in range(2):-            t_out = (t + 2 - 3) // 2 + 1-            mask = mask[::2][:t_out]-            t = len(mask)--        return f"{self.boa_token}{self.audio_token * int(mask.sum())}{self.eoa_token}"+        num_audio_tokens_total = self.compute_sscp(mask)+        return f"{self.boa_token}{self.audio_token * num_audio_tokens_total}{self.eoa_token}"      def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, audio_lengths=None, **kwargs):         """@@ -435,24 +447,105 @@ def _compute_audio_num_tokens(self, audio_waveform, sampling_rate: int) -> int:             return 0          # Step 2: SSCP subsampling. Two stride-2 Conv2d layers (kernel=3, pad=1)-        # from EmbeddingGemma2AudioSubSampleConvProjection(Layer) in modeling_embedding_gemma2.py.-        # These are architectural constants, not exposed via EmbeddingGemma2AudioConfig-        # (note: config.conv_kernel_size=5 is the *conformer* depthwise conv, NOT-        # this one). The layer count tracks len(subsampling_conv_channels)==2.-        # Keep in sync if the audio encoder's subsampling stack changes.-        sscp_num_layers = 2-        sscp_kernel, sscp_stride, sscp_padding = 3, 2, 1-        t = num_mel_frames-        for _ in range(sscp_num_layers):-            # Conv1d/2d output-length along the time axis (dilation=1).-            t = (t + 2 * sscp_padding - sscp_kernel) // sscp_stride + 1+        num_audio_tokens = self.compute_sscp(num_mel_frames)          # Cap at the configured maximum-        return min(t, self.audio_seq_length)+        return min(num_audio_tokens, self.audio_seq_length)      @property     def unused_input_names(self) -> list[str]:         return ["num_soft_tokens_per_image", "num_soft_tokens_per_video"] +    @staticmethod+    def timestamp_fmt(sec: int | float) -> str:+        # Round (not floor): at fractional fps (e.g. 29.97) sampled timestamps land just below the nominal second+        sec = round(sec)+        return f"{sec // 60:02d}:{sec % 60:02d}"++    def replace_video_with_audio_token(+        self,+        video_inputs: dict,+        video_idx: int,+        processed_audio: dict | None = None,+        video_to_audio_indices: list | None = None,+        **kwargs: VideosKwargs,+    ) -> str:+        # When `load_audio_from_video` -> format is `mm:ss, vision_tokens, audio_tokens, mm:ss, vision_tokens, audio_tokens [...]`+        metadata = video_inputs["video_metadata"][video_idx]+        num_soft_tokens = video_inputs["num_soft_tokens_per_video"][video_idx]++        # Round (not floor) so 29.97fps timestamps like 0.9676 / 1.9686 land on 1s / 2s+        frame_seconds = [round(s) for s in metadata.timestamps]+        num_frames = len(frame_seconds)++        # Mirrors `max(1, int(metadata.duration * fps))` in `sample_frames` for fps=1+        total_seconds = max(1, int(metadata.duration))++        num_audio_tokens_per_sec = self.audio_frame_boundary(seconds=1)++        mapped_audio_idx = video_to_audio_indices[video_idx]+        audio_mask = processed_audio["input_features_mask"][mapped_audio_idx]+        num_audio_tokens_total = self.compute_sscp(audio_mask)++        # Each 1s chunk of video/audio is wrapped in special tokens:+        # "00:00 <BOI> [frame image tokens] <EOI> 00:00 <BOA> [audio tokens] <EOA> ..."+        segments = []+        audio_secs = range(frame_seconds[0], total_seconds)+        counts = [+            min(num_audio_tokens_per_sec, max(0, num_audio_tokens_total - k * num_audio_tokens_per_sec))+            for k in range(len(audio_secs))+        ]+        # Token count per audio second; the last chunk absorbs rounding/sscp leftovers+        if counts:+            counts[-1] += num_audio_tokens_total - sum(counts)+        audio_tokens = dict(zip(audio_secs, counts))+        for i, frame_sec in enumerate(frame_seconds):+            segments.append(+                f"{self.timestamp_fmt(frame_sec)} {self.boi_token}{self.video_token * num_soft_tokens}{self.eoi_token}"+            )++            # 1s audio chunks that follow this frame before the next one+            end_sec = frame_seconds[i + 1] if i < num_frames - 1 else total_seconds+            for sec in range(frame_sec, end_sec):+                if audio_tokens.get(sec, 0) > 0:+                    segments.append(+                        f"{self.timestamp_fmt(sec)} {self.boa_token}{self.audio_token * audio_tokens[sec]}{self.eoa_token}"+                    )++        return " ".join(segments)++    def audio_frame_boundary(self, seconds: float) -> int:+        """Number of `audio_tokens` whose analysis window falls within [0, seconds)."""+        # early exit if audio truncated and current video frame has no corresponding track+        if seconds == 0:+            return 0++        input_raw_length = round(seconds * self.feature_extractor.sampling_rate)+        boundary = (+            input_raw_length - self.feature_extractor.frame_length // 2 - 1+        ) // self.feature_extractor.hop_length + 1+        mels_per_chunk = max(boundary, 0)+        return self.compute_sscp(mels_per_chunk)++    @staticmethod+    def compute_sscp(mask_or_len: int | np.ndarray):+        """+        SSCP subsampling. Two stride-2 Conv2d layers (kernel=3, pad=1)+        values same as in EmbeddingGemma2AudioSubSampleConvProjection(Layer)+        """+        sscp_num_layers = 2+        sscp_kernel, sscp_stride, sscp_padding = 3, 2, 1+        if isinstance(mask_or_len, (int, np.integer)):+            for _ in range(sscp_num_layers):+                # Conv1d/2d output-length along the time axis (dilation=1).+                mask_or_len = (mask_or_len + 2 * sscp_padding - sscp_kernel) // sscp_stride + 1+            return mask_or_len++        for _ in range(2):+            mask_or_len = mask_or_len[::sscp_stride][+                : (len(mask_or_len) + 2 * sscp_padding - sscp_kernel) // sscp_stride + 1+            ]+        return int(mask_or_len.sum())+  __all__ = ["EmbeddingGemma2Processor"]

@@ -69,6 +69,8 @@ class EmbeddingGemma2VideoProcessorKwargs(VideosKwargs, total=False):         The strategy used to cut the total number of sampled frames down to fit into the budget.         Can be set only to "uniform" or "truncate". Applied after FPS-based sampling, and on its         own when FPS-based sampling is off or not applicable.+    load_audio_from_video (`bool`, *optional*):+        Whether to load the audio track of an input video or not.     """      patch_size: int@@ -78,6 +80,7 @@ class EmbeddingGemma2VideoProcessorKwargs(VideosKwargs, total=False):     add_timestamps: bool     max_frames: int | None     overflow_strategy: str | None+    load_audio_from_video: bool   _SUPPORTED_SOFT_TOKENS = (70, 140, 280, 560, 1120)

@@ -22,12 +22,15 @@  from ...feature_extraction_utils import BatchFeature from ...image_utils import ImageInput-from ...processing_utils import ProcessingKwargs, ProcessorMixin, Unpack, VideosKwargs+from ...processing_utils import OmniModalProcessorMixin, ProcessingKwargs, Unpack, VideosKwargs from ...tokenization_utils_base import AudioInput, PreTokenizedInput, TextInput-from ...utils import auto_docstring+from ...utils import auto_docstring, logging from ...video_utils import VideoInput  +logger = logging.get_logger(__name__)++ # Redefine kwargs for videos because Qwen-Omni uses some kwargs for processing omni # and does not use them in video processor class class Qwen2_5_OmniVideosKwargs(VideosKwargs, total=False):@@ -53,9 +56,11 @@ class Qwen2_5_OmniVideosKwargs(VideosKwargs, total=False):     max_frames (`int`, *optional*):         Maximum number of frames to extract from the video. Longer videos will be truncated or sampled to fit         within this limit.-    use_audio_in_video (`bool`, *optional*, defaults to `False`):+    load_audio_from_video (`bool`, *optional*, defaults to `False`):         Whether to incorporate audio information when processing videos. When enabled, audio tokens are         interleaved with video tokens based on temporal alignment, creating a unified multimodal representation.+    use_audio_in_video (`bool`, *optional*, defaults to `False`):+        Deprecated arg - use `load_audio_from_video` instead.     seconds_per_chunk (`float`, *optional*, defaults to `2.0`):         The duration (in seconds) of each video chunk when splitting long videos. This parameter controls how         videos are divided into temporal segments for processing.@@ -72,6 +77,7 @@ class Qwen2_5_OmniVideosKwargs(VideosKwargs, total=False):     min_frames: int     max_frames: int     use_audio_in_video: bool+    load_audio_from_video: bool     seconds_per_chunk: float     position_id_per_seconds: int | float @@ -102,7 +108,7 @@ class Qwen2_5OmniProcessorKwargs(ProcessingKwargs, total=False):   @auto_docstring-class Qwen2_5OmniProcessor(ProcessorMixin):+class Qwen2_5OmniProcessor(OmniModalProcessorMixin):     valid_processor_kwargs = Qwen2_5OmniProcessorKwargs      def __init__(@@ -138,6 +144,19 @@ def __call__(         seconds_per_chunk = output_kwargs["videos_kwargs"].pop("seconds_per_chunk")         position_id_per_seconds = output_kwargs["videos_kwargs"].pop("position_id_per_seconds")         use_audio_in_video = output_kwargs["videos_kwargs"].pop("use_audio_in_video")+        load_audio_from_video = output_kwargs["videos_kwargs"].pop("load_audio_from_video", False)+        if use_audio_in_video:+            logger.warning(+                "`use_audio_in_video` is depreacted and will be removed in v5.25, use `load_audio_from_video` instead"+            )+            load_audio_from_video = True++        if load_audio_from_video:+            audio, video_to_audio_indices = self._resolve_audio_from_video(+                videos,+                audio=audio,+                audio_from_video_indices=None,+            )          if audio is not None:             output_kwargs["audio_kwargs"]["padding"] = "max_length"  # Support "max_length" padding only here@@ -187,7 +206,7 @@ def __call__(                 image_grid_thw,                 video_grid_thw,                 video_second_per_grid=video_second_per_grid,-                use_audio_in_video=use_audio_in_video,+                use_audio_in_video=load_audio_from_video,                 position_id_per_seconds=position_id_per_seconds,                 seconds_per_chunk=seconds_per_chunk,             )

@@ -43,7 +43,7 @@ ) from ...modeling_rope_utils import RopeParameters from ...modeling_utils import PreTrainedModel-from ...processing_utils import ProcessorMixin, Unpack+from ...processing_utils import OmniModalProcessorMixin, Unpack from ...tokenization_utils_base import TextInput from ...utils import auto_docstring, can_return_tuple, logging from ...utils.generic import (@@ -2772,7 +2772,7 @@ class Qwen3OmniMoeProcessorKwargs(Qwen2_5OmniProcessorKwargs):     }  -class Qwen3OmniMoeProcessor(Qwen2_5OmniProcessor, ProcessorMixin):+class Qwen3OmniMoeProcessor(Qwen2_5OmniProcessor):     def replace_multimodal_special_tokens(         self,         text,@@ -2874,6 +2874,19 @@ def __call__(         fps = output_kwargs["videos_kwargs"].get("fps", 1.0)         fps = fps if fps is not None else 1.0         n_window = output_kwargs["audio_kwargs"].pop("n_window", 50)+        load_audio_from_video = output_kwargs["videos_kwargs"].pop("load_audio_from_video", False)+        if use_audio_in_video:+            logger.warning(+                "`use_audio_in_video` is depreacted and will be removed in v5.25, use `load_audio_from_video` instead"+            )+            load_audio_from_video = True++        if load_audio_from_video:+            audio, video_to_audio_indices = self._resolve_audio_from_video(+                videos,+                audio=audio,+                audio_from_video_indices=None,+            )          if audio is not None:             audio_inputs = self.feature_extractor(audio, **output_kwargs["audio_kwargs"])@@ -2919,7 +2932,7 @@ def __call__(             image_grid_thw,             video_grid_thw,             video_second_per_grid=video_second_per_grid,-            use_audio_in_video=use_audio_in_video,+            use_audio_in_video=load_audio_from_video,             position_id_per_seconds=position_id_per_seconds,             seconds_per_chunk=seconds_per_chunk,         )@@ -2932,7 +2945,7 @@ def __call__(         )      def apply_chat_template(self, conversations, chat_template=None, **kwargs):-        return ProcessorMixin.apply_chat_template(self, conversations, chat_template, **kwargs)+        return OmniModalProcessorMixin.apply_chat_template(self, conversations, chat_template, **kwargs)   __all__ = [

@@ -25,12 +25,15 @@ from ...audio_utils import AudioInput from ...feature_extraction_utils import BatchFeature from ...image_utils import ImageInput-from ...processing_utils import ProcessingKwargs, ProcessorMixin, Unpack, VideosKwargs+from ...processing_utils import OmniModalProcessorMixin, ProcessingKwargs, Unpack, VideosKwargs from ...tokenization_utils_base import TextInput-from ...utils import auto_docstring+from ...utils import auto_docstring, logging from ...video_utils import VideoInput  +logger = logging.get_logger(__name__)++ # Redefine kwargs for videos because Qwen-Omni uses some kwargs for processing omni # and does not use them in video processor class class Qwen3OmniMoeVideosKwargs(VideosKwargs, total=False):@@ -56,9 +59,11 @@ class Qwen3OmniMoeVideosKwargs(VideosKwargs, total=False):     max_frames (`int`, *optional*):         Maximum number of frames to extract from the video. Longer videos will be truncated or sampled to fit         within this limit.-    use_audio_in_video (`bool`, *optional*, defaults to `False`):+    load_audio_from_video (`bool`, *optional*, defaults to `False`):         Whether to incorporate audio information when processing videos. When enabled, audio tokens are         interleaved with video tokens based on temporal alignment, creating a unified multimodal representation.+    use_audio_in_video (`bool`, *optional*, defaults to `False`):+        Deprecated arg - use `load_audio_from_video` instead.     seconds_per_chunk (`float`, *optional*, defaults to `2.0`):         The duration (in seconds) of each video chunk when splitting long videos. This parameter controls how         videos are divided into temporal segments for processing.@@ -75,6 +80,7 @@ class Qwen3OmniMoeVideosKwargs(VideosKwargs, total=False):     min_frames: int     max_frames: int     use_audio_in_video: bool+    load_audio_from_video: bool     seconds_per_chunk: float     position_id_per_seconds: int | float @@ -116,7 +122,7 @@ def _get_feat_extract_output_lengths(input_lengths, n_window=50):   @auto_docstring-class Qwen3OmniMoeProcessor(ProcessorMixin):+class Qwen3OmniMoeProcessor(OmniModalProcessorMixin):     valid_processor_kwargs = Qwen3OmniMoeProcessorKwargs      def __init__(@@ -155,6 +161,19 @@ def __call__(         fps = output_kwargs["videos_kwargs"].get("fps", 1.0)         fps = fps if fps is not None else 1.0         n_window = output_kwargs["audio_kwargs"].pop("n_window", 50)+        load_audio_from_video = output_kwargs["videos_kwargs"].pop("load_audio_from_video", False)+        if use_audio_in_video:+            logger.warning(+                "`use_audio_in_video` is depreacted and will be removed in v5.25, use `load_audio_from_video` instead"+            )+            load_audio_from_video = True++        if load_audio_from_video:+            audio, video_to_audio_indices = self._resolve_audio_from_video(+                videos,+                audio=audio,+                audio_from_video_indices=None,+            )          if audio is not None:             audio_inputs = self.feature_extractor(audio, **output_kwargs["audio_kwargs"])@@ -200,7 +219,7 @@ def __call__(             image_grid_thw,             video_grid_thw,             video_second_per_grid=video_second_per_grid,-            use_audio_in_video=use_audio_in_video,+            use_audio_in_video=load_audio_from_video,             position_id_per_seconds=position_id_per_seconds,             seconds_per_chunk=seconds_per_chunk,         )

@@ -57,6 +57,8 @@     direct_transformers_import,     hf_api,     is_torch_available,+    is_torchcodec_available,+    is_torchcodec_greater_or_equal,     list_repo_templates,     logging,     resolve_revision,@@ -76,6 +78,9 @@ from .video_utils import VideoInput, VideoMetadataType, make_batched_videos  +if is_torchcodec_available():+    import torchcodec+ if is_torch_available():     import torch @@ -731,8 +736,7 @@ def prepare_inputs_layout(             text = list(text).copy()          if audio is not None and self._audio_processor is not None:-            sampling_rate = kwargs.get("sampling_rate", self._audio_processor.sampling_rate)-            audio = self._audio_processor.fetch_audio(audio, sampling_rate=sampling_rate)+            audio = self._audio_processor.fetch_audio(audio, sampling_rate=self._audio_processor.sampling_rate)             audio = make_list_of_audio(audio)          if images is not None and hasattr(self, "image_processor"):@@ -1991,72 +1995,90 @@ def validate_init_kwargs(processor_config, valid_kwargs):          return unused_kwargs, valid_kwargs -    def apply_chat_template(+    def _multimodal_inputs_from_conversations(         self,-        conversation: list[dict[str, str]] | list[list[dict[str, str]]],-        chat_template: str | None = None,-        tools: list[dict] | None = None,-        documents: list[dict[str, str]] | None = None,-        add_generation_prompt: bool = False,-        continue_final_message: bool | str = False,-        return_assistant_tokens_mask: bool = False,-        tokenize: bool = False,-        return_tensors: str | TensorType | None = None,-        return_dict: bool = False,+        conversations: list[list[dict[str, str]]],+        audio_loading_params: dict,         load_audio_from_video: bool = False,-        processor_kwargs: dict | None = None,-        **kwargs,-    ) -> str:+    ):         """-        Similar to the `apply_chat_template` method on tokenizers, this method applies a Jinja template to input-        conversations to turn them into a single tokenizable string.+        Traverse through conversation and extract all provided multimodal data,+        including URL/path or already decoded inputs. Images and video are returned+        in nested format while audio is flattened into a single list for all batches.+        """+        batch_images, batch_videos, batch_audios = [], [], []+        for conversation in conversations:+            images, videos = [], []+            for message in conversation:+                content = message.get("content") or []+                if isinstance(content, str):+                    continue -        The input is expected to be in the following format, where each message content is a list consisting of text and-        optionally image or video inputs. One can also provide an image, video, URL or local path which will be used to form-        `pixel_values` when `return_dict=True`. If not provided, one will get only the formatted text, optionally tokenized text.+                new_content = []  # don't modify user's dict in-place!++                # Single pass so that audio keeps the exact order of the conversation.+                # Audio models do not accept nested list of audios (yet!) so we construct a flat input audio list+                for block, next_block in zip(content, [*content[1:], None]):+                    new_content.append(block)+                    block_type = block["type"]+                    if block_type == "image":+                        images.extend(block[key] for key in ["image", "url", "path", "base64"] if key in block)+                    elif block_type == "video":+                        video_fnames = [block[key] for key in ["video", "url", "path"] if key in block]+                        videos.extend(video_fnames)+                        if load_audio_from_video:+                            batch_audios.extend(load_audio(fname, **audio_loading_params) for fname in video_fnames)+                            # One audio entry per video, right after it, so the template emits the `audio` token in the same spot.+                            # Deprecated since we dont want to dummy-insert, let users choose where to put audio!+                            if next_block != {"type": "audio"}:+                                new_content.append({"type": "audio"})+                                logger.warning_once(+                                    "When setting `load_audio_from_video=True` you must add an empty `audio` entry after each video in the conversation. "+                                    "From v5.25 the entry will NOT be added automatically when calling `processor.apply_chat_template()`"+                                )+                    elif block_type == "audio":+                        audio_fnames = [block[key] for key in ["audio", "url", "path"] if key in block]+                        batch_audios.extend(load_audio(fname, **audio_loading_params) for fname in audio_fnames) -        conversation = [-            {-                "role": "user",-                "content": [-                    {"type": "image", "url": "https://www.ilankelman.org/stopsigns/australia.jpg"},-                    {"type": "text", "text": "Please describe this image in detail."},-                ],-            },-        ]+                message["content"] = new_content -        Args:-            conversation (`Union[list[Dict, [str, str]], list[list[dict[str, str]]]]`):-                The conversation to format.-            chat_template (`Optional[str]`, *optional*):-                The Jinja template to use for formatting the conversation. If not provided, the tokenizer's-                chat template is used.-        """-        processor_kwargs = processor_kwargs or {}+            # Currently all processors can accept nested list of batches, but not flat list of visuals+            # So we'll make a batched list of images and let the processor handle it+            batch_images.append(images)+            batch_videos.append(videos)+        return batch_images, batch_videos, batch_audios, {} +    def _resolve_chat_template(self, chat_template: str | None) -> str:+        """Return the template string to render, resolving defaults and template names."""         if chat_template is None:             if isinstance(self.chat_template, dict) and "default" in self.chat_template:-                chat_template = self.chat_template["default"]-            elif isinstance(self.chat_template, dict):+                return self.chat_template["default"]+            if isinstance(self.chat_template, dict):                 raise ValueError(                     'The processor has multiple chat templates but none of them are named "default". You need to specify'                     " which one to use by passing the `chat_template` argument. Available templates are: "                     f"{', '.join(self.chat_template.keys())}"                 )-            elif self.chat_template is not None:-                chat_template = self.chat_template-            else:-                raise ValueError(-                    "Cannot use apply_chat_template because this processor does not have a chat template."-                )-        else:-            if isinstance(self.chat_template, dict) and chat_template in self.chat_template:-                # It's the name of a template, not a full template string-                chat_template = self.chat_template[chat_template]-            else:-                # It's a template string, render it directly-                pass+            if self.chat_template is not None:+                return self.chat_template+            raise ValueError("Cannot use apply_chat_template because this processor does not have a chat template.") +        if isinstance(self.chat_template, dict) and chat_template in self.chat_template:+            # It's the name of a template, not a full template string+            return self.chat_template[chat_template]+        # It's a template string, render it directly+        return chat_template++    def _prepare_processor_kwargs(+        self,+        chat_template: str,+        processor_kwargs: dict,+        kwargs: dict,+        add_generation_prompt: bool,+        continue_final_message: bool,+        return_assistant_tokens_mask: bool,+    ) -> dict:+        """Handle legacy **kwargs, validate flag combinations, and set offset-mapping options."""         # Users might still be passing processing kwargs in `**kwargs` so we need to filter         # out additional kwargs that the template expects via Jinja2 template introspection         template_kwargs = _get_template_variables(chat_template)@@ -2091,31 +2113,32 @@ def apply_chat_template(                     "`return_assistant_tokens_mask` is not possible with slow tokenizers. Make sure you have `tokenizers` installed. "                     "If the error persists, open an issue to support a Fast tokenizer for your model."                 )-            else:-                processor_kwargs["return_offsets_mapping"] = (-                    True  # force offset mapping so we can infer token boundaries-                )-                processor_kwargs["return_text_replacement_offsets"] = True+            # force offset mapping so we can infer token boundaries+            processor_kwargs["return_offsets_mapping"] = True+            processor_kwargs["return_text_replacement_offsets"] = True++        return processor_kwargs -        # Set the sampling rate to load the audio files if user hasn't already passed with `kwargs`.+    def _resolve_audio_loading_params(self, processor_kwargs: dict) -> dict:+        """Pick the sampling rate and backend used to load audio files."""         audio_kwargs_from_user = processor_kwargs.get("audio_kwargs", {})-        sampling_rate = kwargs.get(-            "sampling_rate", processor_kwargs.get("sampling_rate", audio_kwargs_from_user.get("sampling_rate"))-        )++        sampling_rate = processor_kwargs.get("sampling_rate", audio_kwargs_from_user.get("sampling_rate"))         if sampling_rate is None:             if hasattr(self._audio_processor, "sampling_rate"):                 sampling_rate = self._audio_processor.sampling_rate             else:                 sampling_rate = 16_000 -        load_audio_backend = kwargs.get(-            "load_audio_backend",-            processor_kwargs.get("load_audio_backend", audio_kwargs_from_user.get("load_audio_backend")),-        )-        if load_audio_backend is None:+        backend = processor_kwargs.get("load_audio_backend", audio_kwargs_from_user.get("load_audio_backend"))+        if backend is None:             default_audio_kwargs = self.valid_processor_kwargs._defaults.get("audio_kwargs", {})-            load_audio_backend = default_audio_kwargs.get("load_audio_backend", "auto")+            backend = default_audio_kwargs.get("load_audio_backend", "auto")++        return {"sampling_rate": sampling_rate, "backend": backend} +    def _normalize_conversations(self, conversation) -> tuple[list, bool]:+        """Batch a single conversation if needed and convert OpenAI image_url blocks to HF image blocks."""         if isinstance(conversation, (list, tuple)) and (             isinstance(conversation[0], (list, tuple)) or hasattr(conversation[0], "content")         ):@@ -2128,7 +2151,7 @@ def apply_chat_template(         # Normalize OpenAI-style "image_url" content blocks to HuggingFace-style "image" blocks         # OpenAI format: {"type": "image_url", "image_url": {"url": "..."}}         # HuggingFace format: {"type": "image", "url": "..."}-        for conversation_idx, conversation in enumerate(conversations):+        for conversation in conversations:             for message in conversation:                 if not isinstance(message.get("content"), list):                     continue@@ -2142,58 +2165,94 @@ def apply_chat_template(                         new_content.append(content)                 message["content"] = new_content +        return conversations, is_batched++    def _adjust_assistant_masks_indices(self, processed_dict: dict[str, Any], generation_indices: list[list[int]]):+        assistant_masks = []+        offset_mapping = processed_dict.pop("offset_mapping")+        input_ids = processed_dict["input_ids"]+        # We do some corrections here to ensure the assistant masks aren't+        # misaligned when we expand up image tokens+        replacement_offsets = processed_dict.pop("text_replacement_offsets", None)+        if replacement_offsets is None or len(replacement_offsets) == 0:+            replacement_offsets = [[]] * len(input_ids)+        for i in range(len(input_ids)):+            current_mask = [0] * len(input_ids[i])+            placeholder_ends = [r["span"][1] for r in replacement_offsets[i]]+            chars_gained = [0] + [r["new_span"][1] - r["span"][1] for r in replacement_offsets[i]]+            for span in generation_indices[i]:+                # Shift the span past any placeholders that were expanded before it+                start_char, end_char = (+                    char + chars_gained[bisect.bisect_right(placeholder_ends, char)] for char in span+                )+                # Mask every token overlapping the span. Zero-width tokens (padding, added specials) never+                # match, and a span truncated away simply matches nothing+                for pos, (token_start, token_end) in enumerate(offset_mapping[i]):+                    if token_start < end_char and token_end > start_char:+                        current_mask[pos] = 1+            assistant_masks.append(current_mask)+        processed_dict["assistant_masks"] = assistant_masks+        return processed_dict++    def apply_chat_template(+        self,+        conversation: list[dict[str, str]] | list[list[dict[str, str]]],+        chat_template: str | None = None,+        tools: list[dict] | None = None,+        documents: list[dict[str, str]] | None = None,+        add_generation_prompt: bool = False,+        continue_final_message: bool | str = False,+        return_assistant_tokens_mask: bool = False,+        tokenize: bool = False,+        return_tensors: str | TensorType | None = None,+        return_dict: bool = False,+        load_audio_from_video: bool = False,+        processor_kwargs: dict | None = None,+        **kwargs,+    ) -> str:+        """+        Similar to the `apply_chat_template` method on tokenizers, this method applies a Jinja template to input+        conversations to turn them into a single tokenizable string.++        The input is expected to be in the following format, where each message content is a list consisting of text and+        optionally image or video inputs. One can also provide an image, video, URL or local path which will be used to form+        `pixel_values` when `return_dict=True`. If not provided, one will get only the formatted text, optionally tokenized text.++        conversation = [+            {+                "role": "user",+                "content": [+                    {"type": "image", "url": "https://www.ilankelman.org/stopsigns/australia.jpg"},+                    {"type": "text", "text": "Please describe this image in detail."},+                ],+            },+        ]++        Args:+            conversation (`Union[list[Dict, [str, str]], list[list[dict[str, str]]]]`):+                The conversation to format.+            chat_template (`Optional[str]`, *optional*):+                The Jinja template to use for formatting the conversation. If not provided, the tokenizer's+                chat template is used.+        """+        processor_kwargs = processor_kwargs or {}++        chat_template = self._resolve_chat_template(chat_template)+        processor_kwargs = self._prepare_processor_kwargs(+            chat_template,+            processor_kwargs,+            kwargs,+            add_generation_prompt=add_generation_prompt,+            continue_final_message=continue_final_message,+            return_assistant_tokens_mask=return_assistant_tokens_mask,+        )+        audio_loading_params = self._resolve_audio_loading_params(processor_kwargs)+        conversations, is_batched = self._normalize_conversations(conversation)+         if tokenize:-            batch_images, batch_videos = [], []-            batch_audios = []-            for conversation in conversations:-                images, videos = [], []-                for message in conversation:-                    content = message.get("content") or []-                    if isinstance(content, str):-                        continue-                    visuals = [-                        content_block for content_block in content if content_block["type"] in ["image", "video"]-                    ]-                    audio_fnames = [-                        content_block[key]-                        for content_block in content-                        for key in ["audio", "url", "path"]-                        if key in content_block and content_block["type"] == "audio"-                    ]-                    image_fnames = [-                        vision_info[key]-                        for vision_info in visuals-                        for key in ["image", "url", "path", "base64"]-                        if key in vision_info and vision_info["type"] == "image"-                    ]-                    images.extend(image_fnames)-                    video_fnames = [-                        vision_info[key]-                        for vision_info in visuals-                        for key in ["video", "url", "path"]-                        if key in vision_info and vision_info["type"] == "video"-                    ]-                    videos.extend(video_fnames)--                    # Audio models do not accept nested list of audios (yet!) so we construct a flat input audio list-                    if not load_audio_from_video:-                        for fname in audio_fnames:-                            batch_audios.append(-                                load_audio(fname, sampling_rate=sampling_rate, backend=load_audio_backend)-                            )-                    else:-                        for fname in video_fnames:-                            # This updates the template in-place and adds audio entry-                            # to ensure `audio` token is added by jinja-                            message["content"].append({"type": "audio"})-                            batch_audios.append(-                                load_audio(fname, sampling_rate=sampling_rate, backend=load_audio_backend)-                            )--                # Currently all processors can accept nested list of batches, but not flat list of visuals-                # So we'll make a batched list of images and let the processor handle it-                batch_images.append(images)-                batch_videos.append(videos)+            batch_images, batch_videos, batch_audios, additional_inputs = self._multimodal_inputs_from_conversations(+                conversations, audio_loading_params, load_audio_from_video+            )          # `kwargs` overwrite special tokens if both are present         template_kwargs = {**self.tokenizer.special_tokens_map, **kwargs}@@ -2212,34 +2271,37 @@ def apply_chat_template(             prompt = prompt[0]          if tokenize:-            # Tokenizer's `apply_chat_template` never adds special tokens when tokenizing-            # But processor's `apply_chat_template` didn't have an option to tokenize, so users had to format the prompt-            # and pass it to the processor. Users thus never worried about special tokens relying on processor handling-            # everything internally. The below line is to keep BC for that and be able to work with model that have-            # special tokens in the template (consistent with tokenizers). We dont want to raise warning, it will flood command line-            # without actionable solution for users+            # Tokenizer's `apply_chat_template` never adds special tokens. Processor's `apply_chat_template`+            # originally had no tokenize option, so users formatted the prompt themselves and passed it to the+            # processor, relying on it to handle everything internally (never worrying about special tokens).+            # This keeps BC for that flow and supports templates containing special tokens (consistent with+            # tokenizers). No warning is raised: it would flood the command line with nothing actionable for users.             single_prompt = prompt[0] if is_batched else prompt             if self.tokenizer.bos_token is not None and single_prompt.startswith(self.tokenizer.bos_token):                 processor_kwargs["add_special_tokens"] = False -            # Always sample frames by default unless explicitly set to `False` by users. If users do not pass `num_frames`/`fps`-            # sampling should not done for BC.+            # Always sample frames by default unless explicitly set to `False` by users. If users do not pass+            # `num_frames`/`fps` sampling should not done for BC.             if "do_sample_frames" not in processor_kwargs and (                 processor_kwargs.get("fps") is not None or processor_kwargs.get("num_frames") is not None             ):                 processor_kwargs["do_sample_frames"] = True -            # Set only is user passes a non-None value. Otherwise wa want to use each processor's own defaults+            # Set only if user passes a non-None value. Otherwise we want to use each processor's own defaults             if return_tensors:                 processor_kwargs["return_tensors"] = return_tensors              # Audio was loaded/resampled by us above, so let the audio processor know at which rate.             # (we additionally preserve the location of the kwarg in the nested structure kwargs -> processor -> audio)             if batch_audios:+                audio_kwargs_from_user = processor_kwargs.get("audio_kwargs", {})                 if "sampling_rate" in audio_kwargs_from_user:-                    processor_kwargs["audio_kwargs"] = {**audio_kwargs_from_user, "sampling_rate": sampling_rate}+                    processor_kwargs["audio_kwargs"] = {+                        **audio_kwargs_from_user,+                        "sampling_rate": audio_loading_params["sampling_rate"],+                    }                 else:-                    processor_kwargs["sampling_rate"] = sampling_rate+                    processor_kwargs["sampling_rate"] = audio_loading_params["sampling_rate"]              images_exist = any((im is not None) for im_list in batch_images for im in im_list)             videos_exist = any((vid is not None) for vid_list in batch_videos for vid in vid_list)@@ -2248,35 +2310,13 @@ def apply_chat_template(                 images=batch_images if images_exist else None,                 videos=batch_videos if videos_exist else None,                 audio=batch_audios or None,+                **additional_inputs,                 **processor_kwargs,             )              if return_dict:                 if return_assistant_tokens_mask:-                    assistant_masks = []-                    offset_mapping = out.pop("offset_mapping")-                    input_ids = out["input_ids"]-                    # We do some corrections here to ensure the assistant masks aren't-                    # misaligned when we expand up image tokens-                    replacement_offsets = out.pop("text_replacement_offsets", None)-                    if replacement_offsets is None or len(replacement_offsets) == 0:-                        replacement_offsets = [[]] * len(input_ids)-                    for i in range(len(input_ids)):-                        current_mask = [0] * len(input_ids[i])-                        placeholder_ends = [r["span"][1] for r in replacement_offsets[i]]-                        chars_gained = [0] + [r["new_span"][1] - r["span"][1] for r in replacement_offsets[i]]-                        for span in generation_indices[i]:-                            # Shift the span past any placeholders that were expanded before it-                            start_char, end_char = (-                                char + chars_gained[bisect.bisect_right(placeholder_ends, char)] for char in span-                            )-                            # Mask every token overlapping the span. Zero-width tokens (padding, added specials) never-                            # match, and a span truncated away simply matches nothing-                            for pos, (token_start, token_end) in enumerate(offset_mapping[i]):-                                if token_start < end_char and token_end > start_char:-                                    current_mask[pos] = 1-                        assistant_masks.append(current_mask)-                    out["assistant_masks"] = assistant_masks+                    out = self._adjust_assistant_masks_indices(out, generation_indices)                     out.convert_to_tensors(tensor_type=return_tensors)                 return out             else:@@ -2386,6 +2426,279 @@ def _check_special_mm_tokens(self, text: list[str], text_inputs: "BatchFeature",                     )  +class OmniModalProcessorMixin(ProcessorMixin):+    """+    This is a mixin used to provide additional functionality for omni-modal processor classes,+    including an API to load video and the corresponding audio track. New functions shared by omni+    models should be added here rather than in `ProcessorMixin` to keep simple processors apart from+    more involved models.+    Omni modal models are recommended to use this mixin from now on to get more functionality supported+    for cross-modal interactions.+    """++    @auto_docstring+    def __call__(+        self,+        images: ImageInput | None = None,+        text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None,+        videos: VideoInput | None = None,+        audio: AudioInput | None = None,+        audio_from_video_indices: list[int] | None = None,+        **kwargs: Unpack[ProcessingKwargs],+    ):+        r"""+        audio_from_video_indices (`list[int]`, *optional*):+            A list that describes the source of each audio slot where+                - `None` -> standalone audio associated with no video track+                - `i`    -> audio extracted from videos[i]+        """+        images, text, videos, audio = self.prepare_inputs_layout(+            images=images, text=text, videos=videos, audio=audio, **kwargs+        )+        self.validate_inputs(images=images, text=text, videos=videos, audio=audio, **kwargs)++        merged_kwargs = self._merge_kwargs(+            self.valid_processor_kwargs,+            tokenizer_init_kwargs=self.tokenizer.init_kwargs if hasattr(self, "tokenizer") else {},+            **kwargs,+        )++        # TODO @guarin: move it to processor level argument. Used by EG2 and qwen-omni series+        load_audio_from_video = merged_kwargs["videos_kwargs"].get("load_audio_from_video")+        if load_audio_from_video:+            audio, video_to_audio_indices = self._resolve_audio_from_video(+                videos,+                audio=audio,+                audio_from_video_indices=audio_from_video_indices,+            )++        processed_images = processed_videos = processed_audio = {}+        images_replacements = videos_replacements = audio_replacements = []+        if images is not None and hasattr(self, "image_processor"):+            processed_images, images_replacements = self._process_images(images, **merged_kwargs["images_kwargs"])+        if audio is not None and self._audio_processor is not None:+            processed_audio, audio_replacements = self._process_audio(audio, **merged_kwargs["audio_kwargs"])+            if load_audio_from_video:+                audio_replacements = [+                    sample for idx, sample in enumerate(audio_replacements) if idx not in video_to_audio_indices+                ]+        if videos is not None and hasattr(self, "video_processor"):+            if load_audio_from_video:+                additional_kwargs = {+                    "audio": audio,+                    "processed_audio": processed_audio,+                    "audio_kwargs": merged_kwargs["audio_kwargs"],+                    "video_to_audio_indices": video_to_audio_indices,+                }+                processed_videos, videos_replacements = self._process_videos(+                    videos, **merged_kwargs["videos_kwargs"], **additional_kwargs+                )+            else:+                processed_videos, videos_replacements = self._process_videos(videos, **merged_kwargs["videos_kwargs"])++        text_inputs = {}+        return_tensors = merged_kwargs["text_kwargs"].get("return_tensors", None)+        if getattr(self, "tokenizer", None) is not None and text is not None:+            return_mm_token_type_ids = merged_kwargs["text_kwargs"].pop("return_mm_token_type_ids", False)+            return_text_replacement_offsets = merged_kwargs["text_kwargs"].pop(+                "return_text_replacement_offsets", False+            )++            text, text_replacement_offsets = self.get_text_with_replacements(+                text,+                images_replacements,+                videos_replacements,+                audio_replacements,+            )+            text_inputs = self.tokenizer(text, **merged_kwargs["text_kwargs"])+            self._check_special_mm_tokens(text, text_inputs, modalities=["image", "video", "audio"])++            if return_text_replacement_offsets:+                text_inputs["text_replacement_offsets"] = text_replacement_offsets++            if return_mm_token_type_ids:+                text_inputs["mm_token_type_ids"] = self.create_mm_token_type_ids(text_inputs["input_ids"])++        # Pop unused keys from the inputs, e.g. inputs used only to compute number of image tokens+        data = {**text_inputs, **processed_images, **processed_videos, **processed_audio}+        data = {k: v for k, v in data.items() if k not in self.unused_input_names}++        if not kwargs.get("return_metadata"):+            data.pop("video_metadata", None)++        return BatchFeature(data, tensor_type=return_tensors, skip_tensor_conversion=self.skip_tensor_conversion)++    def _resolve_audio_from_video(+        self,+        videos: list,+        audio: list | None = None,+        audio_from_video_indices: list[int | None] | None = None,+    ) -> tuple[list, list[int | None]]:+        """+        Resolve audio inputs and their relationship to input videos+        when `load_audio_from_video=True` and model requires native interleaving.++        `audio_from_video_indices` describes the source of each audio slot:+            - `None` -> standalone audio associated with no video track+            - `i`    -> audio extracted from videos[i]++        Returns:+            `audio (list[str | np.ndarray | torch.Tensor])`:+                Resolved audio inputs, with audio-from-video appended where needed.+                The order of audio will be aligned correctly with the input text, as+                long as users pass valid `audio_from_video_indices` in ambiguous situations+            `video_to_audio_indices (list[int, int])`:+                video_to_audio_indices[i] gives the audio index associated with+                videos[i], or None if the video has no associated audio.+        """+        audio = list(audio or [])+        videos = make_batched_videos(videos)++        if audio_from_video_indices is None:+            if audio:+                raise ValueError(+                    "Mixing `audio` with `load_audio_from_video=True` requires `audio_from_video_indices`."+                )+            audio_from_video_indices = list(range(len(videos)))  # videos only: one audio per video++        if audio_from_video_indices.count(None) != len(audio):+            raise ValueError(+                f"`audio_from_video_indices` has {audio_from_video_indices.count(None)} standalone slots but {len(audio)} audio inputs."+            )++        if not is_torchcodec_greater_or_equal("0.17"):+            raise ValueError("Loading audio track from video file requires torchcodec>=0.17 installed!")++        standalone_audio = iter(audio)+        audio_resolved = []+        video_to_audio = [None] * len(videos)  # reverse mapping, `None` means no audio++        for video_idx in audio_from_video_indices:+            if video_idx is None:+                audio_resolved.append(next(standalone_audio))+                continue++            # Only `torchcodec` can decode audio track from all types of video file formats+            # Don't raise a load-error if video is silent and instead skip it+            cont_metadata = torchcodec.decoders.get_container_metadata(videos[video_idx])+            if not any(stream.media_type == "audio" for stream in cont_metadata.streams):+                continue++            loaded_audio = load_audio(+                videos[video_idx],+                backend="torchcodec",+                sampling_rate=self._audio_processor.sampling_rate,+            )+            video_to_audio[video_idx] = len(audio_resolved)+            audio_resolved.append(loaded_audio)++        # Empty list resolves to `None` since processors cannot handle empty inputs+        audio_resolved = audio_resolved if audio_resolved else None+        return audio_resolved, video_to_audio++    def _process_videos(+        self,+        videos: VideoInput,+        audio: AudioInput | None = None,+        processed_audio: dict | None = None,+        audio_kwargs: dict | None = None,+        video_to_audio_indices: list | None = None,+        **kwargs: Unpack[VideosKwargs],+    ):+        # pop so it is not passed down to video processor+        load_audio_from_video = kwargs.pop("load_audio_from_video", None)+        processed_videos = self.video_processor(videos, **kwargs)++        video_replacements = []+        videos = make_batched_videos(videos)+        for idx in range(len(videos)):+            replacement_text = self.replace_video_token(+                processed_videos,+                video_idx=idx,+                processed_audio=processed_audio,+                load_audio_from_video=load_audio_from_video,+                video_to_audio_indices=video_to_audio_indices,+                **kwargs,+            )+            video_replacements.append(replacement_text)+        return processed_videos, video_replacements++    def replace_video_token(+        self,+        video_inputs: dict,+        video_idx: int,+        audio: AudioInput | None = None,+        processed_audio: dict | None = None,+        audio_kwargs: dict | None = None,+        video_to_audio_indices: list | None = None,+        **kwargs: Unpack[VideosKwargs],+    ) -> str:+        raise NotImplementedError()++    # Opt-in method, only models that support interleaving video-audio can override it+    # Not used by default and should be called explicitly by overriding `_process_video`+    # since models might have diff conditions to apply video+audio interleaved tokens+    def replace_video_with_audio_token(+        self, video_inputs: dict, video_idx: int, audio_inputs: dict, audio_idx: int, **kwargs: VideosKwargs+    ) -> str:+        raise NotImplementedError()++    def _multimodal_inputs_from_conversations(+        self,+        conversations: list[list[dict[str, str]]],+        audio_loading_params: dict,+        load_audio_from_video: bool = False,+    ):+        """+        Traverse through conversation and extract all provided multimodal data,+        including URL/path or already decoded inputs. Images and video are returned+        in nested format while audio is flattened into a single list for all batches.+        If `load_audio_from_video=True`, then each video is linked to audio within+        a batch via `audio_from_video_indices`+        """+        batch_images, batch_videos = [], []+        batch_audios, batch_audio_from_video_indices = [], []+        video_idx = 0+        for conversation in conversations:+            images, videos = [], []+            for message in conversation:+                content = message.get("content") or []+                if isinstance(content, str):+                    continue++                image_fnames = [+                    content_block[key]+                    for content_block in content+                    for key in ["image", "url", "path", "base64"]+                    if key in content_block and content_block["type"] == "image"+                ]+                images.extend(image_fnames)++                # Keep track of audio inputs and whether they comes from video or not+                for content_block in content:+                    if content_block["type"] == "video":+                        video = next(v for k in ("video", "url", "path") if (v := content_block.get(k)) is not None)+                        videos.append(video)+                        if load_audio_from_video:+                            batch_audio_from_video_indices.append(video_idx)++                        video_idx += 1+                    elif content_block["type"] == "audio":+                        audio = next(v for k in ("audio", "url", "path") if (v := content_block.get(k)) is not None)+                        batch_audios.append(load_audio(audio, **audio_loading_params))+                        batch_audio_from_video_indices.append(None)++            # Currently all processors can accept nested list of batches, but not flat list of visuals+            # So we'll make a batched list of images and let the processor handle it+            batch_images.append(images)+            batch_videos.append(videos)+        additional_inputs = {+            "audio_from_video_indices": batch_audio_from_video_indices,+            "load_audio_from_video": load_audio_from_video,+        }+        return batch_images, batch_videos, batch_audios, additional_inputs++ ProcessorMixin.push_to_hub = copy_func(ProcessorMixin.push_to_hub) if ProcessorMixin.push_to_hub.__doc__ is not None:     ProcessorMixin.push_to_hub.__doc__ = ProcessorMixin.push_to_hub.__doc__.format(

@@ -179,6 +179,7 @@     is_torchao_available,     is_torchaudio_available,     is_torchcodec_available,+    is_torchcodec_greater_or_equal,     is_torchvision_available,     is_triton_available,     is_vision_available,@@ -715,6 +716,21 @@ def decorator(test_case):     return decorator  +def require_torchcodec_greater_or_equal(version: str):+    """+    Decorator marking a test that requires torchcodec version >= `version`.++    These tests are skipped when PyTorch version is less than `version`.+    """++    def decorator(test_case):+        return unittest.skipUnless(+            is_torchcodec_greater_or_equal(version), f"test requires PyTorch version >= {version}"+        )(test_case)++    return decorator++ def require_huggingface_hub_greater_or_equal(version: str):     """     Decorator marking a test that requires huggingface_hub version >= `version`.

@@ -31,6 +31,7 @@     CaptureLogger,     require_torch,     require_torch_accelerator,+    require_torchcodec_greater_or_equal,     slow,     torch_device, )@@ -496,6 +497,7 @@ class EmbeddingGemma2IntegrationTest(MemoryCleanupMixin, unittest.TestCase):     )     AUDIO_URL = "https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/song_1.mp3"     VIDEO_URL = "https://huggingface.co/datasets/hf-internal-testing/fixtures_videos/resolve/main/tennis.mp4"+    SILENT_VIDEO_URL = "https://huggingface.co/datasets/hf-internal-testing/fixtures_videos/resolve/main/karate.mp4"      @classmethod     def setUpClass(cls):@@ -507,6 +509,7 @@ def setUpClass(cls):         cls.image_2 = url_to_local_path(cls.IMAGE_2_URL)         cls.audio = url_to_local_path(cls.AUDIO_URL)         cls.video = url_to_local_path(cls.VIDEO_URL)+        cls.video_silent = url_to_local_path(cls.SILENT_VIDEO_URL)      @staticmethod     def _pool(model, inputs):@@ -705,6 +708,62 @@ def test_model_multiple_modalities(self):         expected = self._load_expected("multiple_modalities")         torch.testing.assert_close(embeddings.cpu(), expected["embeddings"], rtol=self.RTOL, atol=self.ATOL) +    @require_torchcodec_greater_or_equal("0.17.0")+    def test_model_video_and_audio(self):+        """Test that we can load the audio track from video file and interleave them frame by frame."""+        model = EmbeddingGemma2Model.from_pretrained(self.checkpoint_name, device_map=torch_device)++        conversations = [+            [+                {+                    "role": "user",+                    "content": [+                        {"type": "text", "text": "A man practicing karate."},+                        {"type": "video", "url": self.video_silent},+                    ],+                },+            ],+            [+                {+                    "role": "user",+                    "content": [+                        {"type": "text", "text": "A man playing tennis."},+                        {"type": "video", "url": self.video},+                    ],+                },+            ],+            [+                {+                    "role": "user",+                    "content": [+                        {"type": "text", "text": "A song"},+                        {"type": "audio", "url": self.audio},+                    ],+                },+            ],+        ]+        inputs = self.processor.apply_chat_template(+            conversations,+            tokenize=True,+            return_dict=True,+            return_tensors="pt",+            load_audio_from_video=True,+            processor_kwargs={+                "videos_kwargs": {+                    "overflow_strategy": "uniform",+                    "max_frames": 10,+                    "do_sample_frames": True,+                    "add_timestamps": True,+                },+                "audio_kwargs": {"truncation": False},+                "text_kwargs": {"padding": True},+            },+        ).to(model.device)+        embeddings = self._pool(model, inputs)++        expected = self._load_expected("video_and_audio")+        torch.testing.assert_close(embeddings.cpu(), expected["embeddings"], rtol=self.RTOL, atol=self.ATOL)+     def test_model_manual_placeholders(self):         """         Placeholders written in the text interleave the media with it, in the order the media items are given.

@@ -18,17 +18,20 @@ import numpy as np  from transformers import EmbeddingGemma2Processor-from transformers.testing_utils import get_tests_dir, require_torch, require_vision+from transformers.testing_utils import require_torch, require_torchcodec_greater_or_equal, require_vision+from transformers.utils import is_torch_available from transformers.video_utils import VideoMetadata -from ...test_processing_common import ProcessorTesterMixin+from ...test_processing_common import ProcessorTesterMixin, url_to_local_path  -SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece.model")+if is_torch_available():+    import torch   @require_vision class EmbeddingGemma2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):+    model_id = "google/embeddinggemma-2"     processor_class = EmbeddingGemma2Processor     videos_unstructured_max_length = 570     videos_text_kwargs_max_length = 570@@ -51,11 +54,6 @@ def _setup_video_processor(cls):         }         return video_processor_class(**video_processor_kwargs) -    @classmethod-    def _setup_feature_extractor(cls):-        feature_extractor_class = cls._get_component_class_from_processor("feature_extractor")-        return feature_extractor_class()-     @classmethod     def _setup_image_processor(cls):         image_processor_class = cls._get_component_class_from_processor("image_processor")@@ -66,63 +64,23 @@ def _setup_image_processor(cls):         }         return image_processor_class(**image_processor_kwargs) -    @classmethod-    def _setup_tokenizer(cls):-        tokenizer_class = cls._get_component_class_from_processor("tokenizer")-        extra_special_tokens = {-            "image_token": "<|image|>",-            "video_token": "<|video|>",-            "boi_token": "<start_of_image>",-            "eoi_token": "<end_of_image>",-            "audio_token": "<|audio|>",-            "boa_token": "<start_of_audio>",-            "eoa_token": "<end_of_audio>",-        }-        tokenizer = tokenizer_class.from_pretrained(-            SAMPLE_VOCAB, keep_accents=True, extra_special_tokens=extra_special_tokens-        )-        tokenizer.pad_token_id = tokenizer.eos_token_id-        return tokenizer-     @classmethod     def tearDownClass(cls):         shutil.rmtree(cls.tmpdirname, ignore_errors=True) -    # TODO: remove `_CHAT_TEMPLATE` and set `model_id` once the checkpoint is available on the Hub.-    _CHAT_TEMPLATE = (-        "{%- for msg in messages if msg.get('role') == 'system' -%}"-        "{%- if msg.get('content') is string -%}"-        "{{ msg['content'] }}"-        "{%- else -%}"-        "{%- for item in msg['content'] if item.get('type') == 'text' -%}"-        "{{ item['text'] }}"-        "{%- endfor -%}"-        "{%- endif -%}"-        "{%- endfor -%}"-        "{%- for msg in messages if msg.get('role') != 'system' -%}"-        "{%- if msg.get('content') is string -%}"-        "{{ msg['content'] }}"-        "{%- else -%}"-        "{%- set existing_text = msg['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | join -%}"-        "{%- set has_manual_placeholders = ('<|image|>' in existing_text) or ('<|video|>' in existing_text) or ('<|audio|>' in existing_text) -%}"-        "{%- for item in msg['content'] -%}"-        "{%- if item.get('type') == 'text' -%}"-        "{{ item['text'] }}"-        "{%- elif not has_manual_placeholders and item.get('type') == 'image' -%}"-        "<|image|>"-        "{%- elif not has_manual_placeholders and item.get('type') == 'video' -%}"-        "<|video|>"-        "{%- elif not has_manual_placeholders and item.get('type') == 'audio' -%}"-        "<|audio|>"-        "{%- endif -%}"-        "{%- endfor -%}"-        "{%- endif -%}"-        "{%- endfor -%}"-    )-     @staticmethod     def prepare_processor_dict():-        return {"image_seq_length": 3}+        return {"image_seq_length": 3, "audio_seq_length": 280}++    @property+    def video_sampling_expectations(self):+        return [+            {"max_frames": 3, "fps": None, "expected_dim": 0, "output_length": 3},+            {"overflow_strategy": None, "max_frames": None, "fps": 10, "expected_dim": 0, "output_length": 3},+            {"do_sample_frames": False, "fps": 10, "expected_dim": 0, "output_length": 11},+            {"do_sample_frames": False, "expected_dim": 0, "output_length": 11},+            {"expected_dim": 0, "output_length": 1},+        ]      # Override as EmbeddingGemma 2 needs images to be an explicitly nested batch     def prepare_images_inputs(self, batch_size: int | None = None):@@ -264,15 +222,15 @@ def test_multimodal_and_nested_inputs_without_text(self):         aud3 = np.zeros(1600, dtype=np.float32)          # 1. Nested audio list: batch size 2, sample 0 has 2 audios, sample 1 has 1 audio-        out_aud = processor(audio=[[aud1, aud2], [aud3]], return_tensors="pt")+        out_aud = processor(audio=[[aud1, aud2], [aud3]], padding=True, return_tensors="pt")         self.assertEqual(out_aud["input_ids"].shape[0], 2)         self.assertGreater(             (out_aud["input_ids"][0] == processor.audio_token_id).sum().item(),             (out_aud["input_ids"][1] == processor.audio_token_id).sum().item(),         )          # 2. Nested video list: batch size 2, sample 0 has 2 videos, sample 1 has 1 video-        out_vid = processor(videos=[[vid1, vid2], [vid3]], do_sample_frames=False, return_tensors="pt")+        out_vid = processor(videos=[[vid1, vid2], [vid3]], padding=True, do_sample_frames=False, return_tensors="pt")         self.assertEqual(out_vid["input_ids"].shape[0], 2)         self.assertGreater(             (out_vid["input_ids"][0] == processor.video_token_id).sum().item(),@@ -283,6 +241,7 @@ def test_multimodal_and_nested_inputs_without_text(self):         out_mixed = processor(             images=[[img1, img2], [img3]],             audio=[[aud1], [aud2, aud3]],+            padding=True,             return_tensors="pt",         )         self.assertEqual(out_mixed["input_ids"].shape[0], 2)@@ -293,18 +252,19 @@ def test_multimodal_and_nested_inputs_without_text(self):          # 4. Mismatched outer batch sizes across modalities with text=None raises ValueError         with self.assertRaisesRegex(ValueError, "inconsistently sized modality batches"):-            processor(images=[[img1], [img2]], audio=[aud1], return_tensors="pt")+            processor(images=[[img1], [img2]], audio=[aud1], padding=True, return_tensors="pt")          # 5. Nested audio and video lists also work when explicit text placeholders are provided         out_aud_with_text = processor(-            text=["<|audio|> <|audio|>", "<|audio|>"], audio=[[aud1, aud2], [aud3]], return_tensors="pt"+            text=["<|audio|> <|audio|>", "<|audio|>"], audio=[[aud1, aud2], [aud3]], padding=True, return_tensors="pt"         )         self.assertTrue((out_aud["input_ids"] == out_aud_with_text["input_ids"]).all())          out_vid_with_text = processor(             text=["<|video|> <|video|>", "<|video|>"],             videos=[[vid1, vid2], [vid3]],             do_sample_frames=False,+            padding=True,             return_tensors="pt",         )         self.assertTrue((out_vid["input_ids"] == out_vid_with_text["input_ids"]).all())@@ -374,7 +334,6 @@ def test_validate_inputs_multimodal_placeholder_counts(self):     def test_chat_template_ordering_and_manual_placeholders(self):         """System prompts precede media, content order is preserved, and manual markers disable auto-insertion."""         processor = self.get_processor()-        processor.chat_template = self._CHAT_TEMPLATE         img = np.random.randint(0, 256, size=(56, 56, 3), dtype=np.uint8)         aud = np.zeros(1600, dtype=np.float32) @@ -442,3 +401,292 @@ def test_chat_template_ordering_and_manual_placeholders(self):         self.assertNotIn("<|audio|>", rendered_partial)         with self.assertRaisesRegex(ValueError, "audio"):             processor(text=[rendered_partial], images=[[img]], audio=[aud], return_tensors="pt")++    @require_torchcodec_greater_or_equal("0.17.0")+    def test_chat_template_audio_from_video(self):+        processor = self.get_processor()+        video_file = url_to_local_path(+            "https://huggingface.co/datasets/hf-internal-testing/test-videos/resolve/main/sample_demo_1_320x240.mp4"+        )+        message_video = [+            {+                "role": "user",+                "content": [+                    {"type": "video", "path": video_file},+                    {"type": "text", "text": "Which of these animals is making the sound?"},+                ],+            },+            {+                "role": "assistant",+                "content": [{"type": "text", "text": "It is a cow."}],+            },+            {+                "role": "user",+                "content": [+                    {"type": "text", "text": "Tell me all about this animal."},+                ],+            },+        ]++        audio_file = url_to_local_path(+            "https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/f2641_0_throatclearing.wav"+        )+        message_audio = [+            {+                "role": "user",+                "content": [+                    {"type": "audio", "path": audio_file},+                    {"type": "text", "text": "Which can you hear?"},+                ],+            },+        ]++        silent_video_file = url_to_local_path(+            "https://huggingface.co/datasets/hf-internal-testing/test-videos/resolve/main/karate.mp4"+        )+        message_silent_video = [+            {+                "role": "user",+                "content": [+                    {"type": "video", "path": silent_video_file},+                    {"type": "text", "text": "Describe the man"},+                ],+            },+        ]++        out_dict = processor.apply_chat_template(+            [message_silent_video, message_audio, message_video],+            add_generation_prompt=True,+            tokenize=True,+            return_dict=True,+            return_tensors="pt",+            padding=True,+            truncation=False,+            load_audio_from_video=True,+            # video sampled to 3 frames+            overflow_strategy="uniform",+            max_frames=3,+            do_sample_frames=True,+            add_timestamps=True,+            load_audio_backend="torchcodec",+        )+        self.assertTrue(self.audio_input_name in out_dict)+        self.assertTrue(self.videos_input_name in out_dict)++        self.assertEqual(out_dict["input_ids"].shape[-1], 229)+        self.assertListEqual(list(out_dict[self.audio_input_name].shape[:2]), [2, 290])+        self.assertListEqual(list(out_dict[self.videos_input_name].shape[:2]), [4, 630])++        # Audio can be truncated shorter than video+        out_dict = processor.apply_chat_template(+            [message_silent_video, message_audio],+            add_generation_prompt=True,+            tokenize=True,+            return_dict=True,+            return_tensors="pt",+            padding=True,+            truncation=True,+            max_length=16_000,+            load_audio_from_video=True,+            # video sampled to 3 frames+            overflow_strategy="uniform",+            max_frames=3,+            do_sample_frames=True,+            add_timestamps=True,+            load_audio_backend="torchcodec",+        )++        self.assertEqual(out_dict["input_ids"].shape[-1], 229)+        self.assertListEqual(list(out_dict[self.audio_input_name].shape[:2]), [1, 99])+        self.assertListEqual(list(out_dict[self.videos_input_name].shape[:2]), [3, 630])++        with self.assertRaisesRegex(ValueError, "`audio_from_video_indices` has 2 standalone slots"):+            processor(+                videos=[video_file, silent_video_file],+                audio=[audio_file],+                text=f"{self.video_token}{self.audio_token}{self.video_token}",+                padding=True,+                audio_from_video_indices=[0, None, None],+                return_tensors="pt",+                load_audio_from_video=True,+                overflow_strategy="uniform",+                max_frames=3,+                do_sample_frames=True,+                add_timestamps=True,+                load_audio_backend="torchcodec",+            )++    def _test_apply_chat_template(+        self,+        modality: str,+        batch_size: int,+        return_tensors: str,+        input_name: str,+        processor_name: str,+        input_data: list[str],+    ):+        # override - EG2 rejects `num_frames` if passed to sample video, must use different args+        processor = self.get_processor()+        if processor.chat_template is None:+            self.skipTest("Processor has no chat template")++        if processor_name not in self.processor_class.get_attributes():+            self.skipTest(f"{processor_name} attribute not present in {self.processor_class}")++        # some models have only Fast image processor+        if getattr(processor, processor_name).__class__.__name__.endswith("Fast"):+            return_tensors = "pt"++        batch_messages = [+            [+                {"role": "system", "content": [{"type": "text", "text": "You are a helpful assistant."}]},+                {"role": "user", "content": [{"type": "text", "text": "Describe this."}]},+            ]+        ] * batch_size++        # Test that jinja can be applied+        formatted_prompt = processor.apply_chat_template(batch_messages, add_generation_prompt=True, tokenize=False)+        self.assertEqual(len(formatted_prompt), batch_size)++        # Test that tokenizing with template and directly with `self.tokenizer` gives same output+        formatted_prompt_tokenized = processor.apply_chat_template(+            batch_messages, add_generation_prompt=True, tokenize=True, return_tensors=return_tensors+        )+        add_special_tokens = True+        if processor.tokenizer.bos_token is not None and formatted_prompt[0].startswith(processor.tokenizer.bos_token):+            add_special_tokens = False+        tok_output = processor.tokenizer(+            formatted_prompt, return_tensors=return_tensors, add_special_tokens=add_special_tokens+        )+        expected_output = tok_output.input_ids+        self.assertListEqual(expected_output.tolist(), formatted_prompt_tokenized.tolist())++        # Test that kwargs passed to processor's `__call__` are actually used+        tokenized_prompt_100 = processor.apply_chat_template(+            batch_messages,+            add_generation_prompt=True,+            tokenize=True,+            return_tensors=return_tensors,+            processor_kwargs={+                "padding": "max_length",+                "truncation": True,+                "max_length": self.chat_template_max_length,+            },+        )+        self.assertEqual(len(tokenized_prompt_100[0]), self.chat_template_max_length)++        # Test that `return_dict=True` returns text related inputs in the dict+        out_dict_text = processor.apply_chat_template(+            batch_messages,+            add_generation_prompt=True,+            tokenize=True,+            return_dict=True,+            return_tensors=return_tensors,+        )+        self.assertTrue(all(key in out_dict_text for key in ["input_ids", "attention_mask"]))+        self.assertEqual(len(out_dict_text["input_ids"]), batch_size)+        self.assertEqual(len(out_dict_text["attention_mask"]), batch_size)++        # Test that with modality URLs and `return_dict=True`, we get modality inputs in the dict+        for idx, url in enumerate(input_data[:batch_size]):+            batch_messages[idx][1]["content"] = [batch_messages[idx][1]["content"][0], {"type": modality, "url": url}]++        out_dict = processor.apply_chat_template(+            batch_messages,+            add_generation_prompt=True,+            tokenize=True,+            return_dict=True,+            return_tensors=return_tensors,+            # No more than 2 frames and explicitly disable other ways to sample+            processor_kwargs={"overflow_strategy": "uniform", "max_frames": 2, "fps": None},+        )+        input_name = getattr(self, input_name)+        self.assertTrue(input_name in out_dict)+        self.assertEqual(len(out_dict["input_ids"]), batch_size)+        self.assertEqual(len(out_dict["attention_mask"]), batch_size)++        # Qwen-style pixels don't scale with bs same way as other models+        # calculate expected video token count based on video_grid_thw+        if (grid_thw := out_dict.get(f"{modality}_grid_thw")) is not None:+            mm_len = sum(thw[0] * thw[1] * thw[2] for thw in grid_thw)+        elif (num_frames_per_video := out_dict.get("num_frames_per_video")) is not None:+            # Models that concatenate the frames of every video along a single axis+            mm_len = sum(num_frames_per_video)+        else:+            mm_len = batch_size++        self.assertEqual(len(out_dict[input_name]), mm_len)++        return_tensor_to_type = {"pt": torch.Tensor, "np": np.ndarray, None: list}+        for k in out_dict:+            self.assertIsInstance(out_dict[k], return_tensor_to_type[return_tensors])++        # Test continue from final message+        assistant_message = {+            "role": "assistant",+            "content": [{"type": "text", "text": "It is the sound of"}],+        }+        for idx, url in enumerate(input_data[:batch_size]):+            batch_messages[idx] = batch_messages[idx] + [assistant_message]+        continue_prompt = processor.apply_chat_template(batch_messages, continue_final_message=True, tokenize=False)+        for prompt in continue_prompt:+            self.assertTrue(prompt.endswith("It is the sound of"))  # no `eos` token at the end++    @unittest.skip("Fails on runners only - i suppose it is some torchcodec version thingy, skip for now")+    def test_apply_chat_template_video_frame_sampling(self):+        # override - model doesn\t raise an error and doesn't+        # accept image frame URLs as one video+        processor = self.get_processor()+        messages = [+            [+                {+                    "role": "user",+                    "content": [+                        {+                            "type": "video",+                            "url": url_to_local_path(+                                "https://huggingface.co/datasets/hf-internal-testing/test-videos/resolve/main/tiny_video_320x240.mp4"+                            ),+                        },+                        {"type": "text", "text": "What is shown in this video?"},+                    ],+                },+            ]+        ]++        for processor_kwargs in self.video_sampling_expectations:+            exp_output_length = processor_kwargs.pop("output_length")+            expected_dim = processor_kwargs.pop("expected_dim")++            out_dict_with_video = processor.apply_chat_template(+                messages,+                add_generation_prompt=True,+                tokenize=True,+                return_dict=True,+                return_tensors="pt",+                processor_kwargs=processor_kwargs,+            )+            self.assertTrue(self.videos_input_name in out_dict_with_video)+            self.assertEqual(out_dict_with_video[self.videos_input_name].shape[expected_dim], exp_output_length)++        messages[0][0]["content"][0] = {+            "type": "video",+            "url": [+                url_to_local_path(+                    "https://huggingface.co/datasets/hf-internal-testing/fixtures_image_utils/resolve/main/australia.jpg"+                )+            ]+            * 2,+        }++        # Load video as a list of frames (i.e. images).+        # NOTE: each frame should have same size because we assume they come from one video+        out_dict_with_video = processor.apply_chat_template(+            messages,+            add_generation_prompt=True,+            tokenize=True,+            return_dict=True,+            return_tensors="pt",+            processor_kwargs={"do_sample_frames": False},+        )+        self.assertTrue(self.videos_input_name in out_dict_with_video)

@@ -23,10 +23,9 @@     Qwen2_5OmniProcessor, ) from transformers.testing_utils import (-    require_librosa,     require_torch,     require_torchaudio,-    require_torchcodec,+    require_torchcodec_greater_or_equal,     require_torchvision,     require_vision, )@@ -167,8 +166,7 @@ def test_post_process_multimodal_output_single_audio(self):         self.assertEqual(len(audio_outputs), 1)         self.assertTrue(np.array_equal(audio_outputs[0], np.arange(6, dtype=np.float32))) -    @require_librosa-    @require_torchcodec+    @require_torchcodec_greater_or_equal("0.17.0")     def test_chat_template_audio_from_video(self):         processor = self.get_processor()         if processor.chat_template is None:

@@ -21,10 +21,9 @@  from transformers import Qwen3OmniMoeProcessor from transformers.testing_utils import (-    require_librosa,     require_torch,     require_torchaudio,-    require_torchcodec,+    require_torchcodec_greater_or_equal,     require_torchvision,     require_vision, )@@ -164,8 +163,7 @@ def test_post_process_multimodal_output_single_audio(self):         self.assertEqual(len(audio_outputs), 1)         self.assertTrue(np.array_equal(audio_outputs[0], np.arange(6, dtype=np.float32))) -    @require_librosa-    @require_torchcodec+    @require_torchcodec_greater_or_equal("0.17.0")     def test_chat_template_audio_from_video(self):         processor = self.get_processor()         if processor.chat_template is None:

@@ -24,10 +24,8 @@ from unittest.mock import patch  import numpy as np-from huggingface_hub import hf_hub_download from parameterized import parameterized -from transformers import ProcessorMixin from transformers.processing_utils import MODALITY_TO_AUTOPROCESSOR_MAPPING from transformers.testing_utils import (     check_json_file_has_correct_format,@@ -1489,34 +1487,32 @@ def test_apply_chat_template_video_frame_sampling(self):                 processor_kwargs={"do_sample_frames": True},             ) -    @require_librosa     @require_torchcodec     def test_chat_template_audio_from_video(self):         processor = self.get_processor()         if processor.chat_template is None:             self.skipTest("Processor has no chat template") -        signature = inspect.signature(processor.__call__)-        if "videos" not in {*signature.parameters.keys()} or (-            signature.parameters.get("videos") is not None-            and signature.parameters["videos"].annotation == inspect._empty-        ):-            self.skipTest(f"{self.processor_class} does not support video inputs")+        if "video_processor" not in self.processor_class.get_attributes():+            self.skipTest(f"{self.processor_class.__name__} does not support video inputs")          if (             "feature_extractor" not in self.processor_class.get_attributes()-            or "audio_processor" not in self.processor_class.get_attributes()+            and "audio_processor" not in self.processor_class.get_attributes()         ):-            self.skipTest(f"feature_extractor attribute not present in {self.processor_class}")+            self.skipTest(f"{self.processor_class.__name__} does not support audio inputs") -        video_file_path = hf_hub_download(-            repo_id="hf-internal-testing/test-videos", filename="sample_demo_1_320x240.mp4", repo_type="dataset"+        video_file_path = url_to_local_path(+            "https://huggingface.co/datasets/hf-internal-testing/test-videos/resolve/main/sample_demo_1_320x240.mp4"         )-        messages = [+        video_expected_shape = processor.video_processor(videos=video_file_path)[self.videos_input_name].shape++        Video_audio_message = [             {                 "role": "user",                 "content": [                     {"type": "video", "path": video_file_path},+                    {"type": "audio"},                     {"type": "text", "text": "Which of these animals is making the sound?"},                 ],             },@@ -1532,25 +1528,52 @@ def test_chat_template_audio_from_video(self):             },         ] -        formatted_prompt = processor.apply_chat_template([messages], add_generation_prompt=True, tokenize=False)-        self.assertEqual(len(formatted_prompt), 1)  # batch size=1+        out_dict = processor.apply_chat_template(+            Video_audio_message,+            add_generation_prompt=True,+            tokenize=True,+            return_dict=True,+            return_tensors="pt",+            load_audio_backend="torchcodec",+            load_audio_from_video=True,+        )+        self.assertTrue(self.audio_input_name in out_dict)+        self.assertTrue(self.videos_input_name in out_dict)++        self.assertEqual(out_dict["input_ids"].shape[0], 1)  # batch-size=1+        self.assertEqual(out_dict["attention_mask"].shape[0], 1)  # batch-size=1+        self.assertEqual(out_dict[self.audio_input_name].shape[0], 1)  # 1 audio from video+        self.assertListEqual(list(out_dict[self.videos_input_name].shape), list(video_expected_shape)) +        audio_file_path = url_to_local_path(+            "https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/glass-breaking-151256.mp3"+        )+        audio_message = [+            {+                "role": "user",+                "content": [+                    {"type": "audio", "path": audio_file_path},+                    {"type": "text", "text": "What do you hear in this track?"},+                ],+            },+        ]         out_dict = processor.apply_chat_template(-            messages,+            [Video_audio_message, audio_message],             add_generation_prompt=True,             tokenize=True,             return_dict=True,             return_tensors="pt",+            padding=True,+            load_audio_backend="torchcodec",             load_audio_from_video=True,         )         self.assertTrue(self.audio_input_name in out_dict)         self.assertTrue(self.videos_input_name in out_dict) -        # should always have input_ids and attention_mask-        self.assertEqual(len(out_dict["input_ids"]), 1)  # batch-size=1-        self.assertEqual(len(out_dict["attention_mask"]), 1)  # batch-size=1-        self.assertEqual(len(out_dict[self.audio_input_name]), 1)  # 1 audio in the conversation-        self.assertEqual(len(out_dict[self.videos_input_name]), 1)  # 1 video in the conversation+        self.assertEqual(out_dict["input_ids"].shape[0], 2)+        self.assertEqual(out_dict["attention_mask"].shape[0], 2)+        self.assertEqual(out_dict[self.audio_input_name].shape[0], 2)  # 2 audios+        self.assertListEqual(list(out_dict[self.videos_input_name].shape), list(video_expected_shape))      def test_chat_template_jinja_kwargs(self):         """Tests that users can pass any kwargs and they will be used in jinja templates."""@@ -1802,8 +1825,9 @@ def does_processor_return_mm_offsets(processor_class, method_name: str):         Updated processor need to override a `replace_xxx_method` which is the only         reliable way to tell if processor is old format or not.         """+        parent_class = processor_class.__mro__[1]         child_method = getattr(processor_class, method_name)-        base_method = getattr(ProcessorMixin, method_name)+        base_method = getattr(parent_class, method_name)         return child_method is not base_method      def test_replacement_offsets(self):

@@ -19,7 +19,7 @@ import os import subprocess import tempfile-from collections import Counter, OrderedDict+from collections import Counter, OrderedDict, defaultdict from typing import Any  from sort_auto_mappings import sort_auto_mapping@@ -181,35 +181,42 @@ def get_all_config_mappings(): def get_all_processor_mappings(config_mapping: dict[str, str]):     files_to_names = {         "image_processing_pil": (-            "PilBackend",+            ("PilBackend",),             "IMAGE_PROCESSOR_MAPPING_NAMES_PIL",             MISSING_IMAGE_PROCESSOR_MAPPING_NAMES,         ),         "image_processing": (-            "TorchvisionBackend",+            ("TorchvisionBackend",),             "IMAGE_PROCESSOR_MAPPING_NAMES_TV",             MISSING_IMAGE_PROCESSOR_MAPPING_NAMES,         ),         "video_processing": (-            "BaseVideoProcessor",+            ("BaseVideoProcessor",),             "VIDEO_PROCESSOR_MAPPING_NAMES",             MISSING_VIDEO_PROCESSOR_MAPPING_NAMES,         ),         "feature_extraction": (-            "SequenceFeatureExtractor",+            ("SequenceFeatureExtractor",),             "FEATURE_EXTRACTOR_MAPPING_NAMES",             MISSING_FEATURE_EXTRACTOR_MAPPING_NAMES,         ),-        "processing": ("ProcessorMixin", "PROCESSOR_MAPPING_NAMES", MISSING_PROCESSOR_MAPPING_NAMES),+        "processing": (+            ("ProcessorMixin", "OmniModalProcessorMixin"),+            "PROCESSOR_MAPPING_NAMES",+            MISSING_PROCESSOR_MAPPING_NAMES,+        ),     } -    all_mappings = {}-    for processor_filename, (parent_class_name, mapping_name, missing_mapping_names) in files_to_names.items():-        all_mappings[mapping_name] = build_processor_mapping(-            config_mapping=config_mapping,-            processor_filename=processor_filename,-            parent_class_name=parent_class_name,-        )+    all_mappings = defaultdict(dict)+    for processor_filename, (parent_class_names, mapping_name, missing_mapping_names) in files_to_names.items():+        for parent_class_name in parent_class_names:+            all_mappings[mapping_name].update(+                build_processor_mapping(+                    config_mapping=config_mapping,+                    processor_filename=processor_filename,+                    parent_class_name=parent_class_name,+                )+            )          # Make sure users aren't duplicating the same keys manually         # Skip image processor until pil <-> tv backend are merged into one mapping

@@ -91,7 +91,7 @@     "https://huggingface.co/datasets/hf-internal-testing/fixtures_image_utils/resolve/main/two_dogs.jpg",     "https://huggingface.co/datasets/hf-internal-testing/fixtures_image_utils/resolve/main/australia.jpg",     "https://huggingface.co/datasets/hf-internal-testing/fixtures_videos/resolve/main/tiny_video.mp4",-    "https://huggingface.co/datasets/hf-internal-testing/fixtures_videos/resolve/main/tiny_video.mp4",+    "https://huggingface.co/datasets/hf-internal-testing/fixtures_videos/resolve/main/karate.mp4",     "https://huggingface.co/datasets/hf-internal-testing/fixtures_image_utils/resolve/main/pipeline-cat-chonk.jpeg",     # we should rely on this single dataset for our tests     "https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/bcn_weather.mp3",

@@ -947,8 +947,8 @@ def common_partial_suffix(str1: str, str2: str) -> str:             common_suffix = str1[-i] + common_suffix         else:             break-    # We do not allow full string suffix-    if common_suffix == str1 or common_suffix == str2:+    # We do not allow full string suffix except for `OmniModalProcessorMixin`+    if (common_suffix == str1 or common_suffix == str2) and "ProcessorMixin" not in common_suffix:         common_suffix = ""     return common_suffix