Add an opt-in per-frame pixel cap (cap_pixels_per_frame) to the Qwen3-VL video processor

@@ -47,13 +47,26 @@ class CohereCompassVideoProcessorInitKwargs(VideosKwargs, total=False):         The minimum number of frames to sample from video.     max_frames (`int`, *optional*, defaults to 768):         The maximum number of frames to sample from video.+    cap_pixels_per_frame (`bool`, *optional*):+        Whether to cap each frame's pixel cost the way the reference implementation (qwen-vl-utils) does:+        per-frame pixels are limited to `min(max_video_tokens * factor**2, size["longest_edge"] / num_frames)`+        and floored at `1.05 * size["shortest_edge"]`, so token cost scales with clip duration. Without the+        cap, videos that sample few frames spend the whole `size["longest_edge"]` budget on those frames and+        keep near-native per-frame resolution, so a short clip can cost almost as many tokens as a long+        video. If unset, the current uncapped behavior is kept and a warning is emitted: the default will+        change to `True` in v5.22, after which the argument will be removed.+    max_video_tokens (`int`, *optional*, defaults to 768):+        The per-frame token ceiling applied by `cap_pixels_per_frame`, in vision tokens per frame+        (qwen-vl-utils' `VIDEO_MAX_TOKEN_NUM`).     """      patch_size: int     temporal_patch_size: int     merge_size: int     min_frames: int     max_frames: int+    cap_pixels_per_frame: bool+    max_video_tokens: int   def smart_resize(@@ -96,26 +109,27 @@ def smart_resize( class CohereCompassVideoProcessor(BaseVideoProcessor):     resample = PILImageResampling.BICUBIC     size = {"shortest_edge": 128 * 32 * 32, "longest_edge": 32 * 32 * 768}-    max_image_size = {"longest_edge": 28 * 28 * 2 * 30000}     image_mean = IMAGENET_STANDARD_MEAN     image_std = IMAGENET_STANDARD_STD     do_resize = True     do_rescale = True     do_normalize = True     do_convert_rgb = True-    do_sample_frames = True     patch_size = 16     temporal_patch_size = 2-    max_duration = None     merge_size = 2+    min_frames = 4+    max_frames = 768+    do_sample_frames = True+    cap_pixels_per_frame = None+    max_video_tokens = 768     valid_kwargs = CohereCompassVideoProcessorInitKwargs+    model_input_names = ["pixel_values_videos", "video_grid_thw"]+    max_image_size = {"longest_edge": 28 * 28 * 2 * 30000}+    max_duration = None     num_frames = None     fps = 2 -    model_input_names = ["pixel_values_videos", "video_grid_thw"]-    min_frames = 4-    max_frames = 768-     def __init__(self, **kwargs: Unpack[CohereCompassVideoProcessorInitKwargs]):         super().__init__(**kwargs) @@ -175,21 +189,30 @@ def resize(         resample: "PILImageResampling | tvF.InterpolationMode | int | None",         factor: int,         temporal_factor: int,+        cap_pixels_per_frame: bool | None = None,         **kwargs,     ) -> "torch.Tensor":         """Resize dynamically based on input video aspect ratio."""         if not size.shortest_edge or not size.longest_edge:             raise ValueError(f"`size` dict must contain 'shortest_edge' and 'longest_edge' keys but got {size}.") +        num_frames = videos.shape[1]+        max_pixels = size.longest_edge+        if cap_pixels_per_frame:+            # per-frame pixels are capped at `max_video_tokens` patches or the budget's even share per frame+            frame_cap = self.max_video_tokens * factor * factor+            pixels_per_frame = max(min(frame_cap, size.longest_edge // num_frames), int(size.shortest_edge * 1.05))+            max_pixels = pixels_per_frame * num_frames+         height, width = videos.shape[-2:]         resized_height, resized_width = smart_resize(             height=height,             width=width,-            num_frames=videos.shape[1],+            num_frames=num_frames,             factor=factor,             temporal_factor=temporal_factor,             min_pixels=size.shortest_edge,-            max_pixels=size.longest_edge,+            max_pixels=max_pixels,         )         return super().resize(             image=videos,@@ -239,25 +262,36 @@ def patchify(      def _preprocess(         self,-        videos: list[torch.Tensor],-        do_convert_rgb: bool = True,-        do_resize: bool = True,-        size: SizeDict | None = None,-        resample: "PILImageResampling | tvF.InterpolationMode | int | None" = PILImageResampling.BICUBIC,-        do_rescale: bool = True,-        rescale_factor: float = 1 / 255.0,-        do_normalize: bool = True,-        image_mean: float | list[float] | None = None,-        image_std: float | list[float] | None = None,+        videos: list["torch.Tensor"],+        do_convert_rgb: bool,+        do_resize: bool,+        size: SizeDict,+        resample: "PILImageResampling | tvF.InterpolationMode | int | None",+        do_rescale: bool,+        rescale_factor: float,+        do_normalize: bool,+        image_mean: float | list[float] | None,+        image_std: float | list[float] | None,         patch_size: int | None = None,         temporal_patch_size: int | None = None,         merge_size: int | None = None,+        cap_pixels_per_frame: bool | None = None,         return_tensors: str | TensorType | None = None,         **kwargs,     ):+        if cap_pixels_per_frame is None:+            logger.warning_once(+                "CohereCompass video processing does not apply the per-frame pixel cap the reference "+                "implementation (qwen-vl-utils) applies, so some videos cost far more tokens than they "+                "would there. In v5.22 the capped behavior will become the default and "+                "`cap_pixels_per_frame` will be removed. Pass `cap_pixels_per_frame=True` to adopt the "+                "reference behavior now, or `False` to keep the current behavior and silence this "+                "warning."+            )+            cap_pixels_per_frame = False+        # Group videos by size for batched resizing         grouped_videos, grouped_videos_index = group_videos_by_shape(videos)         resized_videos_grouped = {}-         for shape, stacked_videos in grouped_videos.items():             if do_convert_rgb:                 stacked_videos = self.convert_to_rgb(stacked_videos)@@ -268,6 +302,7 @@ def _preprocess(                     resample=resample,                     factor=patch_size * merge_size,                     temporal_factor=temporal_patch_size,+                    cap_pixels_per_frame=cap_pixels_per_frame,                 )             resized_videos_grouped[shape] = stacked_videos         resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)@@ -296,12 +331,55 @@ def _preprocess(         processed_grids = reorder_videos(processed_grids, grouped_videos_index)         pixel_values_videos = torch.cat(processed_videos, dim=0)         video_grid_thw = torch.tensor(processed_grids)-        data = {-            "pixel_values_videos": pixel_values_videos,-            "video_grid_thw": video_grid_thw,-        } -        return BatchFeature(data=data, tensor_type=return_tensors)+        return BatchFeature(+            data={"pixel_values_videos": pixel_values_videos, "video_grid_thw": video_grid_thw},+            tensor_type=return_tensors,+        )++    def get_num_of_video_patches(self, num_frames: int, height: int, width: int, videos_kwargs=None):+        """+        A utility that returns number of video patches a given video size.++        Args:+            num_frames (`int`):+                Number of frames in the input video.+            height (`int`):+                Height of the input video.+            width (`int`):+                Width of the input video.+            videos_kwargs (`dict`, *optional*)+                Any kwargs to override defaults of the video processor.+        Returns:+            `Tuple(int, int)`: Number of placeholder tokens required and number of patches per image.+        """+        videos_kwargs = videos_kwargs if videos_kwargs is not None else {}+        min_pixels = videos_kwargs.get("min_pixels", None) or self.size["shortest_edge"]+        max_pixels = videos_kwargs.get("max_pixels", None) or self.size["longest_edge"]+        patch_size = videos_kwargs.get("patch_size", None) or self.patch_size+        merge_size = videos_kwargs.get("merge_size", None) or self.merge_size+        temporal_patch_size = videos_kwargs.get("temporal_patch_size", None) or self.temporal_patch_size+        cap_pixels_per_frame = videos_kwargs.get("cap_pixels_per_frame", None) or self.cap_pixels_per_frame++        factor = patch_size * merge_size+        if cap_pixels_per_frame:+            # Keep the count in sync with `resize` when the per-frame cap is active.+            frame_cap = self.max_video_tokens * factor * factor+            pixels_per_frame = max(min(frame_cap, max_pixels // num_frames), int(min_pixels * 1.05))+            max_pixels = pixels_per_frame * num_frames++        resized_height, resized_width = smart_resize(+            num_frames,+            height,+            width,+            temporal_factor=temporal_patch_size,+            factor=factor,+            min_pixels=min_pixels,+            max_pixels=max_pixels,+        )+        grid_h, grid_w = resized_height // patch_size, resized_width // patch_size+        grid_t = num_frames // temporal_patch_size+        return grid_t * grid_h * grid_w   __all__ = ["CohereCompassVideoProcessor"]

@@ -31,11 +31,14 @@     SizeDict, ) from ...processing_utils import Unpack, VideosKwargs-from ...utils import TensorType, auto_docstring+from ...utils import TensorType, auto_docstring, logging from ...video_processing_utils import BaseVideoProcessor from ...video_utils import VideoMetadata, group_videos_by_shape, reorder_videos  +logger = logging.get_logger(__name__)++ # Copied from transformers.models.qwen2_vl.image_processing_qwen2_vl.smart_resize def smart_resize(     height: int, width: int, factor: int = 28, min_pixels: int = 56 * 56, max_pixels: int = 14 * 14 * 4 * 1280@@ -84,6 +87,17 @@ class Qwen2VLVideoProcessorInitKwargs(VideosKwargs, total=False):         The maximum number of frames that can be sampled.     use_token_compression (`bool`, *optional*, defaults to `True`):         Whether to compress videos when processing or not.+    cap_pixels_per_frame (`bool`, *optional*):+        Whether to bound a video's total pixel cost the way the reference implementation+        (qwen-vl-utils) does: on top of the per-frame `size["longest_edge"]` cap, each frame is+        limited to an even share of the total-video pixel budget (`max_video_tokens` tokens'+        worth of pixels), floored at `1.05 * size["shortest_edge"]`, so densely sampled videos+        cannot grow without bound. If unset, the current behavior (no total bound) is kept and a+        warning is emitted: the default will change to `True` in v5.22, after which the argument+        will be removed.+    max_video_tokens (`int`, *optional*, defaults to 128000):+        The model context length assumed when deriving the total-video pixel budget used by+        `cap_pixels_per_frame` (the budget is 90% of this many tokens.     """      min_pixels: int@@ -93,6 +107,8 @@ class Qwen2VLVideoProcessorInitKwargs(VideosKwargs, total=False):     merge_size: int     min_frames: int     max_frames: int+    cap_pixels_per_frame: bool+    max_video_tokens: int   @auto_docstring@@ -111,6 +127,8 @@ class Qwen2VLVideoProcessor(BaseVideoProcessor):     min_frames = 4     max_frames = 768     do_sample_frames = False  # Set to False for BC, recommended to set `True` in new models+    cap_pixels_per_frame = None+    max_video_tokens = 128000     valid_kwargs = Qwen2VLVideoProcessorInitKwargs     model_input_names = ["pixel_values_videos", "video_grid_thw"] @@ -213,19 +231,30 @@ def resize(         size: SizeDict,         resample: "PILImageResampling | tvF.InterpolationMode | int | None",         factor: int,+        temporal_factor: int = 2,+        cap_pixels_per_frame: bool | None = None,         **kwargs,     ) -> "torch.Tensor":         """Resize dynamically based on input video aspect ratio."""         if not size.shortest_edge or not size.longest_edge:             raise ValueError(f"`size` dict must contain 'shortest_edge' and 'longest_edge' keys but got {size}.") +        max_pixels = size.longest_edge+        if cap_pixels_per_frame:+            # the per-frame cap (`size.longest_edge`) is bounded by an even share of the `max_video_tokens`+            num_frames = videos.shape[1]+            total_pixels = int(self.max_video_tokens * factor * factor * 0.9)+            max_pixels = max(+                min(max_pixels, total_pixels * temporal_factor // num_frames), int(size.shortest_edge * 1.05)+            )+         height, width = videos.shape[-2:]         resized_height, resized_width = smart_resize(             height,             width,             factor=factor,             min_pixels=size.shortest_edge,-            max_pixels=size.longest_edge,+            max_pixels=max_pixels,         )         return super().resize(             image=videos,@@ -288,9 +317,20 @@ def _preprocess(         patch_size: int | None = None,         temporal_patch_size: int | None = None,         merge_size: int | None = None,+        cap_pixels_per_frame: bool | None = None,         return_tensors: str | TensorType | None = None,         **kwargs,     ):+        if cap_pixels_per_frame is None:+            logger.warning_once(+                "Qwen2VL video processing does not apply the per-frame pixel cap the reference "+                "implementation (qwen-vl-utils) applies, so some videos cost far more tokens than they "+                "would there. In v5.22 the capped behavior will become the default and "+                "`cap_pixels_per_frame` will be removed. Pass `cap_pixels_per_frame=True` to adopt the "+                "reference behavior now, or `False` to keep the current behavior and silence this "+                "warning."+            )+            cap_pixels_per_frame = False         # Group videos by size for batched resizing         grouped_videos, grouped_videos_index = group_videos_by_shape(videos)         resized_videos_grouped = {}@@ -303,6 +343,8 @@ def _preprocess(                     size=size,                     resample=resample,                     factor=patch_size * merge_size,+                    temporal_factor=temporal_patch_size,+                    cap_pixels_per_frame=cap_pixels_per_frame,                 )             resized_videos_grouped[shape] = stacked_videos         resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)

@@ -27,28 +27,29 @@ from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache from ...configuration_utils import PreTrainedConfig-from ...image_utils import IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD+from ...image_utils import IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, PILImageResampling, SizeDict from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling from ...modeling_rope_utils import RopeParameters, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import ProcessingKwargs, Unpack, VideosKwargs-from ...utils import auto_docstring, can_return_tuple, logging+from ...utils import auto_docstring, can_return_tuple, is_torchvision_available, logging from ...utils.deprecation import deprecate_kwarg from ...utils.generic import (     maybe_autocast,     merge_with_config_defaults, ) from ...utils.output_capturing import capture_outputs+from ...video_processing_utils import BaseVideoProcessor from ...video_utils import VideoMetadata from ...vision_utils import (     get_vision_attention_seqlens,     get_vision_interpolation_indices_and_weights,     get_vision_position_ids, ) from ..auto.modeling_auto import AutoModel-from ..glm4v.video_processing_glm4v import Glm4vVideoProcessor+from ..glm4v.video_processing_glm4v import smart_resize from ..llama.modeling_llama import LlamaRotaryEmbedding from ..qwen2_5_vl.modeling_qwen2_5_vl import (     Qwen2_5_VLCausalLMOutputWithPast,@@ -65,6 +66,7 @@     VisionRotaryEmbedding, ) from ..qwen2_vl.processing_qwen2_vl import Qwen2VLProcessor+from ..qwen2_vl.video_processing_qwen2_vl import Qwen2VLVideoProcessor from ..qwen3.modeling_qwen3 import (     Qwen3Attention,     Qwen3DecoderLayer,@@ -74,6 +76,10 @@ )  +if is_torchvision_available():+    from torchvision.transforms.v2 import functional as tvF++ logger = logging.get_logger(__name__)  @@ -1111,24 +1117,131 @@ class Qwen3VLVideoProcessorInitKwargs(VideosKwargs, total=False):         The minimum number of frames to sample from video.     max_frames (`int`, *optional*, defaults to 768):         The maximum number of frames to sample from video.+    cap_pixels_per_frame (`bool`, *optional*):+        Whether to cap each frame's pixel cost the way the reference implementation (qwen-vl-utils) does:+        per-frame pixels are limited to `min(max_video_tokens * factor**2, size["longest_edge"] / num_frames)`+        and floored at `1.05 * size["shortest_edge"]`, so token cost scales with clip duration. Without the+        cap, videos that sample few frames spend the whole `size["longest_edge"]` budget on those frames and+        keep near-native per-frame resolution, so a short clip can cost almost as many tokens as a long+        video. If unset, the current uncapped behavior is kept and a warning is emitted: the default will+        change to `True` in v5.22, after which the argument will be removed.+    max_video_tokens (`int`, *optional*, defaults to 768):+        The per-frame token ceiling applied by `cap_pixels_per_frame`, in vision tokens per frame+        (qwen-vl-utils' `VIDEO_MAX_TOKEN_NUM`).     """      patch_size: int     temporal_patch_size: int     merge_size: int     min_frames: int     max_frames: int+    cap_pixels_per_frame: bool+    max_video_tokens: int  -class Qwen3VLVideoProcessor(Glm4vVideoProcessor):+class Qwen3VLVideoProcessor(Qwen2VLVideoProcessor):     size = {"shortest_edge": 128 * 32 * 32, "longest_edge": 32 * 32 * 768}+    max_image_size = {"longest_edge": 28 * 28 * 2 * 30000}     image_mean = IMAGENET_STANDARD_MEAN     image_std = IMAGENET_STANDARD_STD+    do_sample_frames = True     patch_size = 16     min_frames = 4     max_frames = 768     max_duration = None     num_frames = None+    fps = 2+    cap_pixels_per_frame = None+    max_video_tokens = 768++    def __init__(self, **kwargs: Unpack[Qwen3VLVideoProcessorInitKwargs]):+        BaseVideoProcessor.__init__(self, **kwargs)++    def _standardize_kwargs(self, **super_kwargs):+        raise NotImplementedError("No need to override, fallback to base class implementation")++    def resize(+        self,+        videos: "torch.Tensor",+        size: SizeDict,+        resample: "PILImageResampling | tvF.InterpolationMode | int | None",+        factor: int,+        temporal_factor: int,+        cap_pixels_per_frame: bool | None = None,+        **kwargs,+    ) -> "torch.Tensor":+        """Resize dynamically based on input video aspect ratio."""+        if not size.shortest_edge or not size.longest_edge:+            raise ValueError(f"`size` dict must contain 'shortest_edge' and 'longest_edge' keys but got {size}.")++        num_frames = videos.shape[1]+        max_pixels = size.longest_edge+        if cap_pixels_per_frame:+            # per-frame pixels are capped at `max_video_tokens` patches or the budget's even share per frame+            frame_cap = self.max_video_tokens * factor * factor+            pixels_per_frame = max(min(frame_cap, size.longest_edge // num_frames), int(size.shortest_edge * 1.05))+            max_pixels = pixels_per_frame * num_frames++        height, width = videos.shape[-2:]+        resized_height, resized_width = smart_resize(+            height=height,+            width=width,+            num_frames=num_frames,+            factor=factor,+            temporal_factor=temporal_factor,+            min_pixels=size.shortest_edge,+            max_pixels=max_pixels,+        )+        return BaseVideoProcessor.resize(+            self,+            image=videos,+            size=SizeDict(height=resized_height, width=resized_width),+            resample=resample,+        )++    def get_num_of_video_patches(self, num_frames: int, height: int, width: int, videos_kwargs=None):+        """+        A utility that returns number of video patches a given video size.++        Args:+            num_frames (`int`):+                Number of frames in the input video.+            height (`int`):+                Height of the input video.+            width (`int`):+                Width of the input video.+            videos_kwargs (`dict`, *optional*)+                Any kwargs to override defaults of the video processor.+        Returns:+            `Tuple(int, int)`: Number of placeholder tokens required and number of patches per image.+        """+        videos_kwargs = videos_kwargs if videos_kwargs is not None else {}+        min_pixels = videos_kwargs.get("min_pixels", None) or self.size["shortest_edge"]+        max_pixels = videos_kwargs.get("max_pixels", None) or self.size["longest_edge"]+        patch_size = videos_kwargs.get("patch_size", None) or self.patch_size+        merge_size = videos_kwargs.get("merge_size", None) or self.merge_size+        temporal_patch_size = videos_kwargs.get("temporal_patch_size", None) or self.temporal_patch_size+        cap_pixels_per_frame = videos_kwargs.get("cap_pixels_per_frame", None) or self.cap_pixels_per_frame++        factor = patch_size * merge_size+        if cap_pixels_per_frame:+            # Keep the count in sync with `resize` when the per-frame cap is active.+            frame_cap = self.max_video_tokens * factor * factor+            pixels_per_frame = max(min(frame_cap, max_pixels // num_frames), int(min_pixels * 1.05))+            max_pixels = pixels_per_frame * num_frames++        resized_height, resized_width = smart_resize(+            num_frames,+            height,+            width,+            temporal_factor=temporal_patch_size,+            factor=factor,+            min_pixels=min_pixels,+            max_pixels=max_pixels,+        )+        grid_h, grid_w = resized_height // patch_size, resized_width // patch_size+        grid_t = num_frames // temporal_patch_size+        return grid_t * grid_h * grid_w      def sample_frames(         self,

@@ -50,13 +50,26 @@ class Qwen3VLVideoProcessorInitKwargs(VideosKwargs, total=False):         The minimum number of frames to sample from video.     max_frames (`int`, *optional*, defaults to 768):         The maximum number of frames to sample from video.+    cap_pixels_per_frame (`bool`, *optional*):+        Whether to cap each frame's pixel cost the way the reference implementation (qwen-vl-utils) does:+        per-frame pixels are limited to `min(max_video_tokens * factor**2, size["longest_edge"] / num_frames)`+        and floored at `1.05 * size["shortest_edge"]`, so token cost scales with clip duration. Without the+        cap, videos that sample few frames spend the whole `size["longest_edge"]` budget on those frames and+        keep near-native per-frame resolution, so a short clip can cost almost as many tokens as a long+        video. If unset, the current uncapped behavior is kept and a warning is emitted: the default will+        change to `True` in v5.22, after which the argument will be removed.+    max_video_tokens (`int`, *optional*, defaults to 768):+        The per-frame token ceiling applied by `cap_pixels_per_frame`, in vision tokens per frame+        (qwen-vl-utils' `VIDEO_MAX_TOKEN_NUM`).     """      patch_size: int     temporal_patch_size: int     merge_size: int     min_frames: int     max_frames: int+    cap_pixels_per_frame: bool+    max_video_tokens: int   def smart_resize(@@ -99,26 +112,27 @@ def smart_resize( class Qwen3VLVideoProcessor(BaseVideoProcessor):     resample = PILImageResampling.BICUBIC     size = {"shortest_edge": 128 * 32 * 32, "longest_edge": 32 * 32 * 768}-    max_image_size = {"longest_edge": 28 * 28 * 2 * 30000}     image_mean = IMAGENET_STANDARD_MEAN     image_std = IMAGENET_STANDARD_STD     do_resize = True     do_rescale = True     do_normalize = True     do_convert_rgb = True-    do_sample_frames = True     patch_size = 16     temporal_patch_size = 2-    max_duration = None     merge_size = 2+    min_frames = 4+    max_frames = 768+    do_sample_frames = True+    cap_pixels_per_frame = None+    max_video_tokens = 768     valid_kwargs = Qwen3VLVideoProcessorInitKwargs+    model_input_names = ["pixel_values_videos", "video_grid_thw"]+    max_image_size = {"longest_edge": 28 * 28 * 2 * 30000}+    max_duration = None     num_frames = None     fps = 2 -    model_input_names = ["pixel_values_videos", "video_grid_thw"]-    min_frames = 4-    max_frames = 768-     def __init__(self, **kwargs: Unpack[Qwen3VLVideoProcessorInitKwargs]):         super().__init__(**kwargs) @@ -178,21 +192,30 @@ def resize(         resample: "PILImageResampling | tvF.InterpolationMode | int | None",         factor: int,         temporal_factor: int,+        cap_pixels_per_frame: bool | None = None,         **kwargs,     ) -> "torch.Tensor":         """Resize dynamically based on input video aspect ratio."""         if not size.shortest_edge or not size.longest_edge:             raise ValueError(f"`size` dict must contain 'shortest_edge' and 'longest_edge' keys but got {size}.") +        num_frames = videos.shape[1]+        max_pixels = size.longest_edge+        if cap_pixels_per_frame:+            # per-frame pixels are capped at `max_video_tokens` patches or the budget's even share per frame+            frame_cap = self.max_video_tokens * factor * factor+            pixels_per_frame = max(min(frame_cap, size.longest_edge // num_frames), int(size.shortest_edge * 1.05))+            max_pixels = pixels_per_frame * num_frames+         height, width = videos.shape[-2:]         resized_height, resized_width = smart_resize(             height=height,             width=width,-            num_frames=videos.shape[1],+            num_frames=num_frames,             factor=factor,             temporal_factor=temporal_factor,             min_pixels=size.shortest_edge,-            max_pixels=size.longest_edge,+            max_pixels=max_pixels,         )         return super().resize(             image=videos,@@ -242,25 +265,36 @@ def patchify(      def _preprocess(         self,-        videos: list[torch.Tensor],-        do_convert_rgb: bool = True,-        do_resize: bool = True,-        size: SizeDict | None = None,-        resample: "PILImageResampling | tvF.InterpolationMode | int | None" = PILImageResampling.BICUBIC,-        do_rescale: bool = True,-        rescale_factor: float = 1 / 255.0,-        do_normalize: bool = True,-        image_mean: float | list[float] | None = None,-        image_std: float | list[float] | None = None,+        videos: list["torch.Tensor"],+        do_convert_rgb: bool,+        do_resize: bool,+        size: SizeDict,+        resample: "PILImageResampling | tvF.InterpolationMode | int | None",+        do_rescale: bool,+        rescale_factor: float,+        do_normalize: bool,+        image_mean: float | list[float] | None,+        image_std: float | list[float] | None,         patch_size: int | None = None,         temporal_patch_size: int | None = None,         merge_size: int | None = None,+        cap_pixels_per_frame: bool | None = None,         return_tensors: str | TensorType | None = None,         **kwargs,     ):+        if cap_pixels_per_frame is None:+            logger.warning_once(+                "Qwen3VL video processing does not apply the per-frame pixel cap the reference "+                "implementation (qwen-vl-utils) applies, so some videos cost far more tokens than they "+                "would there. In v5.22 the capped behavior will become the default and "+                "`cap_pixels_per_frame` will be removed. Pass `cap_pixels_per_frame=True` to adopt the "+                "reference behavior now, or `False` to keep the current behavior and silence this "+                "warning."+            )+            cap_pixels_per_frame = False+        # Group videos by size for batched resizing         grouped_videos, grouped_videos_index = group_videos_by_shape(videos)         resized_videos_grouped = {}-         for shape, stacked_videos in grouped_videos.items():             if do_convert_rgb:                 stacked_videos = self.convert_to_rgb(stacked_videos)@@ -271,6 +305,7 @@ def _preprocess(                     resample=resample,                     factor=patch_size * merge_size,                     temporal_factor=temporal_patch_size,+                    cap_pixels_per_frame=cap_pixels_per_frame,                 )             resized_videos_grouped[shape] = stacked_videos         resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)@@ -299,12 +334,55 @@ def _preprocess(         processed_grids = reorder_videos(processed_grids, grouped_videos_index)         pixel_values_videos = torch.cat(processed_videos, dim=0)         video_grid_thw = torch.tensor(processed_grids)-        data = {-            "pixel_values_videos": pixel_values_videos,-            "video_grid_thw": video_grid_thw,-        } -        return BatchFeature(data=data, tensor_type=return_tensors)+        return BatchFeature(+            data={"pixel_values_videos": pixel_values_videos, "video_grid_thw": video_grid_thw},+            tensor_type=return_tensors,+        )++    def get_num_of_video_patches(self, num_frames: int, height: int, width: int, videos_kwargs=None):+        """+        A utility that returns number of video patches a given video size.++        Args:+            num_frames (`int`):+                Number of frames in the input video.+            height (`int`):+                Height of the input video.+            width (`int`):+                Width of the input video.+            videos_kwargs (`dict`, *optional*)+                Any kwargs to override defaults of the video processor.+        Returns:+            `Tuple(int, int)`: Number of placeholder tokens required and number of patches per image.+        """+        videos_kwargs = videos_kwargs if videos_kwargs is not None else {}+        min_pixels = videos_kwargs.get("min_pixels", None) or self.size["shortest_edge"]+        max_pixels = videos_kwargs.get("max_pixels", None) or self.size["longest_edge"]+        patch_size = videos_kwargs.get("patch_size", None) or self.patch_size+        merge_size = videos_kwargs.get("merge_size", None) or self.merge_size+        temporal_patch_size = videos_kwargs.get("temporal_patch_size", None) or self.temporal_patch_size+        cap_pixels_per_frame = videos_kwargs.get("cap_pixels_per_frame", None) or self.cap_pixels_per_frame++        factor = patch_size * merge_size+        if cap_pixels_per_frame:+            # Keep the count in sync with `resize` when the per-frame cap is active.+            frame_cap = self.max_video_tokens * factor * factor+            pixels_per_frame = max(min(frame_cap, max_pixels // num_frames), int(min_pixels * 1.05))+            max_pixels = pixels_per_frame * num_frames++        resized_height, resized_width = smart_resize(+            num_frames,+            height,+            width,+            temporal_factor=temporal_patch_size,+            factor=factor,+            min_pixels=min_pixels,+            max_pixels=max_pixels,+        )+        grid_h, grid_w = resized_height // patch_size, resized_width // patch_size+        grid_t = num_frames // temporal_patch_size+        return grid_t * grid_h * grid_w   __all__ = ["Qwen3VLVideoProcessor"]

@@ -35,7 +35,7 @@ ) from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel-from ...processing_utils import ProcessorMixin, Unpack+from ...processing_utils import ProcessorMixin, Unpack, VideosKwargs from ...utils import TensorType, auto_docstring, can_return_tuple, logging from ...utils.generic import (     get_max_seqlen,@@ -65,10 +65,7 @@ from ..qwen2_vl.processing_qwen2_vl import (     Qwen2VLProcessorKwargs, )-from ..qwen2_vl.video_processing_qwen2_vl import (-    Qwen2VLVideoProcessor,-    Qwen2VLVideoProcessorInitKwargs,-)+from ..qwen2_vl.video_processing_qwen2_vl import Qwen2VLVideoProcessor from ..qwen3_vl.processing_qwen3_vl import Qwen3VLProcessor from ..siglip.configuration_siglip import SiglipVisionConfig from ..siglip.modeling_siglip import (@@ -1129,7 +1126,7 @@ def _preprocess(         )  -class VideoLlama3VideoProcessorInitKwargs(Qwen2VLVideoProcessorInitKwargs):+class VideoLlama3VideoProcessorInitKwargs(VideosKwargs, total=False):     r"""     min_pixels (`int`, *optional*, defaults to `56 * 56`):         The min pixels of the image to resize the image.@@ -1149,6 +1146,13 @@ class VideoLlama3VideoProcessorInitKwargs(Qwen2VLVideoProcessorInitKwargs):         Whether to compress videos when processing or not.     """ +    min_pixels: int+    max_pixels: int+    patch_size: int+    temporal_patch_size: int+    merge_size: int+    min_frames: int+    max_frames: int     use_token_compression: bool | None  @@ -1159,6 +1163,10 @@ class VideoLlama3VideoProcessor(Qwen2VLVideoProcessor):     temporal_patch_size = 1     max_frames = 180     return_metadata = True+    # This processor's resize already spreads `size.longest_edge` across the sampled frames, so+    # Qwen2VL's opt-in cap does not apply here.+    cap_pixels_per_frame = AttributeError()+    video_total_seq_len = AttributeError()     valid_kwargs = VideoLlama3VideoProcessorInitKwargs     model_input_names = ["pixel_values_videos", "video_grid_thw", "video_merge_sizes", "video_compression_mask"] 

@@ -58,7 +58,6 @@ class VideoLlama3VideoProcessorInitKwargs(VideosKwargs, total=False):     merge_size: int     min_frames: int     max_frames: int-     use_token_compression: bool | None  

@@ -393,3 +393,91 @@ def test_bc_min_max_pixels(self):             processed = video_processing_loaded(video_inputs, return_tensors="pt")             expected_output_video_shape = [320, 1176]             self.assertListEqual(list(processed.pixel_values_videos.shape), expected_output_video_shape)++    def _process_frames(self, video_processing_class, num_frames, size, frame_size=256, **processor_kwargs):+        video_processor_dict = self.video_processor_dict.copy()+        video_processor_dict.pop("min_pixels", None)+        video_processor_dict.pop("max_pixels", None)+        video_processor_dict["size"] = size+        video_processor_dict["do_sample_frames"] = False+        video_processor_dict.update(processor_kwargs)+        video_processing = video_processing_class(**video_processor_dict)+        video = [np.random.randint(0, 256, (frame_size, frame_size, 3), dtype=np.uint8) for _ in range(num_frames)]+        return video_processing(video, return_tensors="pt")[self.input_name]++    def _expected_capped_seq_len(self, num_frames, frame_size, size, total_seq_len):+        factor = 28+        total_pixels = int(total_seq_len * factor * factor * 0.9)+        max_pixels = max(min(size["longest_edge"], total_pixels * 2 // num_frames), int(size["shortest_edge"] * 1.05))+        expected_height, expected_width = smart_resize(+            frame_size,+            frame_size,+            factor=factor,+            min_pixels=size["shortest_edge"],+            max_pixels=max_pixels,+        )+        return (num_frames // 2) * (expected_height // 14) * (expected_width // 14)++    def test_cap_pixels_per_frame_bounds_dense_videos(self):+        # With the total budget binding, each frame is held to the budget's even share instead of+        # the per-frame `longest_edge` cap. The budget is shrunk via `max_video_tokens` so the+        # test does not need hundreds of frames to make the bound bind.+        size = {"longest_edge": 768 * 28 * 28 * 100, "shortest_edge": 400}+        for video_processing_class in self.video_processor_list:+            uncapped = self._process_frames(video_processing_class, 8, size)+            capped = self._process_frames(+                video_processing_class, 8, size, cap_pixels_per_frame=True, max_video_tokens=128+            )+            self.assertEqual(capped.shape[0], self._expected_capped_seq_len(8, 256, size, total_seq_len=128))+            self.assertLess(capped.shape[0], uncapped.shape[0])++    def test_cap_pixels_per_frame_floors_at_min_pixels(self):+        # When the budget's share drops below `1.05 * shortest_edge`, frames keep a usable+        # resolution instead of collapsing further.+        size = {"longest_edge": 768 * 28 * 28 * 100, "shortest_edge": 40000}+        for video_processing_class in self.video_processor_list:+            capped = self._process_frames(+                video_processing_class, 8, size, cap_pixels_per_frame=True, max_video_tokens=128+            )+            self.assertEqual(capped.shape[0], self._expected_capped_seq_len(8, 256, size, total_seq_len=128))++    def test_cap_pixels_per_frame_noop_when_not_binding(self):+        # At the real budget a short clip never hits the total bound: capped and uncapped agree.+        size = {"longest_edge": 768 * 28 * 28 * 100, "shortest_edge": 400}+        for video_processing_class in self.video_processor_list:+            uncapped = self._process_frames(video_processing_class, 8, size)+            capped = self._process_frames(video_processing_class, 8, size, cap_pixels_per_frame=True)+            self.assertEqual(list(capped.shape), list(uncapped.shape))++    def test_cap_pixels_per_frame_call_time_override(self):+        size = {"longest_edge": 768 * 28 * 28 * 100, "shortest_edge": 400}+        for video_processing_class in self.video_processor_list:+            video_processor_dict = self.video_processor_dict.copy()+            video_processor_dict.pop("min_pixels", None)+            video_processor_dict.pop("max_pixels", None)+            video_processor_dict["size"] = size+            video_processor_dict["do_sample_frames"] = False+            video_processor_dict["max_video_tokens"] = 128+            video_processing = video_processing_class(**video_processor_dict)+            video = [np.random.randint(0, 256, (256, 256, 3), dtype=np.uint8) for _ in range(8)]++            default_out = video_processing(video, return_tensors="pt")[self.input_name]+            capped_out = video_processing(video, return_tensors="pt", cap_pixels_per_frame=True)[self.input_name]++            self.assertEqual(capped_out.shape[0], self._expected_capped_seq_len(8, 256, size, total_seq_len=128))+            self.assertLess(capped_out.shape[0], default_out.shape[0])++    def test_cap_pixels_per_frame_unset_warns_and_false_is_silent(self):+        from transformers.utils import logging as transformers_logging++        logger = transformers_logging.get_logger("transformers.models.qwen2_vl.video_processing_qwen2_vl")+        size = {"longest_edge": 768 * 28 * 28, "shortest_edge": 400}+        for video_processing_class in self.video_processor_list:+            logger.warning_once.cache_clear()+            with self.assertLogs(logger.name, level="WARNING") as logs:+                self._process_frames(video_processing_class, 2, size)+            self.assertTrue(any("cap_pixels_per_frame" in line for line in logs.output))++            logger.warning_once.cache_clear()+            with self.assertNoLogs(logger.name, level="WARNING"):+                self._process_frames(video_processing_class, 2, size, cap_pixels_per_frame=False)

@@ -328,6 +328,85 @@ def test_call_sample_frames(self):             if prev_max_resolution is not None:                 self.video_processor_tester.max_resolution = prev_max_resolution +    def _process_frames(self, video_processing_class, num_frames, size, frame_size=256, **processor_kwargs):+        video_processor_dict = self.video_processor_dict.copy()+        video_processor_dict["size"] = size+        video_processor_dict["do_sample_frames"] = False+        video_processor_dict.update(processor_kwargs)+        video_processing = video_processing_class(**video_processor_dict)+        video = [np.random.randint(0, 256, (frame_size, frame_size, 3), dtype=np.uint8) for _ in range(num_frames)]+        return video_processing(video, return_tensors="pt")[self.input_name]++    def _expected_capped_seq_len(self, num_frames, frame_size, size):+        frame_cap = 768 * 32 * 32+        pixels_per_frame = max(min(frame_cap, size["longest_edge"] // num_frames), int(size["shortest_edge"] * 1.05))+        expected_height, expected_width = smart_resize(+            num_frames,+            frame_size,+            frame_size,+            factor=32,+            min_pixels=size["shortest_edge"],+            max_pixels=pixels_per_frame * num_frames,+        )+        return (num_frames // 2) * (expected_height // 16) * (expected_width // 16)++    def test_cap_pixels_per_frame_caps_short_videos(self):+        # A huge budget over few frames: uncapped keeps near-native frames, capped holds each+        # frame at the qwen-vl-utils 768-patch ceiling.+        size = {"longest_edge": 768 * 32 * 32 * 100, "shortest_edge": 32 * 32}+        for video_processing_class in self.video_processor_list:+            uncapped = self._process_frames(video_processing_class, 4, size, frame_size=1024)+            capped = self._process_frames(video_processing_class, 4, size, frame_size=1024, cap_pixels_per_frame=True)+            self.assertEqual(capped.shape[0], self._expected_capped_seq_len(4, 1024, size))+            self.assertLess(capped.shape[0], uncapped.shape[0])++    def test_cap_pixels_per_frame_keeps_tiny_clips_usable(self):+        # A 2-frame clip (e.g. a memory-profiling probe) stays at the per-frame ceiling instead+        # of collapsing toward min_pixels.+        size = {"longest_edge": 768 * 32 * 32 * 100, "shortest_edge": 32 * 32}+        for video_processing_class in self.video_processor_list:+            capped = self._process_frames(video_processing_class, 2, size, frame_size=1024, cap_pixels_per_frame=True)+            self.assertEqual(capped.shape[0], self._expected_capped_seq_len(2, 1024, size))++    def test_cap_pixels_per_frame_noop_when_not_binding(self):+        # When the budget's even share per frame is already below the ceiling, capped and+        # uncapped agree.+        size = {"longest_edge": 64 * 32 * 32, "shortest_edge": 32 * 32}+        for video_processing_class in self.video_processor_list:+            uncapped = self._process_frames(video_processing_class, 8, size)+            capped = self._process_frames(video_processing_class, 8, size, cap_pixels_per_frame=True)+            self.assertEqual(list(capped.shape), list(uncapped.shape))++    def test_cap_pixels_per_frame_call_time_override(self):+        size = {"longest_edge": 768 * 32 * 32 * 100, "shortest_edge": 32 * 32}+        for video_processing_class in self.video_processor_list:+            video_processor_dict = self.video_processor_dict.copy()+            video_processor_dict["size"] = size+            video_processor_dict["do_sample_frames"] = False+            video_processing = video_processing_class(**video_processor_dict)+            video = [np.random.randint(0, 256, (1024, 1024, 3), dtype=np.uint8) for _ in range(4)]++            default_out = video_processing(video, return_tensors="pt")[self.input_name]+            capped_out = video_processing(video, return_tensors="pt", cap_pixels_per_frame=True)[self.input_name]++            self.assertEqual(capped_out.shape[0], self._expected_capped_seq_len(4, 1024, size))+            self.assertLess(capped_out.shape[0], default_out.shape[0])++    def test_cap_pixels_per_frame_unset_warns_and_false_is_silent(self):+        from transformers.utils import logging as transformers_logging++        logger = transformers_logging.get_logger("transformers.models.qwen3_vl.video_processing_qwen3_vl")+        size = {"longest_edge": 64 * 32 * 32, "shortest_edge": 32 * 32}+        for video_processing_class in self.video_processor_list:+            logger.warning_once.cache_clear()+            with self.assertLogs(logger.name, level="WARNING") as logs:+                self._process_frames(video_processing_class, 2, size)+            self.assertTrue(any("cap_pixels_per_frame" in line for line in logs.output))++            logger.warning_once.cache_clear()+            with self.assertNoLogs(logger.name, level="WARNING"):+                self._process_frames(video_processing_class, 2, size, cap_pixels_per_frame=False)+     def test_num_frames_equal_temporal_patch_size_plus_two(self):         for video_processing_class in self.video_processor_list:             video_processor_dict = self.video_processor_dict.copy()