ggml-org/llama.cpp · #28273

mtmd: fix idefics3 preproc

ngxson · merged Sep 2, 20261 files · 50 + / 0
tools/mtmd/mtmd-image.cpp50 + / 0
@@ -980,6 +980,56 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i     //     // CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737     const clip_image_size original_size = img.get_size();++    // old gguf files have no preprocessor longest size, custom token limits also need the generic size below+    if (hparams.image_longest_edge > 0 && hparams.image_min_pixels <= 0 && hparams.image_max_pixels <= 0) {+        const int    tile_size    = hparams.image_size;+        const int    longest_edge = hparams.image_longest_edge;+        const double aspect_ratio = (double) original_size.width / original_size.height;++        clip_image_size resized_size;+        if (original_size.width >= original_size.height) {+            resized_size.width   = longest_edge;+            resized_size.height  = (int) (longest_edge / aspect_ratio);+            resized_size.height += resized_size.height % 2;+        } else {+            resized_size.height  = longest_edge;+            resized_size.width   = (int) (longest_edge * aspect_ratio);+            resized_size.width  += resized_size.width % 2;+        }++        const int grid_x = (resized_size.width  + tile_size - 1) / tile_size;+        const int grid_y = (resized_size.height + tile_size - 1) / tile_size;+        const clip_image_size refined_size = clip_image_size{grid_x * tile_size, grid_y * tile_size};++        clip_image_u8 resized_img;+        img_tool::resize(img, resized_img, resized_size, hparams.image_resize_algo, PAD_NONE);++        clip_image_u8 refined_img;+        img_tool::resize(resized_img, refined_img, refined_size, hparams.image_resize_algo, PAD_NONE);++        clip_image_u8 overview;+        img_tool::resize(refined_img, overview, {tile_size, tile_size}, hparams.image_resize_algo, PAD_NONE);++        std::vector<clip_image_u8> slices;+        for (int y = 0; y < grid_y; y++) {+            for (int x = 0; x < grid_x; x++) {+                clip_image_u8 slice;+                img_tool::crop(refined_img, slice, x * tile_size, y * tile_size, tile_size, tile_size);+                slices.push_back(std::move(slice));+            }+        }++        LOG_DBG("%s: grid size: %d x %d (%d tiles) + overview\n", __func__, grid_x, grid_y, grid_x * grid_y);++        mtmd_image_preproc_out output;+        output.append_overview(hparams, overview, true);+        output.append(hparams, slices, true);+        output.grid_x = grid_x;+        output.grid_y = grid_y;+        return output;+    }+     const clip_image_size refined_size = img_tool::calc_size_preserved_ratio(         original_size,         { hparams.image_size, std::max(0, hparams.image_min_pixels), std::max(0, hparams.image_max_pixels), hparams.image_longest_edge });