ggml-org/llama.cpp · #28273
mtmd: fix idefics3 preproc
tools/mtmd/mtmd-image.cpp50 + / 0 −
@@ -980,6 +980,56 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i // // CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737 const clip_image_size original_size = img.get_size();++ // old gguf files have no preprocessor longest size, custom token limits also need the generic size below+ if (hparams.image_longest_edge > 0 && hparams.image_min_pixels <= 0 && hparams.image_max_pixels <= 0) {+ const int tile_size = hparams.image_size;+ const int longest_edge = hparams.image_longest_edge;+ const double aspect_ratio = (double) original_size.width / original_size.height;++ clip_image_size resized_size;+ if (original_size.width >= original_size.height) {+ resized_size.width = longest_edge;+ resized_size.height = (int) (longest_edge / aspect_ratio);+ resized_size.height += resized_size.height % 2;+ } else {+ resized_size.height = longest_edge;+ resized_size.width = (int) (longest_edge * aspect_ratio);+ resized_size.width += resized_size.width % 2;+ }++ const int grid_x = (resized_size.width + tile_size - 1) / tile_size;+ const int grid_y = (resized_size.height + tile_size - 1) / tile_size;+ const clip_image_size refined_size = clip_image_size{grid_x * tile_size, grid_y * tile_size};++ clip_image_u8 resized_img;+ img_tool::resize(img, resized_img, resized_size, hparams.image_resize_algo, PAD_NONE);++ clip_image_u8 refined_img;+ img_tool::resize(resized_img, refined_img, refined_size, hparams.image_resize_algo, PAD_NONE);++ clip_image_u8 overview;+ img_tool::resize(refined_img, overview, {tile_size, tile_size}, hparams.image_resize_algo, PAD_NONE);++ std::vector<clip_image_u8> slices;+ for (int y = 0; y < grid_y; y++) {+ for (int x = 0; x < grid_x; x++) {+ clip_image_u8 slice;+ img_tool::crop(refined_img, slice, x * tile_size, y * tile_size, tile_size, tile_size);+ slices.push_back(std::move(slice));+ }+ }++ LOG_DBG("%s: grid size: %d x %d (%d tiles) + overview\n", __func__, grid_x, grid_y, grid_x * grid_y);++ mtmd_image_preproc_out output;+ output.append_overview(hparams, overview, true);+ output.append(hparams, slices, true);+ output.grid_x = grid_x;+ output.grid_y = grid_y;+ return output;+ }+ const clip_image_size refined_size = img_tool::calc_size_preserved_ratio( original_size, { hparams.image_size, std::max(0, hparams.image_min_pixels), std::max(0, hparams.image_max_pixels), hparams.image_longest_edge });