ggml-org/llama.cpp · #28133
mtmd: support DeepSeek-V4-Flash-Vision-Exp
conversion/__init__.py1 + / 0 −
@@ -286,6 +286,7 @@ "CogVLMForCausalLM": "cogvlm", "DeepseekOCR2ForCausalLM": "deepseek", "DeepseekOCRForCausalLM": "deepseek",+ "DeepseekV4ForCausalLM": "deepseek", "Dots3NoteForCausalLM": "dots3", "Dots3NoteForConditionalGeneration": "dots3", "DotsOCRForCausalLM": "dotsocr",conversion/deepseek.py73 + / 0 −
@@ -578,6 +578,9 @@ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Call @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, gen = item+ if (name.startswith(("aligner.", "image_"))+ or name.endswith(".ffn.gate.bias_vl")):+ return None if name.startswith("mtp."): if not cls.mtp_only: cls._skipped_mtp_tensors += 1@@ -1018,3 +1021,73 @@ def set_gguf_parameters(self): self.gguf_writer.add_block_size(self.hparams["dspark_block_size"]) self.gguf_writer.add_target_layers([layer + 1 for layer in self.hparams["dspark_target_layer_ids"]])+++@ModelBase.register("DeepseekV4ForCausalLM")+@ModelBase.example("deepseek-ai/DeepSeek-V4-Flash-Vision-Exp")+class DeepseekV4FlashVisionModel(MmprojModel):+ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ assert self.hparams_vision is not None+ # no preprocessor_config.json in the repo; normalization is (x/255 - 0.5) / 0.5+ # ref: inference/image_processor.py (load_image)+ self.preprocessor_config = {+ "image_mean": [0.5, 0.5, 0.5],+ "image_std": [0.5, 0.5, 0.5],+ **self.preprocessor_config,+ }++ def get_vision_config(self) -> dict[str, Any] | None:+ cfg = self.global_config+ if cfg.get("vision_n_layers", 0) == 0:+ raise ValueError("DeepseekV4FlashVisionModel requires vision_n_layers > 0 in the model config")+ return {+ "num_hidden_layers": cfg["vision_n_layers"],+ "hidden_size": cfg["vision_dim"],+ "num_attention_heads": cfg["vision_n_heads"],+ "intermediate_size": cfg["vision_inter_dim"],+ "patch_size": cfg["vision_patch_size"],+ # dynamic resolution; only used for compat / warmup+ "image_size": cfg["vision_patch_size"] * cfg["vision_downsample_ratio"] * 16,+ "rope_theta": cfg.get("vision_rope_theta", 10000.0),+ "downsample_ratio": cfg["vision_downsample_ratio"],+ "min_pixels": cfg["vision_min_pixels"],+ }++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ assert self.hparams_vision is not None+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.DEEPSEEK4V)+ # vision RMSNorm eps is the pytorch default, NOT the LLM's rms_norm_eps (1e-20)+ # ref: inference/vision.py (RMSNorm)+ self.gguf_writer.add_vision_attention_layernorm_eps(1e-6)+ self.gguf_writer.add_vision_use_silu(True) # SwiGLU MLP+ self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"])+ self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"])+ # hardcoded on the C++ side (see PROJECTOR_TYPE_DEEPSEEK4V in clip.cpp)+ # if future models use different values, add GGUF keys for those+ assert self.global_config["vision_max_n_token"] == 384+ assert self.global_config["vision_max_wh_ratio"] == 8++ @classmethod+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:+ name, _ = item+ if not (name.startswith(("vision.", "aligner.", "image_"))):+ return None+ return super().filter_tensors(item)++ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+ assert self.hparams_vision is not None+ if name == "vision.patch_embed.proj.weight":+ # nn.Linear over flattened (3, p, p) patches == conv2d weight+ p = self.hparams_vision["patch_size"]+ data_torch = data_torch.reshape(data_torch.shape[0], 3, p, p)++ if ".mlp.w1." in name:+ # fused SwiGLU gate+up+ gate, up = data_torch.chunk(2, dim=0)+ yield from super().modify_tensors(gate, name.replace("w1", "w1_gate"), bid)+ yield from super().modify_tensors(up, name.replace("w1", "w1_up"), bid)+ return++ yield from super().modify_tensors(data_torch, name, bid)gguf-py/gguf/constants.py10 + / 0 −
@@ -950,6 +950,9 @@ class MODEL_TENSOR(IntEnum): V_RESMPL_PROJ = auto() # minicpmv V_RESMPL_QUERY = auto() # minicpmv V_TOK_EMBD_IMG_BREAK = auto() # pixtral+ V_TOK_EMBD_IMG_START = auto() # deepseek4v+ V_TOK_EMBD_IMG_END = auto() # deepseek4v+ V_TOK_EMBD_IMG_PAD = auto() # deepseek4v V_MM_PATCH_MERGER = auto() # mistral small 3.1 V_DS_NORM = auto() # qwen3vl V_DS_FC1 = auto() # qwen3vl@@ -1696,6 +1699,9 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_RESMPL_PROJ: "resampler.proj", MODEL_TENSOR.V_RESMPL_QUERY: "resampler.query", MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK: "v.token_embd.img_break", # pixtral+ MODEL_TENSOR.V_TOK_EMBD_IMG_START: "v.token_embd.img_start", # deepseek4v+ MODEL_TENSOR.V_TOK_EMBD_IMG_END: "v.token_embd.img_end", # deepseek4v+ MODEL_TENSOR.V_TOK_EMBD_IMG_PAD: "v.token_embd.img_pad", # deepseek4v MODEL_TENSOR.V_MM_PATCH_MERGER: "mm.patch_merger", # mistral small 3.1 MODEL_TENSOR.V_DS_NORM: "v.deepstack.{bid}.norm", MODEL_TENSOR.V_DS_FC1: "v.deepstack.{bid}.fc1",@@ -2030,6 +2036,9 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_RESMPL_PROJ, MODEL_TENSOR.V_RESMPL_QUERY, MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK,+ MODEL_TENSOR.V_TOK_EMBD_IMG_START,+ MODEL_TENSOR.V_TOK_EMBD_IMG_END,+ MODEL_TENSOR.V_TOK_EMBD_IMG_PAD, MODEL_TENSOR.V_MM_PATCH_MERGER, MODEL_TENSOR.V_MM_MERGER_FC1, MODEL_TENSOR.V_MM_MERGER_FC2,@@ -5645,6 +5654,7 @@ class VisionProjectorType: DOTS3NOTE_A = "dots3note_a" # audio DEEPSEEKOCR = "deepseekocr" DEEPSEEKOCR2 = "deepseekocr2"+ DEEPSEEK4V = "deepseek4v" LFM2A = "lfm2a" # audio MUSIC_FLAMINGO = "musicflamingo" # audio GLM4V = "glm4v"gguf-py/gguf/tensor_mapping.py23 + / 0 −
@@ -1476,6 +1476,7 @@ class TensorNameMap: ## Vision encoder MODEL_TENSOR.V_MMPROJ: (+ "aligner.w{bid}", # deepseek4v (w1 -> mm.1, w2 -> mm.2) "multi_modal_projector.linear_{bid}", "mm_projector.proj.linear_{bid}", # Kimi-K2.5 "visual.merger.mlp.{bid}", # qwen2vl@@ -1515,6 +1516,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_EMBD_PATCH: (+ "vision.patch_embed.proj", # deepseek4v "model.vision_tower.vision_model.embeddings.patch_embedding", # Granite4Vision "vision_tower.vision_model.embeddings.patch_embedding", "model.vision_tower.embeddings.patch_embedding", # minicpmv4_6@@ -1570,6 +1572,7 @@ class TensorNameMap: # TODO: I think these should all be moved to mapping_cfg? MODEL_TENSOR.V_ENC_EMBD_IMGNL: (+ "image_newline", # deepseek4v "model.image_newline", # Deepseek-OCR, Granite4Vision "vit.perceive.image_newline", # HunyuanVL ),@@ -1580,6 +1583,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_QKV: (+ "vision.blocks.{bid}.attn.wqkv", # deepseek4v "visual.blocks.{bid}.attn.qkv", # qwen3vl "vision_tower.blocks.{bid}.attn.qkv", # dots.ocr "vision_encoder.blocks.{bid}.attn.qkv", # dots3note@@ -1667,6 +1671,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_INPUT_NORM: (+ "vision.blocks.{bid}.norm1", # deepseek4v "model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", "model.vision_tower.encoder.layers.{bid}.layer_norm1", # minicpmv4_6@@ -1692,6 +1697,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_ATTN_O: (+ "vision.blocks.{bid}.attn.wo", # deepseek4v "model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", "model.vision_tower.encoder.layers.{bid}.self_attn.out_proj", # minicpmv4_6@@ -1723,6 +1729,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_POST_ATTN_NORM: (+ "vision.blocks.{bid}.norm2", # deepseek4v "model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", "model.vision_tower.encoder.layers.{bid}.layer_norm2", # minicpmv4_6@@ -1749,6 +1756,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_FFN_UP: (+ "vision.blocks.{bid}.mlp.w1_up", # deepseek4v (split from fused w1) "vision_encoder.blocks.{bid}.mlp.fc3", # dots3note "model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1",@@ -1775,6 +1783,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_FFN_GATE: (+ "vision.blocks.{bid}.mlp.w1_gate", # deepseek4v (split from fused w1) "vision_encoder.blocks.{bid}.mlp.fc1", # dots3note "vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf "vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral@@ -1784,6 +1793,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_ENC_FFN_DOWN: (+ "vision.blocks.{bid}.mlp.w2", # deepseek4v "vision_encoder.blocks.{bid}.mlp.fc2", # dots3note "model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", # Granite4Vision "vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2",@@ -1869,6 +1879,7 @@ class TensorNameMap: ), MODEL_TENSOR.V_POST_NORM: (+ "vision.norm", # deepseek4v "model.vision_tower.vision_model.post_layernorm", # Granite4Vision "vision_tower.vision_model.post_layernorm", "model.vision_tower.post_layernorm", # minicpmv4_6@@ -1960,6 +1971,18 @@ class TensorNameMap: "v.token_embd.img_break", # for pixtral, this is a generated vector ), + MODEL_TENSOR.V_TOK_EMBD_IMG_START: (+ "image_start", # deepseek4v+ ),++ MODEL_TENSOR.V_TOK_EMBD_IMG_END: (+ "image_end", # deepseek4v+ ),++ MODEL_TENSOR.V_TOK_EMBD_IMG_PAD: (+ "image_pad", # deepseek4v+ ),+ MODEL_TENSOR.V_MM_PATCH_MERGER: ( "multi_modal_projector.patch_merger.merging_layer", # mistral small 3.1 - hf "patch_merger.merging_layer", # mistraltools/mtmd/CMakeLists.txt1 + / 0 −
@@ -30,6 +30,7 @@ add_library(mtmd models/models.h models/cogvlm.cpp models/conformer.cpp+ models/deepseek4v.cpp models/dots3note.cpp models/dotsocr.cpp models/exaone4_5.cpptools/mtmd/clip-impl.h26 + / 2 −
@@ -153,6 +153,9 @@ #define TN_MM_MERGER_FC1 "mm.merger.fc1.%s" // minimax-m3 patch-merge MLP #define TN_MM_MERGER_FC2 "mm.merger.fc2.%s" #define TN_TOK_IMG_BREAK "v.token_embd.img_break" // pixtral+#define TN_TOK_IMG_START "v.token_embd.img_start" // deepseek4v+#define TN_TOK_IMG_END "v.token_embd.img_end" // deepseek4v+#define TN_TOK_IMG_PAD "v.token_embd.img_pad" // deepseek4v #define TN_TOK_GLM_BOI "adapter.boi" // glm-edge (these embeddings are not in text model) #define TN_TOK_GLM_EOI "adapter.eoi" // glm-edge (these embeddings are not in text model) #define TN_DEEPSTACK_NORM "v.deepstack.%d.norm.%s" // qwen3vl deepstack@@ -296,8 +299,8 @@ // hunyuanvl (shared GGUF tensor names) #define TN_MM_PRE_NORM "mm.pre_norm.%s"-#define TN_TOK_IMG_BEGIN "mm.image_begin"-#define TN_TOK_IMG_END "mm.image_end"+#define TN_MM_IMG_BEGIN "mm.image_begin" // note: legacy name, new models should use v.token_embd.*+#define TN_MM_IMG_END "mm.image_end" // note: legacy name, new models should use v.token_embd.* // deepseek-ocr #define TN_SAM_POS_EMBD "v.sam.pos_embd.%s"@@ -480,6 +483,7 @@ enum projector_type { PROJECTOR_TYPE_DOTS3NOTE_A, PROJECTOR_TYPE_DEEPSEEKOCR, PROJECTOR_TYPE_DEEPSEEKOCR2,+ PROJECTOR_TYPE_DEEPSEEK4V, PROJECTOR_TYPE_LFM2A, PROJECTOR_TYPE_GLM4V, PROJECTOR_TYPE_YOUTUVL,@@ -544,6 +548,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_DOTS3NOTE_A, "dots3note_a"}, { PROJECTOR_TYPE_DEEPSEEKOCR, "deepseekocr"}, { PROJECTOR_TYPE_DEEPSEEKOCR2, "deepseekocr2"},+ { PROJECTOR_TYPE_DEEPSEEK4V, "deepseek4v"}, { PROJECTOR_TYPE_LFM2A, "lfm2a"}, { PROJECTOR_TYPE_GLM4V, "glm4v"}, { PROJECTOR_TYPE_YOUTUVL, "youtuvl"},@@ -655,6 +660,9 @@ struct clip_image_f32 { // appends a learned newline (or EOI) token after the image // no model uses it now (Granite4 Vision moved to anyres), kept for future models bool add_newline = false;+ // deepseek4v: number of leading IMAGE_PAD embeddings, aligns IMAGE_START to the LLM compressor ratio+ // depends on the chunk position, set at tokenize time (see mtmd_tokenizer::add_media)+ int32_t lead_pad = 0; // llava-next "anyres" tiling, used by Granite4 Vision // the whole grid is encoded and assembled in a single graph@@ -771,6 +779,22 @@ static inline void clip_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_ } } +// deepseek4v: layout of the LLM token block built from the aligner grid+struct dsv4_block_layout {+ int rows; // grid rows, padded to an even count+ int row_len; // grid width + 1 newline+ int pad_last; // trailing pads+ int n_out; // total block size, including lead pads and the start/end sentinels+};+static inline dsv4_block_layout dsv4_get_block_layout(int n_llm_w, int n_llm_h, int lead_pad) {+ dsv4_block_layout bl;+ bl.rows = n_llm_h + (n_llm_h % 2);+ bl.row_len = n_llm_w + 1;+ bl.pad_last = (bl.rows / 2 * bl.row_len) % 2 * 2;+ bl.n_out = lead_pad + 1 + bl.rows * bl.row_len + bl.pad_last + 1;+ return bl;+}+ // // logging //tools/mtmd/clip-model.h9 + / 0 −
@@ -100,6 +100,10 @@ struct clip_hparams { std::unordered_set<int32_t> wa_layer_indexes; // explicit layer indexes that use full attention (for irregular patterns like YoutuVL) std::vector<int32_t> wa_pattern_mode; // mimovl: per-layer window-attention mode + // deepseek4v: resize solver caps the LLM token count of the aligner grid+ int32_t dsv4_max_n_token = 0;+ int32_t dsv4_max_wh_ratio = 0;+ // deepseek-ocr (sam) int32_t sam_n_layer = 0; int32_t sam_n_head = 0;@@ -724,6 +728,11 @@ struct clip_model { // pixtral, glm4v ggml_tensor * token_embd_img_break = nullptr;++ // deepseek4v sentinel embeddings (image_newline is reused for IMAGE_NEW_LINE)+ ggml_tensor * token_embd_img_start = nullptr;+ ggml_tensor * token_embd_img_end = nullptr;+ ggml_tensor * token_embd_img_pad = nullptr; ggml_tensor * mm_patch_merger_w = nullptr; ggml_tensor * mm_patch_merger_b = nullptr; tools/mtmd/clip.cpp116 + / 2 −
@@ -1037,6 +1037,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique<clip_graph_kimik25>(ctx, img); } break;+ case PROJECTOR_TYPE_DEEPSEEK4V:+ {+ builder = std::make_unique<clip_graph_deepseek4v>(ctx, img);+ } break; case PROJECTOR_TYPE_COGVLM: { builder = std::make_unique<clip_graph_cogvlm>(ctx, img);@@ -1585,6 +1589,31 @@ struct clip_model_loader { hparams.set_limit_image_tokens(2, 4096); } } break;+ case PROJECTOR_TYPE_DEEPSEEK4V:+ {+ hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;+ hparams.image_pad_color = {127, 127, 127};+ hparams.rope_theta = 10000.0f;+ get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);+ get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);+ hparams.dsv4_max_n_token = 384;+ hparams.dsv4_max_wh_ratio = 8;+ const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;+ // handle min/max token counts from CLI+ if (hparams.custom_image_min_tokens > 0) {+ hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area;+ }+ if (hparams.custom_image_max_tokens > 0) {+ // the cap is on the whole token block, keep some room for the resize solver+ hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16);+ }+ hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area;+ // a small custom max token count also lowers the min-pixel upscale threshold+ hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels);+ // avoid OOM on warmup+ const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token));+ hparams.set_warmup_n_tokens(warmup_side * warmup_side);+ } break; case PROJECTOR_TYPE_GEMMA3: { // default value (used by all model sizes in gemma 3 family)@@ -2714,6 +2743,18 @@ struct clip_model_loader { model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); } break;+ case PROJECTOR_TYPE_DEEPSEEK4V:+ {+ model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));+ model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));+ model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));+ model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));+ // sentinel token embeddings written into the output block+ model.image_newline = get_tensor(TN_IMAGE_NEWLINE);+ model.token_embd_img_start = get_tensor(TN_TOK_IMG_START);+ model.token_embd_img_end = get_tensor(TN_TOK_IMG_END);+ model.token_embd_img_pad = get_tensor(TN_TOK_IMG_PAD);+ } break; case PROJECTOR_TYPE_PIXTRAL: { model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));@@ -3161,8 +3202,8 @@ struct clip_model_loader { model.mm_model_proj_b = get_tensor(string_format(TN_MM_PROJECTOR, "bias")); model.mm_pre_norm_w = get_tensor(string_format(TN_MM_PRE_NORM, "weight")); model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight"));- model.mm_img_begin = get_tensor(TN_TOK_IMG_BEGIN);- model.mm_img_end = get_tensor(TN_TOK_IMG_END);+ model.mm_img_begin = get_tensor(TN_MM_IMG_BEGIN);+ model.mm_img_end = get_tensor(TN_MM_IMG_END); model.image_newline = get_tensor(TN_IMAGE_NEWLINE); model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR, false); } break;@@ -4150,6 +4191,13 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { int y_patch = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size; n_patches = x_patch * y_patch; } break;+ case PROJECTOR_TYPE_DEEPSEEK4V:+ {+ const int out_patch_size = params.patch_size * params.n_merge;+ const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size;+ const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;+ n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out;+ } break; case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_DOTS_OCR: case PROJECTOR_TYPE_DOTS3NOTE_V:@@ -5021,6 +5069,58 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { } set_input_i32("pos_w", pos_data); } break;+ case PROJECTOR_TYPE_DEEPSEEK4V:+ {+ // set the 2D positions (mrope layout, only the first 2 channels are used)+ int n_patches_per_row = image_size_width / patch_size;+ std::vector<int32_t> positions(n_pos * 4, 0);+ for (int i = 0; i < n_pos; i++) {+ positions[i] = i / n_patches_per_row; // row+ positions[n_pos + i] = i % n_patches_per_row; // col+ }+ set_input_i32("positions", positions);++ // token block layout index (see clip_graph_deepseek4v::build)+ // rows [0, n_grid) are the aligner output, the sentinels follow+ const int n_merge = hparams.n_merge;+ const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge;+ const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge;+ const int n_grid = n_llm_w * n_llm_h;+ const int idx_start = n_grid;+ const int idx_end = n_grid + 1;+ const int idx_newline = n_grid + 2;+ const int idx_pad = n_grid + 3;++ const int lead_pad = imgs.entries[0].lead_pad;+ const auto bl = dsv4_get_block_layout(n_llm_w, n_llm_h, lead_pad);++ std::vector<int32_t> idx;+ idx.reserve(bl.n_out);+ for (int i = 0; i < lead_pad; i++) {+ idx.push_back(idx_pad);+ }+ idx.push_back(idx_start);+ // pairs of adjacent rows are interleaved column-wise ("N-layout")+ // ref: build_image_block in inference/image_processor.py+ for (int t = 0; t < bl.rows * bl.row_len; t++) {+ const int g = t / (2 * bl.row_len);+ const int rem = t % (2 * bl.row_len);+ const int c = rem / 2; // column+ const int r = 2 * g + rem % 2; // row+ if (r >= n_llm_h) {+ idx.push_back(idx_pad);+ } else if (c == n_llm_w) {+ idx.push_back(idx_newline);+ } else {+ idx.push_back(r * n_llm_w + c);+ }+ }+ for (int i = 0; i < bl.pad_last; i++) {+ idx.push_back(idx_pad);+ }+ idx.push_back(idx_end);+ set_input_i32("layout_idx", idx);+ } break; case PROJECTOR_TYPE_GLM_EDGE: { // llava and other models@@ -5762,6 +5862,19 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n"); LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens); + // TEMP debugging (parity validation), will be removed before merge+ // when the env var holds a path, dump the raw data: [int32 n_tokens][int32 n_embd][f32 data]+ const char * dump_path = std::getenv("MTMD_DEBUG_EMBEDDINGS");+ if (dump_path && strcmp(dump_path, "1") != 0) {+ FILE * f = fopen(dump_path, "wb");+ if (f) {+ const int32_t hdr[2] = { (int32_t)n_tokens, (int32_t)n_embd };+ fwrite(hdr, sizeof(hdr), 1, f);+ fwrite(emb_data.data(), sizeof(float), emb_data.size(), f);+ fclose(f);+ }+ }+ // Print first few values of first token LOG_INF("Token 0 (first 16 values): "); for (int i = 0; i < std::min((int64_t)16, n_embd); i++) {@@ -5866,6 +5979,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_KIMIK25: case PROJECTOR_TYPE_YASA2:+ case PROJECTOR_TYPE_DEEPSEEK4V: return ctx->model.mm_2_w->ne[1]; case PROJECTOR_TYPE_HUNYUANVL: return ctx->model.mm_model_proj->ne[1];tools/mtmd/models/deepseek4v.cppadded102 + / 0 −
@@ -0,0 +1,102 @@+#include "models.h"++// DeepSeek-V4-Flash-Vision encoder (deepseek4v)+//+// native-resolution ViT (RMSNorm, SwiGLU, 2D RoPE, no CLS / learned pos-embd)+// then the "aligner": 3x3 patch merge (torch.nn.functional.unfold) + 2-layer GELU MLP+//+// the graph outputs the complete LLM token block, built from the aligner output and 4 learned sentinel embeddings:+//+// [PAD]*lead_pad [START] <interleaved rows> [PAD]*pad_last [END]+//+// each aligner row ends with a NEWLINE, an odd row count is padded with a full row of PADs+// pairs of adjacent rows are interleaved column-wise ("N-layout")+// the mapping is precomputed on CPU as the "layout_idx" input (see set_input in clip.cpp)+//+// ref: inference/vision.py and inference/image_processor.py in the HF repo++ggml_cgraph * clip_graph_deepseek4v::build() {+ const int n_merge = hparams.n_merge;++ // 2D input positions+ ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches * 4);+ ggml_set_name(positions, "positions");+ ggml_set_input(positions);++ int sections[4] = {d_head/4, d_head/4, 0, 0};+ auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {+ return ggml_rope_multi(ctx0, cur, positions, nullptr,+ d_head/2, sections, GGML_ROPE_TYPE_VISION,+ 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);+ };++ ggml_tensor * inp = build_inp();+ ggml_tensor * cur = build_vit(+ inp, n_patches,+ NORM_TYPE_RMS,+ hparams.ffn_op,+ nullptr, // no learned pos embd+ add_pos);+ cb(cur, "vit_out", -1);++ // aligner patch merge: zero-pad the patch grid to a multiple of n_merge+ // then F.unfold == im2col with a dummy kernel (same trick as pixtral)+ {+ cur = ggml_reshape_3d(ctx0, cur, n_embd, n_patches_x, n_patches_y);+ cur = ggml_permute(ctx0, cur, 2, 0, 1, 3); // [x, y, n_embd]+ cur = ggml_cont(ctx0, cur);++ const int pad_x = (n_merge - n_patches_x % n_merge) % n_merge;+ const int pad_y = (n_merge - n_patches_y % n_merge) % n_merge;+ if (pad_x || pad_y) {+ cur = ggml_pad(ctx0, cur, pad_x, pad_y, 0, 0);+ }++ ggml_tensor * kernel = ggml_view_3d(ctx0, cur, n_merge, n_merge, cur->ne[2], 0, 0, 0);+ cur = ggml_im2col(ctx0, kernel, cur, n_merge, n_merge, 0, 0, 1, 1, true, inp->type);+ cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]);++ // aligner MLP (F.gelu in the reference == erf-based gelu)+ cur = build_ffn(cur,+ model.mm_1_w, model.mm_1_b,+ nullptr, nullptr,+ model.mm_2_w, model.mm_2_b,+ FFN_GELU_ERF,+ -1);+ cb(cur, "aligner_out", -1);+ }++ // assemble the token block: append the sentinel embeddings as extra rows+ // then reorder everything with the precomputed layout index+ {+ const int64_t n_embd_out = cur->ne[0];+ const int64_t n_grid = cur->ne[1]; // n_llm_w * n_llm_h++ // rows n_grid + 0..3, keep in sync with the index computation in set_input+ ggml_tensor * sentinels[] = {+ model.token_embd_img_start,+ model.token_embd_img_end,+ model.image_newline,+ model.token_embd_img_pad,+ };+ for (ggml_tensor * tok : sentinels) {+ cur = ggml_concat(ctx0, cur, ggml_reshape_2d(ctx0, tok, n_embd_out, 1), 1);+ }++ const int n_llm_w = CLIP_ALIGN(n_patches_x, n_merge) / n_merge;+ const int n_llm_h = CLIP_ALIGN(n_patches_y, n_merge) / n_merge;+ const int n_out = dsv4_get_block_layout(n_llm_w, n_llm_h, img.lead_pad).n_out;+ GGML_ASSERT(n_grid == n_llm_w * n_llm_h);++ ggml_tensor * layout_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_out);+ ggml_set_name(layout_idx, "layout_idx");+ ggml_set_input(layout_idx);++ cur = ggml_get_rows(ctx0, cur, layout_idx);+ }++ // build the graph+ ggml_build_forward_expand(gf, cur);++ return gf;+}tools/mtmd/models/models.h5 + / 0 −
@@ -34,6 +34,11 @@ struct clip_graph_pixtral : clip_graph { ggml_cgraph * build() override; }; +struct clip_graph_deepseek4v : clip_graph {+ clip_graph_deepseek4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}+ ggml_cgraph * build() override;+};+ struct clip_graph_qwen2vl : clip_graph { clip_graph_qwen2vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override;tools/mtmd/mtmd-image.cpp102 + / 0 −
@@ -1092,6 +1092,108 @@ clip_image_size mtmd_image_preprocessor_deepseekocr::find_closest_aspect_ratio( return best_ratio; } +//+// DeepSeek-V4-Flash-Vision (deepseek4v)+//+// port of load_image / safe_resize / solve_resize_ratio / grid_tokens from inference/image_processor.py+// the resize solver picks the largest target size (multiple of patch_size) whose LLM token block fits max_n_token+//++// ref: grid_tokens()+mtmd_image_preprocessor_deepseek4v::grid_info mtmd_image_preprocessor_deepseek4v::grid_tokens(int best_height, int best_width, int patch_size, int r) {+ grid_info g;+ g.n_llm_h = ((best_height / patch_size) + r - 1) / r;+ g.n_llm_w = ((best_width / patch_size) + r - 1) / r;+ g.n_tokens = dsv4_get_block_layout(g.n_llm_w, g.n_llm_h, 0).n_out;+ return g;+}++// ref: solve_resize_ratio()+void mtmd_image_preprocessor_deepseek4v::solve_resize_ratio(int height, int width, int p, int r, int max_n_token,+ int & best_height, int & best_width) {+ const double ratio = (double) height / width;+ const double max_w_f = std::sqrt((max_n_token - 2) / ratio + 0.25) - 0.5;+ const double max_h_f = max_w_f * ratio;+ if (max_w_f < 1.0) {+ const int max_w = 1;+ int max_h = (max_n_token - 2) / (max_w + 1);+ if (max_h % 2 == 1) {+ max_h -= 1;+ }+ best_width = max_w * p * r;+ best_height = max_h * p * r;+ } else if (max_h_f < 2.0) {+ const int max_h = 2;+ // guard tiny budgets; cannot be hit with the current lower bound on max_n_token+ const int max_w = std::max(((max_n_token - 2) / max_h) - 1, 2);+ best_width = max_w * p * r;+ best_height = max_h * p * r;+ } else {+ const int max_w_i = (int) std::floor(max_w_f);+ int max_h_i = (int) std::floor(max_h_f);+ if (max_h_i % 2 == 1) {+ max_h_i -= 1;+ }+ const double beta = std::min(+ (double) max_w_i * p * r / width,+ (double) max_h_i * p * r / height);+ best_width = (int) std::floor(width * beta / p) * p;+ best_height = (int) std::floor(height * beta / p) * p;+ }+}++// ref: safe_resize()+void mtmd_image_preprocessor_deepseek4v::safe_resize(int height, int width, int & best_height, int & best_width,+ int p, int r, int max_n_token) {+ max_n_token -= 4 - 1; // reserve room for the position-dependent lead pads (COMPRESS_PAD_TO - 1)+ grid_info g = grid_tokens(best_height, best_width, p, r);+ int budget = max_n_token;+ while (g.n_tokens > max_n_token) {+ solve_resize_ratio(height, width, p, r, budget, best_height, best_width);+ g = grid_tokens(best_height, best_width, p, r);+ budget -= 1;+ }+}++// ref: load_image()+mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip_image_u8 & img) {+ mtmd_image_preproc_out out;++ const int p = hparams.patch_size;+ const int r = hparams.n_merge;+ const int max_n_token = hparams.dsv4_max_n_token;+ const int max_wh = hparams.dsv4_max_wh_ratio;++ const clip_image_size orig = img.get_size();+ int width = orig.width;+ int height = orig.height;+ if (max_wh > 0 && width > height * max_wh) {+ width = height * max_wh;+ }+ if (hparams.image_min_pixels > 0 && width * height > 0+ && width * height < hparams.image_min_pixels) {+ const double up = std::sqrt((double) hparams.image_min_pixels / ((double) width * height));+ width = (int) (width * up);+ height = (int) (height * up);+ }+ int best_width = CLIP_ALIGN(width, p);+ int best_height = CLIP_ALIGN(height, p);+ safe_resize(height, width, best_height, best_width, p, r, max_n_token);++ clip_image_u8 resized;+ if (max_wh > 0 && orig.width >= max_wh * orig.height) {+ // extreme aspect ratio: plain stretch resize, no padding+ img_tool::resize(img, resized, {best_width, best_height}, hparams.image_resize_algo, PAD_NONE);+ } else {+ // aspect-preserving resize + centered padding (PIL ImageOps.pad)+ img_tool::resize(img, resized, {best_width, best_height}, hparams.image_resize_algo,+ PAD_NEAREST, hparams.image_pad_color);+ }++ out.append(hparams, resized);+ return out;+}+ mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) { mtmd_image_preproc_out output; int grid_w = 0;tools/mtmd/mtmd-image.h16 + / 0 −
@@ -129,6 +129,22 @@ struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor { mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; }; +// ref: inference/image_processor.py in the HF repo (DeepSeek-V4-Flash-Vision)+struct mtmd_image_preprocessor_deepseek4v : mtmd_image_preprocessor {+ mtmd_image_preprocessor_deepseek4v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}+ mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;++private:+ struct grid_info {+ int n_llm_h;+ int n_llm_w;+ int n_tokens; // token count of the block (incl. newline/pad rows and start/end, excl. lead pads)+ };+ static grid_info grid_tokens(int best_height, int best_width, int patch_size, int r);+ static void solve_resize_ratio(int height, int width, int p, int r, int max_n_token, int & best_height, int & best_width);+ static void safe_resize(int height, int width, int & best_height, int & best_width, int p, int r, int max_n_token);+};+ // custom llava-uhd slicing logic for MiniCPM-V struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd { using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;tools/mtmd/mtmd.cpp20 + / 1 −
@@ -27,7 +27,7 @@ #include <vector> // remember to bump this if the serialization format changes-#define MTMD_SERIALIZATION_VERSION 1+#define MTMD_SERIALIZATION_VERSION 2 struct mtmd_serialization { // note: using 64-bit here for future-proofing@@ -105,12 +105,14 @@ void clip_image_f32::serialize(mtmd_serialization & ser) const { // note: buf is intentionally NOT serialized; the loaded clip_image_f32 will always be a placeholder ser.write(add_viewsep); ser.write(add_newline);+ ser.write(lead_pad); ser.write((int32_t)nx_); ser.write((int32_t)ny_); } void clip_image_f32::deserialize(mtmd_serialization & ser) { add_viewsep = ser.read<bool>(); add_newline = ser.read<bool>();+ lead_pad = ser.read<int32_t>(); nx_ = ser.read<int32_t>(); ny_ = ser.read<int32_t>(); buf.clear(); // always a placeholder after loading@@ -824,6 +826,11 @@ struct mtmd_context { img_end = "<|im_end|>"; image_preproc = std::make_unique<mtmd_image_preprocessor_longest_edge>(ctx_v); } break;+ case PROJECTOR_TYPE_DEEPSEEK4V:+ {+ // no vocab tokens are added; the start/end/newline markers are learned embeddings emitted by the encoder+ image_preproc = std::make_unique<mtmd_image_preprocessor_deepseek4v>(ctx_v);+ } break; case PROJECTOR_TYPE_DOTS_OCR: case PROJECTOR_TYPE_DOTS3NOTE_V: {@@ -1451,6 +1458,18 @@ struct mtmd_tokenizer { return 2; } + if (ctx->proj_type_v() == PROJECTOR_TYPE_DEEPSEEK4V) {+ // the text model perceives input in blocks of N tokens (N = COMPRESS_PAD_TO = 4, same as the CSA compress ratio)+ // image need to be aligned to block size, while adding IMAGE_PAD embeddings to the beginning+ // TODO @ngxson : maybe refactor this in the future+ constexpr int32_t align = 4;+ size_t n_past = 0;+ for (const auto & e : cur.entries) {+ n_past += mtmd_input_chunk_get_n_tokens(&e);+ }+ preproc_out.entries[0].lead_pad = align - 1 - (int32_t)(n_past % align);+ }+ size_t n_tokens = 0; for (auto & e : preproc_out.entries) { n_tokens += clip_n_output_tokens(ctx->ctx_v, &e);