[Model] Support for Spark2_5ForCausalLM implementation

@@ -255,6 +255,7 @@     "SeedOssForCausalLM": "olmo",     "SmallThinkerForCausalLM": "smallthinker",     "SmolLM3ForCausalLM": "llama",+    "Spark2_5ForCausalLM": "spark2_5",     "SolarOpenForCausalLM": "glm",     "StableLMEpochForCausalLM": "stablelm",     "StableLmForCausalLM": "stablelm",

@@ -1543,6 +1543,9 @@ def get_vocab_base_pre(self, tokenizer) -> str:         if chkhsh == "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7":             # ref: https://huggingface.co/LiquidAI/LFM2.5-8B-A1B             res = "lfm2"+        if chkhsh == "0a766d034107bc736a3f2dc4968fd62e54a3570f1454443e0c5a4cc6bd7941ed":+            # ref: https://huggingface.co/XHToken/Spark-X2.5-1.7B+            res = "spark2_5"         if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5":             # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B             res = "llama-bpe"

@@ -0,0 +1,65 @@+from __future__ import annotations++from collections.abc import Iterable+from typing import TYPE_CHECKING++if TYPE_CHECKING:+    from torch import Tensor++from .base import ModelBase, TextModel, gguf+++@ModelBase.register("Spark2_5ForCausalLM")+@ModelBase.example("XHToken/Spark-X2.5-1.7B")+class Spark2_5Model(TextModel):+    model_arch = gguf.MODEL_ARCH.SPARK2_5++    def set_gguf_parameters(self) -> None:+        super().set_gguf_parameters()++        hparams = self.hparams+        layer_types = hparams["layer_types"]+        if len(layer_types) != self.block_count:+            raise ValueError(+                f"Spark2_5 layer_types length {len(layer_types)} != num_hidden_layers {self.block_count}"+            )+        if any(layer_type not in ("sliding_attention", "full_attention") for layer_type in layer_types):+            raise ValueError(f"Spark2_5 has unsupported layer_types: {layer_types}")+        if hparams.get("gate_attn_act_mode") != "sigmoid" or hparams.get("headwise_attn_output_gate") is not True:+            raise ValueError("Spark2_5 conversion requires head-wise sigmoid attention gates")+        if hparams.get("hidden_act") != "gelu":+            raise ValueError(f"Spark2_5 conversion requires GELU, got {hparams.get('hidden_act')!r}")++        self.gguf_writer.add_vocab_size(hparams["vocab_size"])+        self.gguf_writer.add_sliding_window(hparams["sliding_window"])+        self.gguf_writer.add_sliding_window_pattern(+            [layer_type == "sliding_attention" for layer_type in layer_types]+        )++        head_dim = hparams["head_dim"]+        full_rope = self.rope_parameters["full_attention"]+        swa_rope = self.rope_parameters["sliding_attention"]+        self.gguf_writer.add_rope_dimension_count(+            int(head_dim * float(full_rope["partial_rotary_factor"]))+        )+        self.gguf_writer.add_rope_dimension_count_swa(+            int(head_dim * float(swa_rope["partial_rotary_factor"]))+        )++    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+        if name.endswith(".self_attn.q_k_v_proj.weight"):+            if bid is None:+                raise ValueError(f"Spark2_5 fused QKV tensor has no block id: {name}")+            yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, bid), data_torch+            return++        if name.endswith(".self_attn.g_proj.weight"):+            if bid is None:+                raise ValueError(f"Spark2_5 attention gate tensor has no block id: {name}")+            expected = self.hparams["num_attention_heads"]+            if data_torch.shape[0] != expected:+                raise ValueError(+                    f"Spark2_5 layer {bid} attention gate width {data_torch.shape[0]} != head count {expected}"+                )++        yield from super().modify_tensors(data_torch, name, bid)

@@ -191,6 +191,7 @@ class TOKENIZER_TYPE(IntEnum):     {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"},     # lfm2 variants     {"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "chkhsh": "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7"},+    {"name": "spark2_5", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/XHToken/Spark-X2.5-1.7B", "chkhsh": "0a766d034107bc736a3f2dc4968fd62e54a3570f1454443e0c5a4cc6bd7941ed"}, ]  

@@ -514,6 +514,7 @@ The following templates have active tests in `tests/test-chat.cpp`: | Mistral Small 3.2 | JSON_NATIVE | `[TOOL_CALLS]func[ARGS]{...}` with call ID | | Devstral | JSON_NATIVE | `[TOOL_CALLS]func[ARGS]{...}` without call ID | | StepFun 3.5 Flash | TAG_WITH_TAGGED | `<function=X><parameter=Y>` format |+| Spark2.5 | TAG_WITH_TAGGED | `<tool_call>name<arg_key>...<arg_value>...` format |  ## Adding Support for New Templates 

@@ -619,6 +619,7 @@ class MODEL_ARCH(IntEnum):     PADDLEOCR        = auto()     MIMO2            = auto()     STEP35           = auto()+    SPARK2_5           = auto()     LLAMA_EMBED      = auto()     MAINCODER        = auto()     KIMI_LINEAR      = auto()@@ -1373,6 +1374,7 @@ class MODEL_TENSOR(IntEnum):     MODEL_ARCH.PADDLEOCR:        "paddleocr",     MODEL_ARCH.MIMO2:            "mimo2",     MODEL_ARCH.STEP35:           "step35",+    MODEL_ARCH.SPARK2_5:         "spark2_5",     MODEL_ARCH.LLAMA_EMBED:      "llama-embed",     MODEL_ARCH.MAINCODER:        "maincoder",     MODEL_ARCH.KIMI_LINEAR:      "kimi-linear",@@ -5231,6 +5233,19 @@ class MODEL_TENSOR(IntEnum):         MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,         MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,     ],+    MODEL_ARCH.SPARK2_5: [+        MODEL_TENSOR.TOKEN_EMBD,+        MODEL_TENSOR.OUTPUT_NORM,+        MODEL_TENSOR.OUTPUT,+        MODEL_TENSOR.ATTN_NORM,+        MODEL_TENSOR.ATTN_QKV,+        MODEL_TENSOR.ATTN_GATE,+        MODEL_TENSOR.ATTN_OUT,+        MODEL_TENSOR.FFN_NORM,+        MODEL_TENSOR.FFN_GATE,+        MODEL_TENSOR.FFN_DOWN,+        MODEL_TENSOR.FFN_UP,+    ],     MODEL_ARCH.LLAMA_EMBED: [         MODEL_TENSOR.TOKEN_EMBD,         MODEL_TENSOR.OUTPUT_NORM,

@@ -23,4 +23,6 @@ These templates can be updated with the following commands: ./scripts/get_chat_template.py Qwen/Qwen3-0.6B                               > models/templates/Qwen-Qwen3-0.6B.jinja ./scripts/get_chat_template.py zai-org/GLM-4.5                               > models/templates/zai-org-GLM-4.5.jinja ./scripts/get_chat_template.py deepseek-ai/DeepSeek-V3.1                     > models/templates/deepseek-ai-DeepSeek-V3.1.jinja+./scripts/get_chat_template.py XHToken/Spark-X2.5-1.7B                       > models/templates/Spark2.5.jinja+./scripts/get_chat_template.py XHToken/Spark-X2.5-4B                         > models/templates/Spark2.5.jinja ```

@@ -0,0 +1,110 @@+{%- if not messages %}+    {{- raise_exception('No messages provided.') }}+{%- endif %}++{%- set enable_thinking = enable_thinking | default(true) %}++{#- Render a string or a list of text blocks. -#}+{%- macro render_content(content, context_name) %}+    {%- if content is string %}+        {{- content }}+    {%- elif content is none or content is undefined %}+        {{- '' }}+    {%- elif content is iterable and content is not mapping %}+        {%- for block in content %}+            {%- if block.type == 'text' %}+                {{- block.text }}+            {%- else %}+                {{- raise_exception('Unsupported ' ~ context_name ~ ' content block type: ' ~ (block.type | string)) }}+            {%- endif %}+        {%- endfor %}+    {%- else %}+        {{- raise_exception(context_name ~ ' content must be a string or a list of text blocks') }}+    {%- endif %}+{%- endmacro %}++{#- Default system prompt. -#}+{%- set default_system = 'you are a helpful assistant.' %}++{#- The first message-level system is placed in the initial system block. -#}+{%- set ns = namespace(initial_system='') %}+{%- if messages[0].role == 'system' %}+    {%- set ns.initial_system = render_content(messages[0].content, 'system') %}+{%- endif %}++{#- System block. -#}+{{- '<｜start▁of▁sentence｜><|System|>' + '\n' + default_system }}+{%- if tools %}+    {{- '## Tools' + '\n' + 'You have access to the following functions:' + '\n' + '<tools>' }}+    {%- for tool in tools %}+        {{- '\n' + tool.function | tojson }}+    {%- endfor %}+    {{- '\n' + '</tools>' }}+{%- endif %}+{%- if ns.initial_system %}+    {{- '\n\n' + ns.initial_system }}+{%- endif %}+{{- '<｜end▁of▁sentence｜>' }}++{#- Conversation turns. -#}+{%- for message in messages %}+    {%- if message.role == 'system' %}+        {#- The first system message was consumed by the initial block. -#}+        {%- if not loop.first %}+            {{- '<｜start▁of▁sentence｜><|System|>\n' + render_content(message.content, 'system') + '<｜end▁of▁sentence｜>' }}+        {%- endif %}+    {%- elif message.role == 'user' %}+        {{- '<｜start▁of▁sentence｜><|User|>' + render_content(message.content, 'user') + '<｜end▁of▁sentence｜>' }}+    {%- elif message.role == 'assistant' %}+        {%- set assistant_content = render_content(message.content, 'assistant') %}+        {%- if message.reasoning_content is defined and message.reasoning_content %}+            {%- set reasoning_content = message.reasoning_content %}+        {%- else %}+            {%- set reasoning_content = '' %}+        {%- endif %}+        {{- '<｜start▁of▁sentence｜><|Bot|>' }}+        {%- if reasoning_content %}+            {{- '<think>' + reasoning_content + '</think>' }}+        {%- else %}+            {{- '</think>' }}+        {%- endif %}+        {%- if assistant_content %}+            {{- assistant_content }}+        {%- endif %}+        {%- if message.tool_calls is defined and message.tool_calls is not none %}+            {%- for tool_call in message.tool_calls %}+                {%- if tool_call.function.arguments is not mapping %}+                    {{- raise_exception('tool_call.function.arguments must be a dictionary; normalize JSON strings before apply_chat_template') }}+                {%- endif %}+                {%- set args = tool_call.function.arguments %}+                {{- '<tool_call>' + tool_call.function.name }}+                {%- for k, v in args.items() %}+                    {{- '<arg_key>' ~ k ~ '</arg_key><arg_value>' ~ (v if v is string else v | tojson) ~ '</arg_value>' }}+                {%- endfor %}+                {{- '</tool_call>' }}+            {%- endfor %}+        {%- endif %}+        {{- '<｜end▁of▁sentence｜>' }}+    {%- elif message.role == 'tool' %}+        {%- if loop.previtem is undefined or loop.previtem.role != 'tool' %}+            {{- '<｜start▁of▁sentence｜><|Tool|>' }}+        {%- endif %}+        {{- '<tool_response>' ~ message.content ~ '</tool_response>' }}+        {%- if loop.nextitem is undefined or loop.nextitem.role != 'tool' %}+            {{- '<｜end▁of▁sentence｜>' }}+        {%- endif %}+    {%- else %}+        {{- raise_exception('Unsupported message role: ' ~ message.role) }}+    {%- endif %}+{%- endfor %}++{#- Generation prompt. -#}+{%- if add_generation_prompt %}+    {{- '<｜start▁of▁sentence｜><|Bot|>' }}+    {%- if enable_thinking is defined and enable_thinking %}+        {{- '<think>' }}+    {%- endif %}+    {%- if enable_thinking is defined and not enable_thinking %}+        {{- '</think>' }}+    {%- endif %}+{%- endif %}

@@ -146,6 +146,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {     { LLM_ARCH_PADDLEOCR,        "paddleocr"        },     { LLM_ARCH_MIMO2,            "mimo2"            },     { LLM_ARCH_STEP35,           "step35"           },+    { LLM_ARCH_SPARK2_5,         "spark2_5"         },     { LLM_ARCH_LLAMA_EMBED,      "llama-embed"      },     { LLM_ARCH_MAINCODER,        "maincoder"        },     { LLM_ARCH_KIMI_LINEAR,      "kimi-linear"      },

@@ -147,6 +147,7 @@ enum llm_arch {     LLM_ARCH_PADDLEOCR,     LLM_ARCH_MIMO2,     LLM_ARCH_STEP35,+    LLM_ARCH_SPARK2_5,     LLM_ARCH_LLAMA_EMBED,     LLM_ARCH_MAINCODER,     LLM_ARCH_KIMI_LINEAR,

@@ -27,6 +27,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) {         case LLM_ARCH_APERTUS:         case LLM_ARCH_MIMO2:         case LLM_ARCH_STEP35:+        case LLM_ARCH_SPARK2_5:         case LLM_ARCH_MUSE_GLIMMER:         case LLM_ARCH_MELLUM:         case LLM_ARCH_LAGUNA:

@@ -338,6 +338,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params             return new llama_model_kimi_k3(params);         case LLM_ARCH_STEP35:             return new llama_model_step35(params);+        case LLM_ARCH_SPARK2_5:+            return new llama_model_spark2_5(params);         default:             throw std::runtime_error(std::string("unsupported model architecture: '") + llm_arch_name(arch) + "'");     }@@ -2999,6 +3001,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {         case LLM_ARCH_QWEN3NEXT:         case LLM_ARCH_MIMO2:         case LLM_ARCH_STEP35:+        case LLM_ARCH_SPARK2_5:         case LLM_ARCH_TALKIE:         case LLM_ARCH_MELLUM:             return LLAMA_ROPE_TYPE_NEOX;

@@ -325,6 +325,14 @@ struct llm_tokenizer_bpe : llm_tokenizer {                     "[!\"#$%&'()*+,\\-./:;<=>?@\\[\\\\\\]^_`{|}~][A-Za-z]+|[^\r\n\\p{L}\\p{P}\\p{S}]?[\\p{L}\\p{M}]+| ?[\\p{P}\\p{S}]+[\r\n]*|\\s*[\r\n]+|\\s+(?!\\S)|\\s+",                 };                 break;+            case LLAMA_VOCAB_PRE_TYPE_SPARK2_5:+                regex_exprs = {+                    "\\p{N}{1,3}",+                    "[一-龥぀-ゟ゠-ヿ]+",+                    "[!\"#$%&'()*+,\\-./:;<=>?@\\[\\\\\\]^_`{|}~][A-Za-z]+|[^\r\n\\p{L}\\p{P}\\p{S}]?[\\p{L}\\p{M}]+| ?[\\p{P}\\p{S}]+|[\r\n]|\\s+(?!\\S)|\\s+",+                    "\\p{N}",+                };+                break;             case LLAMA_VOCAB_PRE_TYPE_YOUTU:                 regex_exprs = {                     "[가-힣ㄱ-ㆎ]+|[！…“”‘’—：；，、-〿︰-﹏]+|[ㄅ-ㄯ]+|[一-龥぀-ゟ゠-ヿ]+",@@ -2170,6 +2178,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {                     tokenizer_pre == "deepseek-v3") {                 pre_type = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM;                 clean_spaces = false;+            } else if (+                    tokenizer_pre == "spark2_5") {+                pre_type = LLAMA_VOCAB_PRE_TYPE_SPARK2_5;+                clean_spaces = false;             } else if (                     tokenizer_pre == "youtu") {                 pre_type = LLAMA_VOCAB_PRE_TYPE_YOUTU;

@@ -66,6 +66,7 @@ enum llama_vocab_pre_type {     LLAMA_VOCAB_PRE_TYPE_MELLUM2           = 55,     LLAMA_VOCAB_PRE_TYPE_LAGUNA            = 56,     LLAMA_VOCAB_PRE_TYPE_HY_V4             = 57,+    LLAMA_VOCAB_PRE_TYPE_SPARK2_5          = 58, };  struct LLM_KV;

@@ -2606,3 +2606,16 @@ struct llama_model_step35 : public llama_model_base {      std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override; };+++struct llama_model_spark2_5 : public llama_model_base {+    llama_model_spark2_5(const struct llama_model_params & params) : llama_model_base(params) {}+    void load_arch_hparams(llama_model_loader & ml) override;+    void load_arch_tensors(llama_model_loader & ml) override;++    struct graph : public llm_graph_context {+        graph(const llama_model & model, const llm_graph_params & params);+    };++    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;+};

@@ -0,0 +1,146 @@+#include "models.h"++void llama_model_spark2_5::load_arch_hparams(llama_model_loader & ml) {+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);+    ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);++    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;+    ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);++    hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;+    hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;+    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);++    switch (hparams.n_layer()) {+        case 28: type = LLM_TYPE_1_7B; break;+        default: type = LLM_TYPE_UNKNOWN;+    }+}++void llama_model_spark2_5::load_arch_tensors(llama_model_loader &) {+    LLAMA_LOAD_LOCALS;++    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);++    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);+    output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);+    if (output == nullptr) {+        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);+    }++    for (int i = 0; i < n_layer; ++i) {+        auto & layer = layers[i];++        const int64_t n_head_i = hparams.n_head(i);+        const int64_t n_head_kv_i = hparams.n_head_kv(i);+        const int64_t n_embd_q = hparams.n_embd_head_k(i) * n_head_i;+        const int64_t n_embd_k = hparams.n_embd_head_k(i) * n_head_kv_i;+        const int64_t n_embd_v = hparams.n_embd_head_v(i) * n_head_kv_i;++        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);+        create_tensor_qkv(layer, i, n_embd, n_embd_q, n_embd_k, n_embd_v, 0);+        layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head_i}, 0);+        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q, n_embd}, 0);++        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);+        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);+        layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);+        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);+    }+}++std::unique_ptr<llm_graph_context> llama_model_spark2_5::build_arch_graph(const llm_graph_params & params) const {+    return std::make_unique<graph>(*this, params);+}++llama_model_spark2_5::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {+    const int64_t n_embd_head = hparams.n_embd_head_v();++    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());+    GGML_ASSERT(hparams.swa_type == LLAMA_SWA_TYPE_STANDARD);++    ggml_tensor * inpL = build_inp_embd(model.tok_embd);+    ggml_tensor * inp_pos = build_inp_pos();+    auto * inp_attn = build_attn_inp_kv_iswa();+    ggml_tensor * inp_out_ids = build_inp_out_ids();++    const float kq_scale = 1.0f / sqrtf(float(n_embd_head));++    for (int il = 0; il < n_layer; ++il) {+        ggml_tensor * inpSA = inpL;+        ggml_tensor * cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);+        cb(cur, "attn_norm", il);++        const int64_t n_head_i = hparams.n_head(il);+        const int64_t n_head_kv_i = hparams.n_head_kv(il);+        const int64_t n_rot_i = hparams.n_rot(il);+        const float freq_base_i = model.get_rope_freq_base(cparams, il);+        const float freq_scale_i = model.get_rope_freq_scale(cparams, il);++        ggml_tensor * attn_inp = cur;+        auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head_i, n_head_kv_i, il);++        Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,+                n_rot_i, rope_type, n_ctx_orig, freq_base_i, freq_scale_i,+                ext_factor, attn_factor, beta_fast, beta_slow);+        Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,+                n_rot_i, rope_type, n_ctx_orig, freq_base_i, freq_scale_i,+                ext_factor, attn_factor, beta_fast, beta_slow);+        cb(Qcur, "Qcur_rope", il);+        cb(Kcur, "Kcur_rope", il);++        cur = build_attn(inp_attn,+                nullptr, nullptr, nullptr,+                Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);+        cb(cur, "attn_out", il);++        ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, attn_inp);+        gate = ggml_sigmoid(ctx0, gate);+        cb(gate, "attn_gate", il);++        const int64_t n_tokens_i = cur->ne[1];+        cur = ggml_reshape_3d(ctx0, cur, n_embd_head, n_head_i, n_tokens_i);+        gate = ggml_reshape_3d(ctx0, gate, 1, n_head_i, n_tokens_i);+        cur = ggml_mul(ctx0, cur, gate);+        cur = ggml_reshape_2d(ctx0, cur, n_embd_head * n_head_i, n_tokens_i);+        cb(cur, "attn_gated", il);++        cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);+        cb(cur, "attn_out_proj", il);++        if (il == n_layer - 1 && inp_out_ids) {+            cur = ggml_get_rows(ctx0, cur, inp_out_ids);+            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);+        }++        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);+        cb(ffn_inp, "ffn_inp", il);++        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il);+        cb(cur, "ffn_norm", il);++        cur = build_ffn(cur,+                model.layers[il].ffn_up, nullptr, nullptr,+                model.layers[il].ffn_gate, nullptr, nullptr,+                model.layers[il].ffn_down, nullptr, nullptr,+                nullptr,+                LLM_FFN_GELU, LLM_FFN_PAR, il);+        cb(cur, "ffn_out", il);++        cur = ggml_add(ctx0, cur, ffn_inp);+        cur = build_cvec(cur, il);+        cb(cur, "l_out", il);++        inpL = cur;+    }++    ggml_tensor * cur = build_norm(inpL, model.output_norm, nullptr, LLM_NORM_RMS, -1);+    cb(cur, "result_norm", -1);+    res->t_embd = cur;++    cur = build_lora_mm(model.output, cur);+    cb(cur, "result_output", -1);+    res->t_logits = cur;++    ggml_build_forward_expand(gf, cur);+}

@@ -4405,6 +4405,100 @@ static void test_template_output_peg_parsers(bool detailed_debug) {             .run();     } +    // Spark2.5 uses tagged arguments with forced-open thinking.+    {+        auto tst = peg_tester("models/templates/Spark2.5.jinja", detailed_debug);++        tst.test("Hello, world!\nWhat's up?")+            .enable_thinking(false)+            .expect(message_assist)+            .expect_reconstruction()+            .run();++        tst.test("I'm\nthinking</think>Hello, world!\nWhat's up?")+            .enable_thinking(true)+            .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)+            .expect(message_assist_thoughts)+            .expect_reconstruction()+            .run();++        tst.test(+               "<tool_call>special_function"+               "<arg_key>arg1</arg_key><arg_value>1</arg_value>"+               "</tool_call>")+            .enable_thinking(false)+            .tools({ special_function_tool })+            .expect(message_assist_call)+            .expect_reconstruction()+            .run();++        tst.test(+               "I'm\nthinking</think>"+               "<tool_call>special_function"+               "<arg_key>arg1</arg_key><arg_value>1</arg_value>"+               "</tool_call>")+            .enable_thinking(true)+            .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)+            .tools({ special_function_tool })+            .expect(message_assist_call_thoughts)+            .expect_reconstruction()+            .run();++        tst.test(+               "<tool_call>special_function"+               "<arg_key>arg1</arg_key><arg_value>1</arg_value>"+               "</tool_call>"+               "<tool_call>special_function_with_opt"+               "<arg_key>arg1</arg_key><arg_value>1</arg_value>"+               "<arg_key>arg2</arg_key><arg_value>2</arg_value>"+               "</tool_call>")+            .enable_thinking(false)+            .parallel_tool_calls(true)+            .tools({ special_function_tool, special_function_tool_with_optional_param })+            .expect_tool_calls({+                { "special_function", R"({"arg1": 1})", {} },+                { "special_function_with_opt", R"({"arg1": 1, "arg2": 2})", {} },+            })+            .expect_reconstruction()+            .run();++        tst.test(+               "Preparing updates."+               "<tool_call>magic_int"+               "<arg_key>ref</arg_key><arg_value>42</arg_value>"+               "<arg_key>name</arg_key><arg_value>上海</arg_value>"+               "</tool_call>"+               "<tool_call>amount"+               "<arg_key>orig</arg_key><arg_value>2.5</arg_value>"+               "</tool_call>"+               "<tool_call>toggle"+               "<arg_key>enabled</arg_key><arg_value>true</arg_value>"+               "</tool_call>"+               "<tool_call>set_config"+               "<arg_key>config</arg_key><arg_value>{\"source\": \"spark\", \"options\": {\"strict\": true}}</arg_value>"+               "</tool_call>"+               "<tool_call>nested_args"+               "<arg_key>tags</arg_key><arg_value>[\"alpha\", \"测试\"]</arg_value>"+               "<arg_key>entries</arg_key><arg_value>[{\"id\": 1, \"label\": \"first\"}, {\"id\": 2, \"label\": \"第二\"}]</arg_value>"+               "</tool_call>"+               "<tool_call>empty_args"+               "</tool_call>")+            .enable_thinking(false)+            .parallel_tool_calls(true)+            .tools({ magic_int_tool, amount_tool, toggle_tool, config_tool, nested_args_tool, empty_args_tool })+            .expect_content("Preparing updates.")+            .expect_tool_calls({+                { "magic_int", R"({"ref": 42, "name": "上海"})", {} },+                { "amount", R"({"orig": 2.5})", {} },+                { "toggle", R"({"enabled": true})", {} },+                { "set_config", R"({"config": {"source": "spark", "options": {"strict": true}}})", {} },+                { "nested_args", R"({"tags": ["alpha", "测试"], "entries": [{"id": 1, "label": "first"}, {"id": 2, "label": "第二"}]})", {} },+                { "empty_args", "{}", {} },+            })+            .expect_reconstruction()+            .run();+    }+     // Verify the throw path produces a readable error message, not std::out_of_range.     // #20424 introduced effective_input = generation_prompt + input, but the throw     // uses input.substr(result.end) where result.end is in effective_input space.

@@ -237,7 +237,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {         ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA,              10000.0f);         // SWA pattern: every 5th layer is full attention (matches E2B layer_types)         ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5));-    } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 ||+    } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 ||             arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE) {         std::vector<uint32_t> pattern;         pattern.reserve(n_layer);