[Model] Support Limite 1B - Violetto

@@ -176,6 +176,7 @@     "Lfm3ForCausalLM": "lfm3",     "Lfm3OmniForConditionalGeneration": "lfm3",     "Lfm3VlForConditionalGeneration": "lfm3",+    "LimiteForCausalLM": "limite",     "Llama4ForCausalLM": "llama",     "Llama4ForConditionalGeneration": "llama",     "LlamaBidirectionalModel": "llama",

@@ -0,0 +1,184 @@+from __future__ import annotations++from typing import Iterable, TYPE_CHECKING++import torch++if TYPE_CHECKING:+    from torch import Tensor++from .base import ModelBase, TextModel, gguf+++@ModelBase.register("LimiteForCausalLM")+@ModelBase.example("paradigma-inc/limite-1b-violetto")+class LimiteModel(TextModel):+    model_arch = gguf.MODEL_ARCH.LIMITE++    def __init__(self, *args, **kwargs):+        super().__init__(*args, **kwargs)+        hp = self.hparams++        expected = {+            "pos_mode": "rope",+            "qk_norm": "rms_pre_rope",+            "rope_style": "interleaved_pairs_odd_lane_sign_flip",+            "rope_per_layer": False,+            "rms_norm_has_weight": False,+            "rms_norm_eps_mode": "torch_finfo_default",+            "mlp_type": "swiglu",+            "attn_gate_applied": "per_head_before_o_proj",+            "ve_head_slice": "first_num_key_value_heads",+            "ve_applied_before_qk_norm": True,+            "global_nope": True,+            "global_window": -1,+            "mudd_r_site": "resid",+            "tie_word_embeddings": True,+            "final_softcap": 0.0,+        }+        for key, value in expected.items():+            if hp.get(key) != value:+                raise NotImplementedError(f"Limite: unsupported {key}={hp.get(key)!r}, expected {value!r}")+        if hp["softcap_logits"]["kind"] != "sigmoid":+            raise NotImplementedError(f"Limite: unsupported softcap_logits kind {hp['softcap_logits']['kind']!r}")++        self.n_head    = hp["num_attention_heads"]+        self.n_head_kv = hp["num_key_value_heads"]+        self.head_dim  = hp["head_dim"]+        self.n_pairs   = hp["rope_n_pairs"]++        self.mudd_taps: dict[int, list[int]] = {}+        if hp.get("mudd"):+            if not hp.get("mudd_mlp"):+                raise NotImplementedError("Limite: mudd without mudd_mlp is not supported")+            self.mudd_taps = {int(il): [int(t) for t in taps] for il, taps in hp["mudd_tap_idx"].items()}+            assert all(len(taps) == hp["mudd_taps"] for taps in self.mudd_taps.values())++    def set_vocab(self):+        self._set_vocab_gpt2()++    def set_gguf_parameters(self):+        super().set_gguf_parameters()+        hp = self.hparams++        self.gguf_writer.add_key_length(self.head_dim)+        self.gguf_writer.add_value_length(self.head_dim)++        # torch rms_norm with eps=None uses the fp32 epsilon, even for bf16 inputs+        self.gguf_writer.add_layer_norm_rms_eps(torch.finfo(torch.float32).eps)+        self.gguf_writer.add_attention_scale(hp["attention_softmax_scale"])+        self.gguf_writer.add_xsa_eps(hp["xsa_normalize_eps"] if hp.get("xsa") else 0.0)++        # HF freq_i = base^(-i/(n_pairs-1)), ggml uses freq_base^(-i/n_pairs)+        self.gguf_writer.add_rope_dimension_count(2 * self.n_pairs)+        self.gguf_writer.add_rope_freq_base(hp["rope_base_local"] ** (self.n_pairs / (self.n_pairs - 1)))++        # HF keeps keys k >= q - window, so a window of W spans W + 1 tokens+        global_layers = set(hp["global_layers"])+        self.gguf_writer.add_sliding_window(hp["sliding_window"] + 1)+        self.gguf_writer.add_sliding_window_pattern([il not in global_layers for il in range(self.block_count)])++        cap = hp["softcap_logits"]+        self.gguf_writer.add_final_logit_sigmoid_capping(cap["a"], cap["b"], cap["c"])++        if self.mudd_taps:+            n_taps = hp["mudd_taps"]+            tap_indices = [-1] * (self.block_count * n_taps)+            for il, taps in self.mudd_taps.items():+                tap_indices[il * n_taps:(il + 1) * n_taps] = taps+            self.gguf_writer.add_mudd_feed_forward_length(hp["mudd_inter"])+            self.gguf_writer.add_mudd_tap_count(n_taps)+            self.gguf_writer.add_mudd_tap_indices(tap_indices)++    def tensor_force_quant(self, name, new_name, bid, n_dims):+        # tiny mixing weights, kept at full precision as in the reference+        # the mudd tensors come from a global source tensor, so take the block id from the new name+        if new_name.startswith("blk."):+            bid = int(new_name.split(".")[1])+        for key in (gguf.MODEL_TENSOR.ATTN_GATE, gguf.MODEL_TENSOR.ATTN_VE_GATE, gguf.MODEL_TENSOR.MUDD_DOWN, gguf.MODEL_TENSOR.MUDD_UP):+            if self.match_model_tensor_name(new_name, key, bid):+                return gguf.GGMLQuantizationType.F32+        return super().tensor_force_quant(name, new_name, bid, n_dims)++    def _swap_rope_pairs(self, w: Tensor, n_head: int) -> Tensor:+        # HF rotates each pair by -theta; swapping the pair lanes gives the ggml +theta rotation.+        # q and k are both swapped, so the attention scores do not change.+        idx = torch.arange(self.head_dim)+        idx[:2 * self.n_pairs] = idx[:2 * self.n_pairs].view(-1, 2).flip(-1).reshape(-1)+        w = w.reshape(n_head, self.head_dim, -1)+        return w[:, idx, :].reshape(n_head * self.head_dim, -1)++    def _pad_cols(self, w: Tensor) -> Tensor:+        # gates read only the first channels of the input, pad them to n_embd+        return torch.nn.functional.pad(w, (0, self.hparams["hidden_size"] - w.shape[-1]))++    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+        hp = self.hparams+        T = gguf.MODEL_TENSOR++        def load(n: str) -> Tensor:+            return self.model_tensors[n]().float()++        if name == "model.value_embeds.weight":+            ve = data_torch.reshape(data_torch.shape[0], -1, hp["ve_dim"])[:, :self.n_head_kv]+            ve = torch.nn.functional.pad(ve, (0, self.head_dim - hp["ve_dim"]))+            yield self.format_tensor_name(T.VALUE_EMBD), ve.reshape(ve.shape[0], -1) * hp["ve_gate_scale"]+            return++        if name == "model.mudd.dense1":+            for il in self.mudd_taps:+                yield self.format_tensor_name(T.MUDD_DOWN, il), data_torch+            return+        if name == "model.mudd.dense2":+            # rows [0, n) mix the attention input, rows [n, 2n) mix the residual+            dense2_mlp = load("model.mudd.dense2_mlp")+            bias       = load("model.mudd.bias")+            bias_mlp   = load("model.mudd.bias_mlp")+            for il, taps in self.mudd_taps.items():+                n = len(taps)+                w = torch.cat([data_torch[il, :n], dense2_mlp[il, :n]], 0)+                b = torch.cat([bias[il, :n], bias_mlp[il, :n]], 0)+                yield self.format_tensor_name(T.MUDD_UP, il), w+                yield self.format_tensor_name(T.MUDD_UP, il, ".bias"), b+            return+        if name.startswith("model.mudd."):+            return++        if bid is None:+            yield from super().modify_tensors(data_torch, name, bid)+            return++        prefix = f"model.layers.{bid}."+        name = name.removeprefix(prefix)++        if name in ("self_attn.q_proj.weight", "self_attn.k_proj.weight", "self_attn.v_proj.weight"):+            data_torch = data_torch * load(prefix + "self_attn.qkv_scale")+            if name == "self_attn.q_proj.weight":+                data_torch = self._swap_rope_pairs(data_torch, self.n_head)+            elif name == "self_attn.k_proj.weight":+                data_torch = self._swap_rope_pairs(data_torch, self.n_head_kv)+        elif name == "self_attn.o_proj.weight":+            data_torch = data_torch * load(prefix + "self_attn.o_scale") * load(prefix + "post_lambda_attn") * hp["attn_gate_scale"]+        elif name == "mlp.down_proj.weight":+            data_torch = data_torch * load(prefix + "post_lambda_mlp")+        elif name == "self_attn.attn_gate":+            yield self.format_tensor_name(T.ATTN_GATE, bid), self._pad_cols(data_torch)+            return+        elif name == "self_attn.ve_gate":+            yield self.format_tensor_name(T.ATTN_VE_GATE, bid), self._pad_cols(data_torch[:self.n_head_kv])+            return+        elif name == "self_attn.xsa_alpha":+            if hp.get("xsa") and bid in hp["xsa_layers"]:+                yield self.format_tensor_name(T.ATTN_XSA_ALPHA, bid), torch.tanh(data_torch)+            return+        elif name == "resid_lambda_attn":+            yield self.format_tensor_name(T.ATTN_RESID_SCALE, bid), data_torch.reshape(1)+            return+        elif name == "resid_lambda_mlp":+            yield self.format_tensor_name(T.FFN_RESID_SCALE, bid), data_torch.reshape(1)+            return+        elif name in ("self_attn.qkv_scale", "self_attn.o_scale", "post_lambda_attn", "post_lambda_mlp"):+            # folded into the projections above+            return++        yield from super().modify_tensors(data_torch, prefix + name, bid)

@@ -143,6 +143,7 @@ class LLM:         ATTN_LOGIT_SOFTCAPPING            = "{arch}.attn_logit_softcapping"         ROUTER_LOGIT_SOFTCAPPING          = "{arch}.router_logit_softcapping"         FINAL_LOGIT_SOFTCAPPING           = "{arch}.final_logit_softcapping"+        FINAL_LOGIT_SIGMOID_CAPPING       = "{arch}.final_logit_sigmoid_capping"         SWIN_NORM                         = "{arch}.swin_norm"         RESCALE_EVERY_N_LAYERS            = "{arch}.rescale_every_n_layers"         TIME_MIX_EXTRA_DIM                = "{arch}.time_mix_extra_dim"@@ -227,6 +228,7 @@ class Attention:         ROPE_PATTERN                 = "{arch}.attention.rope_pattern"         VALUE_EXPERT_COUNT           = "{arch}.attention.value_expert_count"         VALUE_EXPERT_USED_COUNT      = "{arch}.attention.value_expert_used_count"+        XSA_EPS                      = "{arch}.attention.xsa_epsilon"          class Indexer:             HEAD_COUNT = "{arch}.attention.indexer.head_count"@@ -301,6 +303,11 @@ class Split:         LLM_KV_SPLIT_COUNT         = "split.count"         LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count" +    class MUDD:+        FEED_FORWARD_LENGTH = "{arch}.mudd.feed_forward_length"+        TAP_COUNT           = "{arch}.mudd.tap_count"+        TAP_INDICES         = "{arch}.mudd.tap_indices"+     class HRM:         LAYERS_PER_STACK = "{arch}.hrm.layers_per_stack"         H_CYCLES         = "{arch}.hrm.h_cycles"@@ -731,6 +738,7 @@ class MODEL_ARCH(IntEnum):     POCKETTTS        = auto()     K2HORIZON        = auto()     KOLIBRI1         = auto()+    LIMITE           = auto()   class VISION_PROJECTOR_TYPE(IntEnum):@@ -865,6 +873,13 @@ class MODEL_TENSOR(IntEnum):     FFN_ROUTED_DOWN      = auto() # Kimi K3 (latent MoE: hidden -> latent)     FFN_ROUTED_UP        = auto() # Kimi K3 (latent MoE: latent -> hidden)     FFN_ROUTED_NORM      = auto() # Kimi K3 (latent MoE: norm on expert output)+    VALUE_EMBD           = auto() # limite+    ATTN_VE_GATE         = auto() # limite+    ATTN_XSA_ALPHA       = auto() # limite+    ATTN_RESID_SCALE     = auto() # limite+    FFN_RESID_SCALE      = auto() # limite+    MUDD_DOWN            = auto() # limite+    MUDD_UP              = auto() # limite     TIME_MIX_W0          = auto()     TIME_MIX_W1          = auto()     TIME_MIX_W2          = auto()@@ -1577,6 +1592,7 @@ class MODEL_TENSOR(IntEnum):     MODEL_ARCH.POCKETTTS:        "pockettts",     MODEL_ARCH.K2HORIZON:        "k2-horizon",     MODEL_ARCH.KOLIBRI1:         "kolibri1",+    MODEL_ARCH.LIMITE:           "limite", }  VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = {@@ -1709,6 +1725,13 @@ class MODEL_TENSOR(IntEnum):     MODEL_TENSOR.FFN_ROUTED_DOWN:           "blk.{bid}.ffn_routed_down",      # Kimi K3     MODEL_TENSOR.FFN_ROUTED_UP:             "blk.{bid}.ffn_routed_up",        # Kimi K3     MODEL_TENSOR.FFN_ROUTED_NORM:           "blk.{bid}.ffn_routed_norm",      # Kimi K3+    MODEL_TENSOR.VALUE_EMBD:                "value_embd",                     # limite+    MODEL_TENSOR.ATTN_VE_GATE:              "blk.{bid}.attn_ve_gate",         # limite+    MODEL_TENSOR.ATTN_XSA_ALPHA:            "blk.{bid}.attn_xsa_alpha",       # limite+    MODEL_TENSOR.ATTN_RESID_SCALE:          "blk.{bid}.attn_resid_scale",     # limite+    MODEL_TENSOR.FFN_RESID_SCALE:           "blk.{bid}.ffn_resid_scale",      # limite+    MODEL_TENSOR.MUDD_DOWN:                 "blk.{bid}.mudd_down",            # limite+    MODEL_TENSOR.MUDD_UP:                   "blk.{bid}.mudd_up",              # limite     MODEL_TENSOR.TIME_MIX_W0:               "blk.{bid}.time_mix_w0",     MODEL_TENSOR.TIME_MIX_W1:               "blk.{bid}.time_mix_w1",     MODEL_TENSOR.TIME_MIX_W2:               "blk.{bid}.time_mix_w2",@@ -6288,6 +6311,25 @@ class MODEL_TENSOR(IntEnum):         MODEL_TENSOR.FFN_UP_SHEXP,         MODEL_TENSOR.FFN_DOWN_SHEXP,     ],+    MODEL_ARCH.LIMITE: [+        MODEL_TENSOR.TOKEN_EMBD,+        MODEL_TENSOR.VALUE_EMBD,+        MODEL_TENSOR.OUTPUT,+        MODEL_TENSOR.ATTN_Q,+        MODEL_TENSOR.ATTN_K,+        MODEL_TENSOR.ATTN_V,+        MODEL_TENSOR.ATTN_OUT,+        MODEL_TENSOR.ATTN_GATE,+        MODEL_TENSOR.ATTN_VE_GATE,+        MODEL_TENSOR.ATTN_XSA_ALPHA,+        MODEL_TENSOR.ATTN_RESID_SCALE,+        MODEL_TENSOR.FFN_RESID_SCALE,+        MODEL_TENSOR.FFN_GATE,+        MODEL_TENSOR.FFN_DOWN,+        MODEL_TENSOR.FFN_UP,+        MODEL_TENSOR.MUDD_DOWN,+        MODEL_TENSOR.MUDD_UP,+    ], }  # tensors that will not be serialized

@@ -876,6 +876,9 @@ def add_router_logit_softcapping(self, value: float) -> None:     def add_final_logit_softcapping(self, value: float) -> None:         self.add_float32(Keys.LLM.FINAL_LOGIT_SOFTCAPPING.format(arch=self.arch), value) +    def add_final_logit_sigmoid_capping(self, scale: float, shift: float, temperature: float) -> None:+        self.add_array(Keys.LLM.FINAL_LOGIT_SIGMOID_CAPPING.format(arch=self.arch), [scale, shift, temperature])+     def add_expert_count(self, count: int) -> None:         self.add_uint32(Keys.LLM.EXPERT_COUNT.format(arch=self.arch), count) @@ -948,6 +951,15 @@ def add_residual_scale(self, value: float) -> None:     def add_embedding_scale(self, value: float) -> None:         self.add_float32(Keys.LLM.EMBEDDING_SCALE.format(arch=self.arch), value) +    def add_mudd_feed_forward_length(self, value: int) -> None:+        self.add_uint32(Keys.MUDD.FEED_FORWARD_LENGTH.format(arch=self.arch), value)++    def add_mudd_tap_count(self, value: int) -> None:+        self.add_uint32(Keys.MUDD.TAP_COUNT.format(arch=self.arch), value)++    def add_mudd_tap_indices(self, value: Sequence[int]) -> None:+        self.add_array(Keys.MUDD.TAP_INDICES.format(arch=self.arch), value)+     def add_hrm_layers_per_stack(self, value: int) -> None:         self.add_uint32(Keys.HRM.LAYERS_PER_STACK.format(arch=self.arch), value) @@ -1151,6 +1163,9 @@ def add_ple_image_token_id(self, value: int) -> None:     def add_attention_scale(self, value: float) -> None:         self.add_float32(Keys.Attention.SCALE.format(arch=self.arch), value) +    def add_xsa_eps(self, value: float) -> None:+        self.add_float32(Keys.Attention.XSA_EPS.format(arch=self.arch), value)+     def add_attn_output_scale(self, value: float) -> None:         self.add_float32(Keys.Attention.OUTPUT_SCALE.format(arch=self.arch), value) 

@@ -145,6 +145,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {     { LLM_ARCH_APERTUS,          "apertus"          },     { LLM_ARCH_MINIMAX_01,       "minimax-01"       },     { LLM_ARCH_HRM_TEXT,         "hrm_text"         },+    { LLM_ARCH_LIMITE,           "limite"           },     { LLM_ARCH_MINIMAX_M2,       "minimax-m2"       },     { LLM_ARCH_MINIMAX_M3,       "minimax-m3"       },     { LLM_ARCH_COGVLM,           "cogvlm"           },@@ -248,6 +249,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {     { LLM_KV_ATTN_LOGIT_SOFTCAPPING,            "%s.attn_logit_softcapping"            },     { LLM_KV_ROUTER_LOGIT_SOFTCAPPING,          "%s.router_logit_softcapping"          },     { LLM_KV_FINAL_LOGIT_SOFTCAPPING,           "%s.final_logit_softcapping"           },+    { LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING,       "%s.final_logit_sigmoid_capping"       },     { LLM_KV_SWIN_NORM,                         "%s.swin_norm"                         },     { LLM_KV_RESCALE_EVERY_N_LAYERS,            "%s.rescale_every_n_layers"            },     { LLM_KV_TIME_MIX_EXTRA_DIM,                "%s.time_mix_extra_dim"                },@@ -268,6 +270,9 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {     { LLM_KV_HRM_H_CYCLES,                      "%s.hrm.h_cycles"                      },     { LLM_KV_HRM_L_CYCLES,                      "%s.hrm.l_cycles"                      },     { LLM_KV_HRM_PREFIX_LM,                     "%s.hrm.prefix_lm"                     },+    { LLM_KV_MUDD_FEED_FORWARD_LENGTH,          "%s.mudd.feed_forward_length"          },+    { LLM_KV_MUDD_TAP_COUNT,                    "%s.mudd.tap_count"                    },+    { LLM_KV_MUDD_TAP_INDICES,                  "%s.mudd.tap_indices"                  },      { LLM_KV_ATTENTION_HEAD_COUNT,                   "%s.attention.head_count"                   },     { LLM_KV_ATTENTION_HEAD_COUNT_KV,                "%s.attention.head_count_kv"                },@@ -292,6 +297,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {     { LLM_KV_ATTENTION_ROPE_PATTERN,                 "%s.attention.rope_pattern"                 },     { LLM_KV_ATTENTION_VALUE_EXPERT_COUNT,           "%s.attention.value_expert_count"           },     { LLM_KV_ATTENTION_VALUE_EXPERT_USED_COUNT,      "%s.attention.value_expert_used_count"      },+    { LLM_KV_ATTENTION_XSA_EPS,                      "%s.attention.xsa_epsilon"                  },      { LLM_KV_ATTENTION_SCALE,                        "%s.attention.scale"                        },     { LLM_KV_ATTENTION_OUTPUT_SCALE,                 "%s.attention.output_scale"                 },@@ -605,6 +611,13 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {     { LLM_TENSOR_FFN_ROUTED_DOWN,                        "blk.%d.ffn_routed_down" },     { LLM_TENSOR_FFN_ROUTED_UP,                          "blk.%d.ffn_routed_up" },     { LLM_TENSOR_FFN_ROUTED_NORM,                        "blk.%d.ffn_routed_norm" },+    { LLM_TENSOR_VALUE_EMBD,                             "value_embd" },+    { LLM_TENSOR_ATTN_VE_GATE,                           "blk.%d.attn_ve_gate" },+    { LLM_TENSOR_ATTN_XSA_ALPHA,                         "blk.%d.attn_xsa_alpha" },+    { LLM_TENSOR_ATTN_RESID_SCALE,                       "blk.%d.attn_resid_scale" },+    { LLM_TENSOR_FFN_RESID_SCALE,                        "blk.%d.ffn_resid_scale" },+    { LLM_TENSOR_MUDD_DOWN,                              "blk.%d.mudd_down" },+    { LLM_TENSOR_MUDD_UP,                                "blk.%d.mudd_up" },     { LLM_TENSOR_SSM_G_B,                                "blk.%d.ssm_g_b" },     { LLM_TENSOR_SSM_NORM,                               "blk.%d.ssm_norm" },     { LLM_TENSOR_ATTN_Q_A_NORM,                          "blk.%d.attn_q_a_norm" },@@ -998,6 +1011,13 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {     {LLM_TENSOR_FFN_ROUTED_DOWN,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},     {LLM_TENSOR_FFN_ROUTED_UP,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},     {LLM_TENSOR_FFN_ROUTED_NORM,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},+    {LLM_TENSOR_VALUE_EMBD,                 {LLM_TENSOR_LAYER_INPUT,     GGML_OP_GET_ROWS}},+    {LLM_TENSOR_ATTN_VE_GATE,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_ATTN_XSA_ALPHA,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},+    {LLM_TENSOR_ATTN_RESID_SCALE,           {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},+    {LLM_TENSOR_FFN_RESID_SCALE,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},+    {LLM_TENSOR_MUDD_DOWN,                  {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_MUDD_UP,                    {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},     {LLM_TENSOR_SSM_G_B,                    {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},     {LLM_TENSOR_TIME_MIX_LERP_X,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},     {LLM_TENSOR_TIME_MIX_LN,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},@@ -1315,6 +1335,7 @@ bool llm_arch_supports_mixed_batch(const llm_arch & arch) {         case LLM_ARCH_EAGLE3:         case LLM_ARCH_DFLASH:         case LLM_ARCH_GEMMA4_ASSISTANT:+        case LLM_ARCH_LIMITE:             return false;         default:             return true;@@ -1357,6 +1378,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {         case LLM_ARCH_KIMI_K3:         case LLM_ARCH_GLM5_NEXT:         case LLM_ARCH_QWEN3TTS:+        case LLM_ARCH_LIMITE:             return false;         default:             return true;

@@ -176,6 +176,7 @@ enum llm_arch {     LLM_ARCH_K2_HORIZON,     LLM_ARCH_BEAM,     LLM_ARCH_KOLIBRI1,+    LLM_ARCH_LIMITE,     LLM_ARCH_UNKNOWN, }; @@ -252,6 +253,7 @@ enum llm_kv {     LLM_KV_ATTN_LOGIT_SOFTCAPPING,     LLM_KV_ROUTER_LOGIT_SOFTCAPPING,     LLM_KV_FINAL_LOGIT_SOFTCAPPING,+    LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING,     LLM_KV_SWIN_NORM,     LLM_KV_RESCALE_EVERY_N_LAYERS,     LLM_KV_TIME_MIX_EXTRA_DIM,@@ -273,6 +275,9 @@ enum llm_kv {     LLM_KV_HRM_L_CYCLES,     LLM_KV_HRM_PREFIX_LM,     LLM_KV_REALTIME_DELAY_TOKENS,+    LLM_KV_MUDD_FEED_FORWARD_LENGTH,+    LLM_KV_MUDD_TAP_COUNT,+    LLM_KV_MUDD_TAP_INDICES,      LLM_KV_ATTENTION_HEAD_COUNT,     LLM_KV_ATTENTION_HEAD_COUNT_KV,@@ -298,6 +303,7 @@ enum llm_kv {     LLM_KV_ATTENTION_ROPE_PATTERN,     LLM_KV_ATTENTION_VALUE_EXPERT_COUNT,     LLM_KV_ATTENTION_VALUE_EXPERT_USED_COUNT,+    LLM_KV_ATTENTION_XSA_EPS,      LLM_KV_ATTENTION_OUTPUT_SCALE,     LLM_KV_ATTENTION_VALUE_SCALE,@@ -612,6 +618,13 @@ enum llm_tensor {     LLM_TENSOR_FFN_ROUTED_DOWN,     // kimi-k3: latent MoE down     LLM_TENSOR_FFN_ROUTED_UP,       // kimi-k3: latent MoE up     LLM_TENSOR_FFN_ROUTED_NORM,     // kimi-k3: latent MoE norm+    LLM_TENSOR_VALUE_EMBD,          // limite: value embeddings+    LLM_TENSOR_ATTN_VE_GATE,        // limite: value embedding gate+    LLM_TENSOR_ATTN_XSA_ALPHA,      // limite: exclusive self attention+    LLM_TENSOR_ATTN_RESID_SCALE,    // limite: residual scale (attn)+    LLM_TENSOR_FFN_RESID_SCALE,     // limite: residual scale (ffn)+    LLM_TENSOR_MUDD_DOWN,           // limite: dense connection mixer+    LLM_TENSOR_MUDD_UP,             // limite: dense connection mixer     LLM_TENSOR_TIME_MIX_W0,     LLM_TENSOR_TIME_MIX_W1,     LLM_TENSOR_TIME_MIX_W2,

@@ -227,6 +227,13 @@ struct llama_hparams {     uint32_t n_hrm_l_cycles = 0;     bool     hrm_prefix_lm = false; +    // limite+    float    f_xsa_eps = 0.0f;+    std::array<float, 3> final_logit_sigmoid_capping = {}; // scale, shift, temperature+    uint32_t n_mudd_ff   = 0;+    uint32_t n_mudd_taps = 0;+    std::array<int32_t, LLAMA_MAX_LAYERS> mudd_tap_idx; // [n_layer][n_mudd_taps], -1 = no mixing+     bool ssm_dt_b_c_rms = false;      float f_clamp_kqv      = 0.0f;

@@ -411,6 +411,7 @@ namespace GGUFMeta {      template bool llama_model_loader::get_arr<std::vector<std::string>>(enum llm_kv kid, std::vector<std::string> & result, bool required);     template bool llama_model_loader::get_arr<std::array<int32_t, 512>>(enum llm_kv kid, std::array<int32_t, 512> & result, bool required);+    template bool llama_model_loader::get_arr<std::array<float, 3>>(enum llm_kv kid, std::array<float, 3> & result, bool required);     template bool llama_model_loader::get_arr<std::vector<int32_t>>(enum llm_kv kid, std::vector<int32_t> & result, bool required);     template bool llama_model_loader::get_arr<std::array<uint32_t, LLAMA_MAX_LAYERS>>(enum llm_kv kid, std::array<uint32_t, LLAMA_MAX_LAYERS> & result, bool required);     template bool llama_model_loader::get_arr<std::vector<uint32_t>>(enum llm_kv kid, std::vector<uint32_t> & result, bool required);

@@ -244,6 +244,16 @@ void llama_model_saver::add_kv_from_model() {     add_kv(LLM_KV_HRM_H_CYCLES,                      hparams.n_hrm_h_cycles);     add_kv(LLM_KV_HRM_L_CYCLES,                      hparams.n_hrm_l_cycles);     add_kv(LLM_KV_HRM_PREFIX_LM,                     hparams.hrm_prefix_lm);+    if (hparams.final_logit_sigmoid_capping[0] != 0.0f) {+        add_kv(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, std::vector<float>(+                hparams.final_logit_sigmoid_capping.begin(), hparams.final_logit_sigmoid_capping.end()));+    }+    add_kv(LLM_KV_MUDD_FEED_FORWARD_LENGTH,          hparams.n_mudd_ff);+    add_kv(LLM_KV_MUDD_TAP_COUNT,                    hparams.n_mudd_taps);+    if (hparams.n_mudd_taps > 0) {+        add_kv(LLM_KV_MUDD_TAP_INDICES, std::vector<int32_t>(+                hparams.mudd_tap_idx.begin(), hparams.mudd_tap_idx.begin() + hparams.n_layer()*hparams.n_mudd_taps));+    }     add_kv(LLM_KV_TOKEN_SHIFT_COUNT,                 hparams.token_shift_count);     add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP,         hparams.n_moe_layer_step);     // add_kv(LLM_KV_FULL_ATTENTION_INTERVAL,           ???); // saved as LLM_KV_ATTENTION_RECURRENT_LAYERS instead@@ -277,6 +287,7 @@ void llama_model_saver::add_kv_from_model() {                 hparams.is_swa_impl.begin(), hparams.is_swa_impl.begin() + hparams.n_layer_all));     }     add_kv(LLM_KV_ATTENTION_SCALE,                   hparams.f_attention_scale);+    add_kv(LLM_KV_ATTENTION_XSA_EPS,                 hparams.f_xsa_eps);     add_kv(LLM_KV_ATTENTION_OUTPUT_SCALE,            hparams.f_attn_out_scale);     add_kv(LLM_KV_ATTENTION_VALUE_SCALE,             hparams.f_attn_value_scale);     add_kv(LLM_KV_ATTENTION_TEMPERATURE_LENGTH,      hparams.attn_temp_length);@@ -501,6 +512,7 @@ void llama_model_saver::add_tensors_from_model() {     add_tensor(model->hc_head_base);     add_tensor(model->hc_head_scale);     add_tensor(model->per_layer_tok_embd);+    add_tensor(model->value_embd);     add_tensor(model->hc_head_norm);     add_tensor(model->hc_head_down);     add_tensor(model->hc_head_up);

@@ -332,6 +332,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params             return new llama_model_hrm_text(params);         case LLM_ARCH_BEAM:             return new llama_model_beam(params);+        case LLM_ARCH_LIMITE:+            return new llama_model_limite(params);         case LLM_ARCH_COGVLM:             return new llama_model_cogvlm(params);         case LLM_ARCH_PANGU_EMBED:@@ -3575,6 +3577,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {         case LLM_ARCH_DOTS3NOTE:         case LLM_ARCH_NANBEIGE:         case LLM_ARCH_POCKETTTS:+        case LLM_ARCH_LIMITE:             return LLAMA_ROPE_TYPE_NORM;         case LLM_ARCH_BAILINGMOE3:             // VL files carry mrope sections; text-only files keep NORM rope

@@ -638,6 +638,15 @@ struct llama_layer {     // gemma4 layer output scale, reused for talkie embedding skip scale     struct ggml_tensor * out_scale = nullptr; +    // limite+    struct ggml_tensor * attn_ve_gate     = nullptr;+    struct ggml_tensor * attn_xsa_alpha   = nullptr;+    struct ggml_tensor * attn_resid_scale = nullptr;+    struct ggml_tensor * ffn_resid_scale  = nullptr;+    struct ggml_tensor * mudd_down        = nullptr;+    struct ggml_tensor * mudd_up          = nullptr;+    struct ggml_tensor * mudd_up_b        = nullptr;+     struct llama_layer_posnet posnet;      struct llama_layer_convnext convnext;@@ -783,6 +792,9 @@ struct llama_model {     struct ggml_tensor * altup_unembd_proj    = nullptr;     struct ggml_tensor * per_layer_tok_embd   = nullptr; +    // limite value embeddings+    struct ggml_tensor * value_embd = nullptr;+     struct ggml_tensor * hc_head_norm = nullptr;     struct ggml_tensor * hc_head_down = nullptr;     struct ggml_tensor * hc_head_up   = nullptr;

@@ -333,6 +333,13 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param     // these are not too big so keep them as it is     quantize &= name.find("per_layer_model_proj") == std::string::npos; +    // limite: very small gates and dense connection mixers+    if (arch == LLM_ARCH_LIMITE) {+        quantize &= name.find("attn_gate.weight")    == std::string::npos;+        quantize &= name.find("attn_ve_gate.weight") == std::string::npos;+        quantize &= name.find("mudd_")               == std::string::npos;+    }+     // do not quantize positional embeddings and token types (BERT)     quantize &= name != LLM_TN(arch)(LLM_TENSOR_POS_EMBD,    "weight");     quantize &= name != LLM_TN(arch)(LLM_TENSOR_TOKEN_TYPES, "weight");

@@ -0,0 +1,252 @@+#include "models.h"++// Limite: pre-norm transformer with value embeddings, exclusive self attention (XSA),+// per-head output gates, learned residual scales and dense (MUDD) cross-layer connections.+// ref: https://github.com/paradigma-inc/limite-violetto++void llama_model_limite::load_arch_hparams(llama_model_loader & ml) {+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);+    ml.get_key(LLM_KV_ATTENTION_SCALE,             hparams.f_attention_scale);+    ml.get_key(LLM_KV_ATTENTION_XSA_EPS,           hparams.f_xsa_eps, false);+    ml.get_arr(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, hparams.final_logit_sigmoid_capping, false);++    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;+    ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);+    load_swa_pattern(ml, 4);++    std::fill(hparams.mudd_tap_idx.begin(), hparams.mudd_tap_idx.end(), -1);+    ml.get_key(LLM_KV_MUDD_TAP_COUNT, hparams.n_mudd_taps, false);+    if (hparams.n_mudd_taps > 0) {+        ml.get_key(LLM_KV_MUDD_FEED_FORWARD_LENGTH, hparams.n_mudd_ff);+        ml.get_arr(LLM_KV_MUDD_TAP_INDICES,         hparams.mudd_tap_idx);+    }++    switch (hparams.n_layer()) {+        case 48: type = LLM_TYPE_1B; break;+        default: type = LLM_TYPE_UNKNOWN;+    }+}++void llama_model_limite::load_arch_tensors(llama_model_loader &) {+    LLAMA_LOAD_LOCALS;++    tok_embd   = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd,       n_vocab}, 0);+    value_embd = create_tensor(tn(LLM_TENSOR_VALUE_EMBD, "weight"), {n_embd_v_gqa, n_vocab}, 0);++    output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);+    // if output is NULL, init from the input tok embed+    if (output == NULL) {+        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);+    }++    const int64_t n_taps = hparams.n_mudd_taps;+    const int64_t n_mudd = hparams.n_mudd_ff;++    for (int i = 0; i < n_layer; ++i) {+        auto & layer = layers[i];++        create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);+        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);++        layer.wqkv_gate      = create_tensor(tn(LLM_TENSOR_ATTN_GATE,      "weight", i), {n_embd, n_head},    TENSOR_NOT_REQUIRED);+        layer.attn_ve_gate   = create_tensor(tn(LLM_TENSOR_ATTN_VE_GATE,   "weight", i), {n_embd, n_head_kv}, TENSOR_NOT_REQUIRED);+        layer.attn_xsa_alpha = create_tensor(tn(LLM_TENSOR_ATTN_XSA_ALPHA, "weight", i), {n_head},            TENSOR_NOT_REQUIRED);++        layer.attn_resid_scale = create_tensor(tn(LLM_TENSOR_ATTN_RESID_SCALE, "weight", i), {1}, 0);+        layer.ffn_resid_scale  = create_tensor(tn(LLM_TENSOR_FFN_RESID_SCALE,  "weight", i), {1}, 0);++        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);+        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);+        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd, n_ff}, 0);++        if (n_taps > 0 && hparams.mudd_tap_idx[i*n_taps] >= 0) {+            // rows [0, n_taps) mix the attention input, rows [n_taps, 2*n_taps) mix the residual+            layer.mudd_down = create_tensor(tn(LLM_TENSOR_MUDD_DOWN, "weight", i), {n_embd, n_mudd},   0);+            layer.mudd_up   = create_tensor(tn(LLM_TENSOR_MUDD_UP,   "weight", i), {n_mudd, 2*n_taps}, 0);+            layer.mudd_up_b = create_tensor(tn(LLM_TENSOR_MUDD_UP,   "bias",   i), {2*n_taps},         0);+        }+    }+}++std::unique_ptr<llm_graph_context> llama_model_limite::build_arch_graph(const llm_graph_params & params) const {+    return std::make_unique<graph>(*this, params);+}++llama_model_limite::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {+    const int64_t n_embd_head = hparams.n_embd_head_k();+    const int64_t n_taps      = hparams.n_mudd_taps;++    GGML_ASSERT(n_embd_head == hparams.n_embd_head_v());++    ggml_tensor * cur;+    ggml_tensor * inpL;++    inpL = build_inp_embd(model.tok_embd);++    // value embeddings are looked up by token id, embeddings input is not supported+    ggml_tensor * inp_ve = ggml_get_rows(ctx0, model.value_embd, res->t_inp_tokens);+    inp_ve = ggml_reshape_3d(ctx0, inp_ve, n_embd_head, n_head_kv, n_tokens);+    cb(inp_ve, "inp_ve", -1);++    inpL = build_norm(inpL, nullptr, nullptr, LLM_NORM_RMS, -1);+    cb(inpL, "inp_norm", -1);++    ggml_tensor * inp_pos = build_inp_pos();++    auto * inp_attn = build_attn_inp_kv_iswa();++    ggml_tensor * inp_out_ids = build_inp_out_ids();++    // hist[il] is the input of layer il, read by the dense connections+    std::vector<ggml_tensor *> hist(n_layer, nullptr);++    for (int il = 0; il < n_layer; ++il) {+        const auto & layer = model.layers[il];++        hist[il] = inpL;++        ggml_tensor * resid = inpL;++        if (layer.mudd_up) {+            const int32_t * taps = hparams.mudd_tap_idx.data() + il*n_taps;++            ggml_tensor * w = build_norm(inpL, nullptr, nullptr, LLM_NORM_RMS, il);+            w = ggml_gelu_erf(ctx0, build_lora_mm(layer.mudd_down, w));+            w = ggml_add(ctx0, build_lora_mm(layer.mudd_up, w), layer.mudd_up_b);+            cb(w, "mudd_w", il);++            ggml_tensor * stack = nullptr;+            for (int64_t j = 0; j < n_taps; ++j) {+                GGML_ASSERT(taps[j] >= 0 && taps[j] <= il);+                ggml_tensor * h = ggml_reshape_3d(ctx0, hist[taps[j]], n_embd, 1, n_tokens);+                stack = stack ? ggml_concat(ctx0, stack, h, 1) : h;+            }++            auto mix = [&](size_t offset) {+                if (n_taps == 4) {+                    return ggml_dsv4_hc_pre(ctx0, stack, ggml_cont(ctx0, ggml_view_2d(ctx0, w, n_taps, n_tokens, w->nb[1], offset)));+                }+                ggml_tensor * wj = ggml_view_3d(ctx0, w, 1, n_taps, n_tokens, w->nb[0], w->nb[1], offset);+                ggml_tensor * m  = ggml_cont(ctx0, ggml_permute(ctx0, ggml_mul(ctx0, stack, wj), 1, 0, 2, 3));+                return ggml_reshape_2d(ctx0, ggml_sum_rows(ctx0, m), n_embd, n_tokens);+            };++            cur   = mix(0);+            resid = mix(n_taps*w->nb[0]);+            cb(cur,   "mudd_attn",  il);+            cb(resid, "mudd_resid", il);+        } else {+            cur = inpL;+        }++        ggml_tensor * attn_in = build_norm(cur, nullptr, nullptr, LLM_NORM_RMS, il);+        cb(attn_in, "attn_norm", il);++        // self-attention+        {+            auto [Qcur, Kcur, Vcur] = build_qkv(layer, attn_in, n_embd_head, n_head, n_head_kv, il);++            if (layer.attn_ve_gate) {+                ggml_tensor * gate = ggml_sigmoid(ctx0, build_lora_mm(layer.attn_ve_gate, attn_in));+                gate = ggml_reshape_3d(ctx0, gate, 1, n_head_kv, n_tokens);++                Vcur = ggml_add(ctx0, Vcur, ggml_mul(ctx0, inp_ve, gate));+                cb(Vcur, "Vcur_ve", il);+            }++            Qcur = ggml_rms_norm(ctx0, Qcur, hparams.f_norm_rms_eps);+            Kcur = ggml_rms_norm(ctx0, Kcur, hparams.f_norm_rms_eps);+            cb(Qcur, "Qcur_normed", il);+            cb(Kcur, "Kcur_normed", il);++            // the full attention layers use no positional encoding+            if (hparams.is_swa(il)) {+                Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,+                                     ext_factor, attn_factor, beta_fast, beta_slow);+                Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,+                                     ext_factor, attn_factor, beta_fast, beta_slow);+                cb(Qcur, "Qcur", il);+                cb(Kcur, "Kcur", il);+            }++            cur = build_attn(inp_attn,+                    nullptr, nullptr, nullptr,+                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, hparams.f_attention_scale, il);+            cb(cur, "attn_raw", il);++            if (layer.attn_xsa_alpha) {+                // remove from each head output its projection on the token's own value+                const int64_t n_group = n_head/n_head_kv;++                ggml_tensor * y  = ggml_reshape_4d(ctx0, cur, n_embd_head, n_group, n_head_kv, n_tokens);+                ggml_tensor * vn = ggml_l2_norm(ctx0, Vcur, hparams.f_xsa_eps);+                vn = ggml_reshape_4d(ctx0, vn, n_embd_head, 1, n_head_kv, n_tokens);+                vn = ggml_repeat(ctx0, vn, y);++                ggml_tensor * proj = ggml_sum_rows(ctx0, ggml_mul(ctx0, y, vn));+                proj = ggml_mul(ctx0, proj, ggml_reshape_4d(ctx0, layer.attn_xsa_alpha, 1, n_group, n_head_kv, 1));++                cur = ggml_sub(ctx0, y, ggml_mul(ctx0, vn, proj));+                cb(cur, "attn_xsa", il);+            }++            cur = ggml_reshape_3d(ctx0, cur, n_embd_head, n_head, n_tokens);++            if (layer.wqkv_gate) {+                ggml_tensor * gate = ggml_sigmoid(ctx0, build_lora_mm(layer.wqkv_gate, attn_in));+                cur = ggml_mul(ctx0, cur, ggml_reshape_3d(ctx0, gate, 1, n_head, n_tokens));+                cb(cur, "attn_gated", il);+            }++            cur = ggml_reshape_2d(ctx0, cur, n_embd_head*n_head, n_tokens);+            cur = build_lora_mm(layer.wo, cur, layer.wo_s);+            cb(cur, "attn_out", il);+        }++        if (il == n_layer - 1 && inp_out_ids) {+            cur   = ggml_get_rows(ctx0, cur,   inp_out_ids);+            resid = ggml_get_rows(ctx0, resid, inp_out_ids);+        }++        cur = ggml_add(ctx0, ggml_mul(ctx0, resid, layer.attn_resid_scale), cur);+        cb(cur, "attn_resid", il);++        ggml_tensor * ffn_inp = cur;++        cur = build_norm(cur, nullptr, nullptr, LLM_NORM_RMS, il);+        cb(cur, "ffn_norm", il);++        cur = build_ffn(cur,+                layer.ffn_up,   nullptr, nullptr,+                layer.ffn_gate, nullptr, nullptr,+                layer.ffn_down, nullptr, nullptr,+                nullptr,+                LLM_FFN_SILU, LLM_FFN_PAR, il);+        cb(cur, "ffn_out", il);++        cur = ggml_add(ctx0, ggml_mul(ctx0, ffn_inp, layer.ffn_resid_scale), cur);++        cur = build_cvec(cur, il);+        cb(cur, "l_out", il);++        inpL = cur;+    }++    cur = build_norm(inpL, nullptr, nullptr, LLM_NORM_RMS, -1);++    cb(cur, "result_norm", -1);+    res->t_embd = cur;++    cur = build_lora_mm(model.output, cur, model.output_s);++    // scale * sigmoid((x + shift) / temperature)+    const auto & cap = hparams.final_logit_sigmoid_capping;+    if (cap[0] != 0.0f && cap[2] != 0.0f) {+        cur = ggml_scale_bias(ctx0, cur, 1.0f/cap[2], cap[1]/cap[2]);+        cur = ggml_sigmoid(ctx0, cur);+        cur = ggml_scale(ctx0, cur, cap[0]);+    }++    cb(cur, "result_output", -1);+    res->t_logits = cur;+}

@@ -2056,6 +2056,19 @@ struct llama_model_plm : public llama_model_base { };  +struct llama_model_limite : public llama_model_base {+    llama_model_limite(const struct llama_model_params & params) : llama_model_base(params) {}+    void load_arch_hparams(llama_model_loader & ml) override;+    void load_arch_tensors(llama_model_loader & ml) override;++    struct graph : public llm_graph_context {+        graph(const llama_model & model, const llm_graph_params & params);+    };++    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;+};++ struct llama_model_hrm_text : public llama_model_base {     llama_model_hrm_text(const struct llama_model_params & params) : llama_model_base(params) {}     void load_arch_hparams(llama_model_loader & ml) override;

@@ -269,6 +269,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {         n_embd = 160; // exercise per-head tensor split granularity with head size 80     } else if (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE) {         n_head = 4;+    } else if (arch == LLM_ARCH_LIMITE) {+        n_head  = 4;+        n_layer = 4; // dense connections on the last two layers     } else if (arch == LLM_ARCH_DEEPSEEK2             || arch == LLM_ARCH_DEEPSEEK32             || arch == LLM_ARCH_GLM_DSA@@ -297,7 +300,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {     if (arch == LLM_ARCH_QWEN3) {         n_head_kv = 1; // MQA coverage     } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER ||-            arch == LLM_ARCH_KOLIBRI1) {+            arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE) {         n_head_kv = 2; // GQA coverage     }     if (arch == LLM_ARCH_BEAM) {@@ -458,7 +461,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {         ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM,     true);     } -    if (arch == LLM_ARCH_KOLIBRI1) {+    if (arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE) {         ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, false);     } @@ -594,6 +597,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {         ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f);     } +    if (arch == LLM_ARCH_LIMITE) {+        ms.add_kv(LLM_KV_ATTENTION_SCALE,             0.1f);+        ms.add_kv(LLM_KV_ATTENTION_XSA_EPS,           1e-4f);+        ms.add_kv(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, std::vector<float>({23.0f, 5.0f, 7.5f}));+        ms.add_kv(LLM_KV_MUDD_FEED_FORWARD_LENGTH,    uint32_t(8));+        ms.add_kv(LLM_KV_MUDD_TAP_COUNT,              uint32_t(2));+        ms.add_kv(LLM_KV_MUDD_TAP_INDICES,            std::vector<int32_t>({-1, -1, -1, -1, 0, 2, 1, 3}));+    }+     // dummy tokenizer: token ids are derived from fixed-size chunks and detokenized as hex ids     {         std::vector<std::string> tokenizer_list(n_vocab);