model : Add support for Metis-1.6 models

@@ -196,6 +196,7 @@     "MambaLMHeadModel": "mamba",     "MapleForCausalLM": "maple",     "MellumForCausalLM": "mellum",+    "Metis16ForCausalLM": "metis",     "MiMoV2FlashForCausalLM": "mimo",     "MiMoV2ForCausalLM": "mimo",     "MiniCPM3ForCausalLM": "minicpm",

@@ -1891,6 +1891,9 @@ def get_vocab_base_pre(self, tokenizer) -> str:         if chkhsh == "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4":             # ref: https://huggingface.co/evilfreelancer/ruGPT3XL             res = "gpt-2"+        if chkhsh == "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9":+            # ref: https://huggingface.co/Lernex/Metis-1.6-Logos+            res = "gpt-2"         if chkhsh == "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7":             # ref: https://huggingface.co/LiquidAI/LFM2.5-8B-A1B             res = "lfm2"

@@ -0,0 +1,204 @@+from __future__ import annotations++import math+from typing import Callable, Iterable, TYPE_CHECKING++import torch++if TYPE_CHECKING:+    from torch import Tensor++from .base import LazyTorchTensor, ModelBase, TextModel, gguf+++@ModelBase.register("Metis16ForCausalLM")+class Metis16Model(TextModel):+    model_arch = gguf.MODEL_ARCH.METIS16++    # read by generate_extra_tensors() or not used at inference: dropped in modify_tensors()+    _folded = {+        "pass_embeddings", "lm_head.weight", "depth_budget.multiplier",+        "depth_memory.pass_embeddings", "depth_memory.anchor_embeddings",+        "depth_memory.state_write.bias", "depth_memory.metadata_write.bias",+    }+    _folded_layer = {+        "mixer_connection.controller.weight", "mixer_connection.controller.bias", "mixer_connection.mix_logits",+        "mixer_connection.read_logits", "mixer_connection.write_logits",+        "moe_connection.controller.weight", "moe_connection.controller.bias", "moe_connection.mix_logits",+        "moe_connection.read_logits", "moe_connection.write_logits",+        "moe.k_router.weight", "moe.k_router.bias", "moe.k_budget.multiplier",+    }++    def __init__(self, *args, **kwargs):+        super().__init__(*args, **kwargs)+        hp = self.hparams+        self.n_pass = hp["num_passes"]+        self.attn_layers = list(hp["attention_layer_indices"])+        self._generated: set[str] = set()+        # one cache slot per pass and layer, while the tensors exist once per physical layer+        self.block_count = self.n_pass * hp["num_hidden_layers"]++    def set_vocab(self):+        self._set_vocab_gpt2()+        # turns end on the first eos id, so llama.cpp must stop on it too+        eos = self.hparams.get("eos_token_id")+        if isinstance(eos, list) and eos:+            self.gguf_writer.add_eot_token_id(eos[0])++    def set_gguf_parameters(self):+        hp = self.hparams+        super().set_gguf_parameters()++        n_layer = hp["num_hidden_layers"]+        self.gguf_writer.add_head_count_kv([hp["num_key_value_heads"] if il % n_layer in self.attn_layers else 0 for il in range(self.block_count)])++        self.gguf_writer.add_expert_feed_forward_length(hp["expert_intermediate_size"])+        self.gguf_writer.add_expert_shared_feed_forward_length(hp["shared_expert_intermediate_size"])+        self.gguf_writer.add_expert_shared_count(1)+        self.gguf_writer.add_moe_latent_size(hp["latent_dim"])++        d_inner = hp["mamba_expand"] * hp["hidden_size"]+        self.gguf_writer.add_ssm_conv_kernel(hp["mamba_d_conv"])+        self.gguf_writer.add_ssm_inner_size(d_inner)+        self.gguf_writer.add_ssm_state_size(hp["mamba_d_state"])+        self.gguf_writer.add_ssm_time_step_rank(d_inner // hp["mamba_head_dim"])+        self.gguf_writer.add_ssm_group_count(hp["mamba_n_groups"])+        self.gguf_writer.add_ssm_norm_epsilon(hp["mamba_norm_eps"])++        self.gguf_writer.add_hyper_connection_count(hp["n_streams"])+        self.gguf_writer.add_metis_pass_count(self.n_pass)+        self.gguf_writer.add_metis_memory_length(hp["memory_dim"])+        self.gguf_writer.add_metis_route_length(hp["route_feature_dim"])+        self.gguf_writer.add_metis_continuation_logit_limit(hp["continuation_logit_limit"])+        self.gguf_writer.add_attention_pass_lora_rank(hp["attention_pass_lora_rank"])++    @classmethod+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:+        name, gen = item+        # the checkpoint leaves these names suffix-less; add the suffix of the GGUF tensor so the mapper keeps it+        if name.endswith((".mixer.impl.mixer.dt_bias", ".moe.selection_bias")):+            name += ".bias"+        elif (name.endswith((".moe.expert_embeddings", ".mixer.pass_gates"))+              or name in ("depth_memory.stream_gate", "stream_embeddings")):+            name += ".weight"+        elif name == "depth_memory.stream_gate_bias":+            name = "depth_memory.stream_gate.bias"+        return super().filter_tensors((name, gen))++    def _load(self, name: str) -> Tensor:+        return LazyTorchTensor.to_eager(self.model_tensors[name]())++    def _emit(self, name: str, data: Tensor) -> tuple[str, Tensor]:+        self._generated.add(name)+        return name, data++    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:+        hp = self.hparams+        r = hp["route_feature_dim"]+        n_stream = hp["n_streams"]+        T = gguf.MODEL_TENSOR+        passes = self._load("pass_embeddings")[: self.n_pass]++        # the mixes depend only on the weights and the pass, so Sinkhorn runs once here+        # the controller runs in the checkpoint dtype, as the reference does+        for il in range(hp["num_hidden_layers"]):+            for src, (mix, read, write) in (+                ("mixer_connection", (T.HC_ATTN_MIX, T.HC_ATTN_READ, T.HC_ATTN_WRITE)),+                ("moe_connection", (T.HC_FFN_MIX, T.HC_FFN_READ, T.HC_FFN_WRITE)),+            ):+                prefix = f"layers.{il}.{src}."+                ctrl_w, ctrl_b = self._load(prefix + "controller.weight"), self._load(prefix + "controller.bias")+                mixes, reads, writes = [], [], []+                for p in range(self.n_pass):+                    delta = torch.nn.functional.linear(passes[p].to(ctrl_w.dtype), ctrl_w, ctrl_b).float()+                    m_delta, r_delta, w_delta = torch.split(delta, [n_stream * n_stream, n_stream, n_stream])+                    log_m = self._load(prefix + "mix_logits").float() + m_delta.view(n_stream, n_stream)+                    for _ in range(hp["mhc_sinkhorn_iterations"]):+                        log_m = log_m - torch.logsumexp(log_m, dim=-1, keepdim=True)+                        log_m = log_m - torch.logsumexp(log_m, dim=-2, keepdim=True)+                    mixes.append(log_m.exp())+                    reads.append(torch.softmax(self._load(prefix + "read_logits").float() + r_delta, dim=-1))+                    writes.append(torch.softmax(self._load(prefix + "write_logits").float() + w_delta, dim=-1))+                act = ctrl_w.dtype+                yield self._emit(self.format_tensor_name(mix, il), torch.stack(mixes).to(act).float())+                yield self._emit(self.format_tensor_name(read, il), torch.stack(reads).to(act).float())+                yield self._emit(self.format_tensor_name(write, il), torch.stack(writes).to(act).float())++        # one bias per memory entry: write biases, pass and anchor embeddings, and the constant routed width+        meta_w = self._load("depth_memory.metadata_write.weight").float()+        mean_k = hp["num_experts_per_tok"] / hp["max_routed_k"]+        base = (self._load("depth_memory.state_write.bias").float() + self._load("depth_memory.metadata_write.bias").float()+                + mean_k * meta_w[:, r])+        pass_e = self._load("depth_memory.pass_embeddings").float()+        anchor_e = self._load("depth_memory.anchor_embeddings").float()+        n_anchor = len(self.attn_layers) + 1+        entry_bias = torch.stack([base + pass_e[p] + anchor_e[a] for p in range(self.n_pass) for a in range(n_anchor)])+        yield self._emit(self.format_tensor_name(T.DEPTH_MEM_ENTRY_BIAS), entry_bias)++    def _split_features(self, data_torch: Tensor, template: str, parts: tuple[str, ...], d: int, bid: int | None) -> Iterable[tuple[str, Tensor]]:+        # the graph reads the concatenated features one block at a time; each block is its own tensor+        for i, part in enumerate(parts):+            piece = data_torch[:, i * d:(i + 1) * d] if i < 3 else data_torch[:, 3 * d:]+            yield from super().modify_tensors(piece, template.format(part), bid)++    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:+        if name in self._generated:+            yield name, data_torch+            return+        if name in self._folded:+            return+        hp = self.hparams+        d = hp["hidden_size"]++        if bid is None:+            if name == "stream_embeddings.weight":+                # the graph keeps the streams innermost: [n_stream, n_embd, n_tokens]+                yield from super().modify_tensors(data_torch.T.contiguous(), name, bid)+                return+            if name == "depth_memory.metadata_write.weight":+                # metadata input becomes [summary, entropy, confidence, continuation]; entropy arrives unnormalized+                r = hp["route_feature_dim"]+                meta_w = data_torch.float()+                entropy_col = meta_w[:, r + 1:r + 2] / math.log(hp["n_routed_experts"])+                data_torch = torch.cat((meta_w[:, :r], entropy_col, meta_w[:, r + 2:]), dim=1)+            elif name == "depth_memory.stream_gate.weight":+                data_torch = data_torch.float() / math.sqrt(d)+            elif name == "depth_memory.route_projection.weight":+                yield from self._split_features(data_torch, "depth_memory.route_{}.weight", ("state", "retr", "diff", "hist"), d, bid)+                return+            elif name == "continuation.hidden.weight":+                yield from self._split_features(data_torch, "continuation.hidden_{}.weight", ("state", "retr", "diff", "route"), d, bid)+                return+            yield from super().modify_tensors(data_torch, name, bid)+            return++        local = name.removeprefix(f"layers.{bid}.")+        if local in self._folded_layer:+            return++        if local == "mixer.impl.mixer.A_log":+            data_torch = -torch.exp(data_torch).unsqueeze(-1)+        elif local == "mixer.impl.mixer.D":+            data_torch = data_torch.unsqueeze(-1)+        elif local == "mixer.impl.mixer.conv1d.weight":+            data_torch = data_torch.squeeze(1)+        elif local == "mixer.impl.mixer.norm.weight":+            data_torch = data_torch.reshape(hp["mamba_n_groups"], -1)+        elif local == "mixer.pass_gates.weight":+            # the reference applies tanh(gate) in the checkpoint dtype+            data_torch = torch.tanh(data_torch[: self.n_pass]).to(torch.bfloat16).float()+        elif local == "moe.shared_expert.gate_up.weight":+            gate, up = data_torch.chunk(2, dim=0)+            yield from super().modify_tensors(gate, name.replace("gate_up", "gate"), bid)+            yield from super().modify_tensors(up, name.replace("gate_up", "up"), bid)+            return++        yield from super().modify_tensors(data_torch, name, bid)++    def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:+        # FP32 in the checkpoint, tiny and read as rows, or computed here at full precision: keep FP32+        if (new_name.startswith(("cont_", "depth_mem_entry_bias", "depth_mem_stream_gate"))+                or new_name.endswith(("ffn_exp_embd.weight", "ffn_gate_inp.weight", "hc_attn_mix.weight", "hc_attn_read.weight",+                                      "hc_attn_write.weight", "hc_ffn_mix.weight", "hc_ffn_read.weight", "hc_ffn_write.weight"))):+            return gguf.GGMLQuantizationType.F32+        return super().tensor_force_quant(name, new_name, bid, n_dims)

@@ -198,6 +198,7 @@ class TOKENIZER_TYPE(IntEnum):     # jina-v2-de variants     {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"},     {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"},+    {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Lernex/Metis-1.6-Logos", "chkhsh": "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9"},     # lfm2 variants     {"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "chkhsh": "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7"},     # hrm-text (DFM Mimir) is SPM-style BPE: normalizer maps ' ' -> '▁', merges

@@ -203,6 +203,7 @@ class Attention:         ICLR_LORA_RANK               = "{arch}.attention.iclr_lora_rank"         VALUE_RESIDUAL_MIX_LORA_RANK = "{arch}.attention.value_residual_mix_lora_rank"         GATE_LORA_RANK               = "{arch}.attention.gate_lora_rank"+        PASS_LORA_RANK               = "{arch}.attention.pass_lora_rank"         REL_BUCKETS_COUNT            = "{arch}.attention.relative_buckets_count"         SLIDING_WINDOW               = "{arch}.attention.sliding_window"         SCALE                        = "{arch}.attention.scale"@@ -315,13 +316,20 @@ class HRM:         L_CYCLES         = "{arch}.hrm.l_cycles"         PREFIX_LM        = "{arch}.hrm.prefix_lm" +    class Metis:+        PASS_COUNT               = "{arch}.metis.pass_count"+        MEMORY_LENGTH            = "{arch}.metis.memory_length"+        ROUTE_LENGTH             = "{arch}.metis.route_length"+        CONTINUATION_LOGIT_LIMIT = "{arch}.metis.continuation_logit_limit"+     class SSM:         CONV_KERNEL    = "{arch}.ssm.conv_kernel"         INNER_SIZE     = "{arch}.ssm.inner_size"         STATE_SIZE     = "{arch}.ssm.state_size"         TIME_STEP_RANK = "{arch}.ssm.time_step_rank"         GROUP_COUNT    = "{arch}.ssm.group_count"         DT_B_C_RMS     = "{arch}.ssm.dt_b_c_rms"+        NORM_EPSILON   = "{arch}.ssm.norm_epsilon"      class KDA:         HEAD_DIM         = "{arch}.kda.head_dim"@@ -675,6 +683,7 @@ class MODEL_ARCH(IntEnum):     NEMOTRON         = auto()     NEMOTRON_H       = auto()     NEMOTRON_H_MOE   = auto()+    METIS16          = auto()     EXAONE           = auto()     EXAONE4          = auto()     EXAONE_MOE       = auto()@@ -824,6 +833,36 @@ class MODEL_TENSOR(IntEnum):     FFN_GATE_TID2EID     = auto()     MOE_LATENT_DOWN      = auto() # nemotron 3 super     MOE_LATENT_UP        = auto() # nemotron 3 super+    STREAM_EMBD          = auto()+    DEPTH_MEM_STATE_WRITE = auto()+    DEPTH_MEM_META_WRITE = auto()+    DEPTH_MEM_ENTRY_BIAS = auto()+    DEPTH_MEM_QUERY      = auto()+    DEPTH_MEM_KEY        = auto()+    DEPTH_MEM_VALUE      = auto()+    DEPTH_MEM_OUT        = auto()+    DEPTH_MEM_ROUTE_STATE = auto()+    DEPTH_MEM_ROUTE_RETR = auto()+    DEPTH_MEM_ROUTE_DIFF = auto()+    DEPTH_MEM_ROUTE_HIST = auto()+    DEPTH_MEM_ROUTE      = auto()+    DEPTH_MEM_STREAM_GATE = auto()+    CONT_HIDDEN_STATE    = auto()+    CONT_HIDDEN_RETR     = auto()+    CONT_HIDDEN_DIFF     = auto()+    CONT_HIDDEN_ROUTE    = auto()+    CONT_HIDDEN          = auto()+    CONT_OUTPUT          = auto()+    HC_ATTN_MIX          = auto()+    HC_ATTN_READ         = auto()+    HC_ATTN_WRITE        = auto()+    HC_FFN_MIX           = auto()+    HC_FFN_READ          = auto()+    HC_FFN_WRITE         = auto()+    ATTN_PASS_LORA_DOWN  = auto()+    ATTN_PASS_LORA_UP    = auto()+    ATTN_PASS_GATE       = auto()+    FFN_EXP_EMBD         = auto()     FFN_RESID_ALPHA      = auto()     FFN_RESID_BETA       = auto()     FFN_PRE_EXP_NORM     = auto()@@ -1533,6 +1572,7 @@ class MODEL_TENSOR(IntEnum):     MODEL_ARCH.NEMOTRON:         "nemotron",     MODEL_ARCH.NEMOTRON_H:       "nemotron_h",     MODEL_ARCH.NEMOTRON_H_MOE:   "nemotron_h_moe",+    MODEL_ARCH.METIS16:          "metis16",     MODEL_ARCH.EXAONE:           "exaone",     MODEL_ARCH.EXAONE4:          "exaone4",     MODEL_ARCH.EXAONE_MOE:       "exaone-moe",@@ -1683,6 +1723,36 @@ class MODEL_TENSOR(IntEnum):     MODEL_TENSOR.FFN_GATE_TID2EID:          "blk.{bid}.ffn_gate_tid2eid",     MODEL_TENSOR.MOE_LATENT_DOWN:           "blk.{bid}.ffn_latent_down",      # nemotron 3 super     MODEL_TENSOR.MOE_LATENT_UP:             "blk.{bid}.ffn_latent_up",        # nemotron 3 super+    MODEL_TENSOR.STREAM_EMBD:               "stream_embd",+    MODEL_TENSOR.DEPTH_MEM_STATE_WRITE:     "depth_mem_state_write",+    MODEL_TENSOR.DEPTH_MEM_META_WRITE:      "depth_mem_meta_write",+    MODEL_TENSOR.DEPTH_MEM_ENTRY_BIAS:      "depth_mem_entry_bias",+    MODEL_TENSOR.DEPTH_MEM_QUERY:           "depth_mem_query",+    MODEL_TENSOR.DEPTH_MEM_KEY:             "depth_mem_key",+    MODEL_TENSOR.DEPTH_MEM_VALUE:           "depth_mem_value",+    MODEL_TENSOR.DEPTH_MEM_OUT:             "depth_mem_out",+    MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE:     "depth_mem_route_state",+    MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR:      "depth_mem_route_retr",+    MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF:      "depth_mem_route_diff",+    MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST:      "depth_mem_route_hist",+    MODEL_TENSOR.DEPTH_MEM_ROUTE:           "depth_mem_route",+    MODEL_TENSOR.DEPTH_MEM_STREAM_GATE:     "depth_mem_stream_gate",+    MODEL_TENSOR.CONT_HIDDEN_STATE:         "cont_hidden_state",+    MODEL_TENSOR.CONT_HIDDEN_RETR:          "cont_hidden_retr",+    MODEL_TENSOR.CONT_HIDDEN_DIFF:          "cont_hidden_diff",+    MODEL_TENSOR.CONT_HIDDEN_ROUTE:         "cont_hidden_route",+    MODEL_TENSOR.CONT_HIDDEN:               "cont_hidden",+    MODEL_TENSOR.CONT_OUTPUT:               "cont_output",+    MODEL_TENSOR.HC_ATTN_MIX:               "blk.{bid}.hc_attn_mix",+    MODEL_TENSOR.HC_ATTN_READ:              "blk.{bid}.hc_attn_read",+    MODEL_TENSOR.HC_ATTN_WRITE:             "blk.{bid}.hc_attn_write",+    MODEL_TENSOR.HC_FFN_MIX:                "blk.{bid}.hc_ffn_mix",+    MODEL_TENSOR.HC_FFN_READ:               "blk.{bid}.hc_ffn_read",+    MODEL_TENSOR.HC_FFN_WRITE:              "blk.{bid}.hc_ffn_write",+    MODEL_TENSOR.ATTN_PASS_LORA_DOWN:       "blk.{bid}.attn_pass_lora_down",+    MODEL_TENSOR.ATTN_PASS_LORA_UP:         "blk.{bid}.attn_pass_lora_up",+    MODEL_TENSOR.ATTN_PASS_GATE:            "blk.{bid}.attn_pass_gate",+    MODEL_TENSOR.FFN_EXP_EMBD:              "blk.{bid}.ffn_exp_embd",     MODEL_TENSOR.FFN_RESID_ALPHA:           "blk.{bid}.ffn_resid_alpha",     MODEL_TENSOR.FFN_RESID_BETA:            "blk.{bid}.ffn_resid_beta",     MODEL_TENSOR.FFN_PRE_EXP_NORM:          "blk.{bid}.ffn_pre_exp_norm",@@ -4825,6 +4895,61 @@ class MODEL_TENSOR(IntEnum):         MODEL_TENSOR.FFN_DOWN,         MODEL_TENSOR.FFN_UP,     ],+    MODEL_ARCH.METIS16: [+        MODEL_TENSOR.TOKEN_EMBD,+        MODEL_TENSOR.OUTPUT_NORM,+        MODEL_TENSOR.OUTPUT,+        MODEL_TENSOR.ATTN_NORM,+        MODEL_TENSOR.FFN_NORM,+        MODEL_TENSOR.STREAM_EMBD,+        MODEL_TENSOR.DEPTH_MEM_STATE_WRITE,+        MODEL_TENSOR.DEPTH_MEM_META_WRITE,+        MODEL_TENSOR.DEPTH_MEM_ENTRY_BIAS,+        MODEL_TENSOR.DEPTH_MEM_QUERY,+        MODEL_TENSOR.DEPTH_MEM_KEY,+        MODEL_TENSOR.DEPTH_MEM_VALUE,+        MODEL_TENSOR.DEPTH_MEM_OUT,+        MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE,+        MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR,+        MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF,+        MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST,+        MODEL_TENSOR.DEPTH_MEM_ROUTE,+        MODEL_TENSOR.DEPTH_MEM_STREAM_GATE,+        MODEL_TENSOR.CONT_HIDDEN_STATE,+        MODEL_TENSOR.CONT_HIDDEN_RETR,+        MODEL_TENSOR.CONT_HIDDEN_DIFF,+        MODEL_TENSOR.CONT_HIDDEN_ROUTE,+        MODEL_TENSOR.CONT_HIDDEN,+        MODEL_TENSOR.CONT_OUTPUT,+        MODEL_TENSOR.HC_ATTN_MIX,+        MODEL_TENSOR.HC_ATTN_READ,+        MODEL_TENSOR.HC_ATTN_WRITE,+        MODEL_TENSOR.HC_FFN_MIX,+        MODEL_TENSOR.HC_FFN_READ,+        MODEL_TENSOR.HC_FFN_WRITE,+        MODEL_TENSOR.ATTN_PASS_LORA_DOWN,+        MODEL_TENSOR.ATTN_PASS_LORA_UP,+        MODEL_TENSOR.ATTN_PASS_GATE,+        MODEL_TENSOR.FFN_EXP_EMBD,+        MODEL_TENSOR.SSM_IN,+        MODEL_TENSOR.SSM_CONV1D,+        MODEL_TENSOR.SSM_DT,+        MODEL_TENSOR.SSM_A,+        MODEL_TENSOR.SSM_D,+        MODEL_TENSOR.SSM_NORM,+        MODEL_TENSOR.SSM_OUT,+        MODEL_TENSOR.ATTN_QKV,+        MODEL_TENSOR.ATTN_OUT,+        MODEL_TENSOR.MOE_LATENT_DOWN,+        MODEL_TENSOR.MOE_LATENT_UP,+        MODEL_TENSOR.FFN_GATE_INP,+        MODEL_TENSOR.FFN_EXP_PROBS_B,+        MODEL_TENSOR.FFN_GATE_UP_EXP,+        MODEL_TENSOR.FFN_DOWN_EXP,+        MODEL_TENSOR.FFN_GATE_SHEXP,+        MODEL_TENSOR.FFN_UP_SHEXP,+        MODEL_TENSOR.FFN_DOWN_SHEXP,+    ],     MODEL_ARCH.NEMOTRON_H: [         MODEL_TENSOR.TOKEN_EMBD,         MODEL_TENSOR.OUTPUT_NORM,

@@ -975,6 +975,18 @@ def add_hrm_l_cycles(self, value: int) -> None:     def add_hrm_prefix_lm(self, value: bool) -> None:         self.add_bool(Keys.HRM.PREFIX_LM.format(arch=self.arch), value) +    def add_metis_pass_count(self, value: int) -> None:+        self.add_uint32(Keys.Metis.PASS_COUNT.format(arch=self.arch), value)++    def add_metis_memory_length(self, value: int) -> None:+        self.add_uint32(Keys.Metis.MEMORY_LENGTH.format(arch=self.arch), value)++    def add_metis_route_length(self, value: int) -> None:+        self.add_uint32(Keys.Metis.ROUTE_LENGTH.format(arch=self.arch), value)++    def add_metis_continuation_logit_limit(self, value: float) -> None:+        self.add_float32(Keys.Metis.CONTINUATION_LOGIT_LIMIT.format(arch=self.arch), value)+     def add_adapter_count(self, count: int) -> None:         self.add_uint32(Keys.Adapters.COUNT.format(arch=self.arch), count) @@ -1086,6 +1098,9 @@ def add_attention_output_group_count(self, count: int) -> None:     def add_attention_output_lora_rank(self, length: int) -> None:         self.add_uint32(Keys.Attention.OUTPUT_LORA_RANK.format(arch=self.arch), length) +    def add_attention_pass_lora_rank(self, length: int) -> None:+        self.add_uint32(Keys.Attention.PASS_LORA_RANK.format(arch=self.arch), length)+     def add_attention_compress_ratios(self, values: Sequence[int]) -> None:         self.add_array(Keys.Attention.COMPRESS_RATIOS.format(arch=self.arch), values) @@ -1267,6 +1282,9 @@ def add_ssm_group_count(self, value: int) -> None:     def add_ssm_dt_b_c_rms(self, value: bool) -> None:         self.add_bool(Keys.SSM.DT_B_C_RMS.format(arch=self.arch), value) +    def add_ssm_norm_epsilon(self, value: float) -> None:+        self.add_float32(Keys.SSM.NORM_EPSILON.format(arch=self.arch), value)+     def add_expert_latent_length(self, value: int) -> None:         self.add_uint32(Keys.LLM.EXPERT_LATENT_LENGTH.format(arch=self.arch), value) 

@@ -35,6 +35,7 @@ class TensorNameMap:             "model.transformer.wte",                     # llada             "embed_tokens",                              # qwen3-embedding             "model.embed",                               # talkie+            "embedding",                                 # metis16         ),          # Masked embeddings@@ -224,6 +225,7 @@ class TensorNameMap:     block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = {         # Attention norm         MODEL_TENSOR.ATTN_NORM: (+            "layers.{bid}.mixer_connection.norm",                   # metis16             "model.lilicorr.layers.{bid}.attn_norm",                # lilicorr             "gpt_neox.layers.{bid}.input_layernorm",                # gptneox             "transformer.h.{bid}.ln_1",                             # gpt2 gpt-j refact qwen jais exaone@@ -294,6 +296,7 @@ class TensorNameMap:             "model.layers.{bid}.linear_attn.in_proj_qkv",                          # qwen3.5             "head.layers.{bid}.self_attn.in_proj",  # laya             "layers.{bid}.linear_attn.in_proj_qkv",                                # qwen3.5 text+            "layers.{bid}.mixer.qkv",                                              # metis16         ),          # Attention query@@ -401,6 +404,7 @@ class TensorNameMap:             "model.layers.{bid}.self_attn.language_expert_dense",           # cogvlm             "model.blocks.{bid}.attn.attn_resid",                           # talkie             "head.layers.{bid}.self_attn.out_proj",  # laya+            "layers.{bid}.mixer.out",                                       # metis16         ),          # Attention output norm@@ -449,6 +453,7 @@ class TensorNameMap:          # Feed-forward norm         MODEL_TENSOR.FFN_NORM: (+            "layers.{bid}.moe_connection.norm",                              # metis16             "model.lilicorr.layers.{bid}.mlp_norm",                          # lilicorr             "model.lilicorr.feature_mlp.{bid}.norm",                         # lilicorr             "gpt_neox.layers.{bid}.post_attention_layernorm",                # gptneox@@ -536,13 +541,15 @@ class TensorNameMap:             "backbone.layers.{bid}.mixer.gate",                 # nemotron-h-moe             "model.layers.{bid}.moe.gate",                      # step3.5             "model.layers.{bid}.router.proj",                   # gemma4+            "layers.{bid}.moe.expert_router",                   # metis16         ),          MODEL_TENSOR.FFN_GATE_INP_SHEXP: (             "model.layers.{bid}.mlp.shared_expert_gate", # qwen2moe         ),          MODEL_TENSOR.FFN_EXP_PROBS_B: (+            "layers.{bid}.moe.selection_bias",                              # metis16             "model.layers.{bid}.mlp.gate.e_score_correction",               # deepseek-v3 dots1             "model.layers.{bid}.mlp.gate.e_score_correction_bias",          # muse-spark             "model.layers.{bid}.mlp.moe_statics.e_score_correction",        # ernie4.5-moe@@ -617,6 +624,7 @@ class TensorNameMap:         ),          MODEL_TENSOR.FFN_UP_SHEXP: (+            "layers.{bid}.moe.shared_expert.up",                     # metis16             "model.layers.{bid}.mlp.shared_expert.up_proj",          # qwen2moe             "model.layers.{bid}.mlp.shared_experts.up_proj",         # deepseek deepseek2             "model.layers.{bid}.feed_forward.shared_expert.up_proj", # llama4@@ -670,6 +678,7 @@ class TensorNameMap:         ),          MODEL_TENSOR.FFN_GATE_SHEXP: (+            "layers.{bid}.moe.shared_expert.gate",                     # metis16             "model.layers.{bid}.mlp.shared_expert.gate_proj",          # qwen2moe             "model.layers.{bid}.mlp.shared_experts.gate_proj",         # deepseek deepseek2             "model.layers.{bid}.feed_forward.shared_expert.gate_proj", # llama4@@ -686,18 +695,94 @@ class TensorNameMap:         MODEL_TENSOR.FFN_GATE_UP_EXP: (             "model.layers.{bid}.mlp.experts.gate_up_proj",             "model.layers.{bid}.experts.gate_up_proj", # gemma4+            "layers.{bid}.moe.local_experts.gate_up",  # metis16         ),          MODEL_TENSOR.MOE_LATENT_DOWN: (             "backbone.layers.{bid}.mixer.fc1_latent_proj",                 # nemotron 3 super             "model.layers.{bid}.mlp.pre_expert_proj",                      # muse-spark+            "layers.{bid}.moe.latent_down",                                # metis16         ),          MODEL_TENSOR.MOE_LATENT_UP: (             "backbone.layers.{bid}.mixer.fc2_latent_proj",                 # nemotron 3 super             "model.layers.{bid}.mlp.post_expert_proj",                     # muse-spark+            "layers.{bid}.moe.latent_up",                                  # metis16         ), +        MODEL_TENSOR.STREAM_EMBD: (+            "stream_embeddings",  # metis16+        ),+        MODEL_TENSOR.DEPTH_MEM_STATE_WRITE: (+            "depth_memory.state_write",  # metis16+        ),+        MODEL_TENSOR.DEPTH_MEM_META_WRITE: (+            "depth_memory.metadata_write",  # metis16+        ),+        MODEL_TENSOR.DEPTH_MEM_QUERY: (+            "depth_memory.query",  # metis16+        ),+        MODEL_TENSOR.DEPTH_MEM_KEY: (+            "depth_memory.key",  # metis16+        ),+        MODEL_TENSOR.DEPTH_MEM_VALUE: (+            "depth_memory.value",  # metis16+        ),+        MODEL_TENSOR.DEPTH_MEM_OUT: (+            "depth_memory.output",  # metis16+        ),+        MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE: (+            "depth_memory.route_state",  # metis16+        ),+        MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR: (+            "depth_memory.route_retr",  # metis16+        ),+        MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF: (+            "depth_memory.route_diff",  # metis16+        ),+        MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST: (+            "depth_memory.route_hist",  # metis16+        ),+        MODEL_TENSOR.DEPTH_MEM_ROUTE: (+            "depth_memory.route_projection",  # metis16+        ),+        MODEL_TENSOR.DEPTH_MEM_STREAM_GATE: (+            "depth_memory.stream_gate",  # metis16+        ),+        MODEL_TENSOR.CONT_HIDDEN_STATE: (+            "continuation.hidden_state",  # metis16+        ),+        MODEL_TENSOR.CONT_HIDDEN_RETR: (+            "continuation.hidden_retr",  # metis16+        ),+        MODEL_TENSOR.CONT_HIDDEN_DIFF: (+            "continuation.hidden_diff",  # metis16+        ),+        MODEL_TENSOR.CONT_HIDDEN_ROUTE: (+            "continuation.hidden_route",  # metis16+        ),+        MODEL_TENSOR.CONT_HIDDEN: (+            "continuation.hidden",  # metis16+        ),+        MODEL_TENSOR.CONT_OUTPUT: (+            "continuation.output",  # metis16+        ),+        MODEL_TENSOR.ATTN_PASS_LORA_DOWN: (+            "layers.{bid}.mixer.pass_lora_down",  # metis16+        ),+        MODEL_TENSOR.ATTN_PASS_LORA_UP: (+            "layers.{bid}.mixer.pass_lora_up",  # metis16+        ),+        MODEL_TENSOR.ATTN_PASS_GATE: (+            "layers.{bid}.mixer.pass_gates",  # metis16+        ),+        MODEL_TENSOR.FFN_EXP_EMBD: (+            "layers.{bid}.moe.expert_embeddings",  # metis16+        ),++++         MODEL_TENSOR.FFN_PRE_EXP_NORM: (             "model.layers.{bid}.mlp.pre_expert_norm",                      # muse-spark         ),@@ -759,6 +844,7 @@ class TensorNameMap:             "model.layers.{bid}.block_sparse_moe.experts.down",     # smallthinker             "model.layers.{bid}.moe.down_proj",                     # step3.5             "model.layers.{bid}.experts.down_proj",                 # gemma4+            "layers.{bid}.moe.local_experts.down",                  # metis16         ),          MODEL_TENSOR.FFN_DOWN_SHEXP: (@@ -771,6 +857,7 @@ class TensorNameMap:             "backbone.layers.{bid}.mixer.shared_experts.down_proj",    # nemotron-h-moe             "model.layers.{bid}.block_sparse_moe.shared_experts.down_proj", # kimi             "model.layers.{bid}.share_expert.down_proj",               # step3.5+            "layers.{bid}.moe.shared_expert.down",                     # metis16         ),          MODEL_TENSOR.FFN_DOWN_CHEXP: (@@ -906,6 +993,7 @@ class TensorNameMap:             "model.layers.{bid}.mamba.in_proj",             # jamba falcon-h1 granite-hybrid             "model.layers.layers.{bid}.mixer.in_proj",      # plamo2             "model.layers.{bid}.linear_attn.in_proj_qkvz",  # qwen3next+            "layers.{bid}.mixer.impl.mixer.in_proj",        # metis16         ),          MODEL_TENSOR.SSM_CONV1D: (@@ -915,6 +1003,7 @@ class TensorNameMap:             "model.layers.layers.{bid}.mixer.conv1d",  # plamo2             "model.layers.{bid}.linear_attn.conv1d",   # qwen3next             "layers.{bid}.linear_attn.conv1d",         # qwen3.5 text+            "layers.{bid}.mixer.impl.mixer.conv1d",    # metis16         ),          MODEL_TENSOR.SSM_X: (@@ -925,6 +1014,7 @@ class TensorNameMap:         ),          MODEL_TENSOR.SSM_DT: (+            "layers.{bid}.mixer.impl.mixer.dt_bias",    # metis16             "model.layers.{bid}.dt_proj",               # mamba-hf             "backbone.layers.{bid}.mixer.dt_proj",      # mamba             "model.layers.{bid}.mamba.dt_proj",         # jamba falcon-h1 granite-hybrid@@ -942,6 +1032,7 @@ class TensorNameMap:         ),          MODEL_TENSOR.SSM_A: (+            "layers.{bid}.mixer.impl.mixer.A_log",    # metis16             "model.layers.{bid}.A_log",               # mamba-hf             "backbone.layers.{bid}.mixer.A_log",      # mamba             "model.layers.{bid}.mamba.A_log",         # jamba falcon-h1 granite-hybrid@@ -965,13 +1056,15 @@ class TensorNameMap:         ),          MODEL_TENSOR.SSM_D: (+            "layers.{bid}.mixer.impl.mixer.D",    # metis16             "model.layers.{bid}.D",               # mamba-hf             "backbone.layers.{bid}.mixer.D",      # mamba             "model.layers.{bid}.mamba.D",         # jamba falcon-h1 granite-hybrid             "model.layers.layers.{bid}.mixer.D",  # plamo2         ),          MODEL_TENSOR.SSM_NORM: (+            "layers.{bid}.mixer.impl.mixer.norm",   # metis16             "model.layers.{bid}.mamba.norm",        # falcon-h1 granite-hybrid             "model.layers.{bid}.linear_attn.norm",  # qwen3next             "layers.{bid}.linear_attn.norm",        # qwen3.5 text@@ -987,6 +1080,7 @@ class TensorNameMap:             "model.layers.{bid}.linear_attn.out_proj",   # qwen3next             "layers.{bid}.linear_attn.out_proj",         # qwen3.5 text             "model.layers.layers.{bid}.mixer.out_proj",  # plamo2+            "layers.{bid}.mixer.impl.mixer.out_proj",    # metis16         ),          MODEL_TENSOR.SSM_ALPHA: (

@@ -101,6 +101,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {     { LLM_ARCH_NEMOTRON,         "nemotron"         },     { LLM_ARCH_NEMOTRON_H,       "nemotron_h"       },     { LLM_ARCH_NEMOTRON_H_MOE,   "nemotron_h_moe"   },+    { LLM_ARCH_METIS16,          "metis16"          },     { LLM_ARCH_EXAONE,           "exaone"           },     { LLM_ARCH_EXAONE4,          "exaone4"          },     { LLM_ARCH_EXAONE_MOE,       "exaone-moe"       },@@ -271,6 +272,10 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {     { LLM_KV_HRM_H_CYCLES,                      "%s.hrm.h_cycles"                      },     { LLM_KV_HRM_L_CYCLES,                      "%s.hrm.l_cycles"                      },     { LLM_KV_HRM_PREFIX_LM,                     "%s.hrm.prefix_lm"                     },+    { LLM_KV_METIS_PASS_COUNT,                  "%s.metis.pass_count"                  },+    { LLM_KV_METIS_MEMORY_LENGTH,               "%s.metis.memory_length"               },+    { LLM_KV_METIS_ROUTE_LENGTH,                "%s.metis.route_length"                },+    { LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT,    "%s.metis.continuation_logit_limit"    },     { LLM_KV_MUDD_FEED_FORWARD_LENGTH,          "%s.mudd.feed_forward_length"          },     { LLM_KV_MUDD_TAP_COUNT,                    "%s.mudd.tap_count"                    },     { LLM_KV_MUDD_TAP_INDICES,                  "%s.mudd.tap_indices"                  },@@ -328,6 +333,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {     { LLM_KV_ATTENTION_KV_SOURCE_LAYERS,               "%s.attention.kv_source_layers"               },     { LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT,           "%s.attention.output_group_count"           },     { LLM_KV_ATTENTION_OUTPUT_LORA_RANK,             "%s.attention.output_lora_rank"             },+    { LLM_KV_ATTENTION_PASS_LORA_RANK,               "%s.attention.pass_lora_rank"               },     { LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE,      "%s.attention.compress_rope_freq_base"      },     { LLM_KV_ATTENTION_COMPRESS_RATIOS,              "%s.attention.compress_ratios"              },     { LLM_KV_ATTENTION_SHARED_KV_LAYERS,             "%s.attention.shared_kv_layers"             },@@ -386,6 +392,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {     { LLM_KV_SSM_TIME_STEP_RANK, "%s.ssm.time_step_rank" },     { LLM_KV_SSM_GROUP_COUNT,    "%s.ssm.group_count"    },     { LLM_KV_SSM_DT_B_C_RMS,     "%s.ssm.dt_b_c_rms"     },+    { LLM_KV_SSM_NORM_EPS,       "%s.ssm.norm_epsilon"   },      { LLM_KV_KDA_HEAD_DIM,         "%s.kda.head_dim"         },     { LLM_KV_KDA_SAFE_GATE,        "%s.kda.safe_gate"        },@@ -547,6 +554,36 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {     { LLM_TENSOR_FFN_EXP_PROBS_B_VL,                     "blk.%d.exp_probs_b_vl" },     { LLM_TENSOR_FFN_LATENT_DOWN,                        "blk.%d.ffn_latent_down" },     { LLM_TENSOR_FFN_LATENT_UP,                          "blk.%d.ffn_latent_up" },+    { LLM_TENSOR_STREAM_EMBD,                            "stream_embd" },+    { LLM_TENSOR_DEPTH_MEM_STATE_WRITE,                  "depth_mem_state_write" },+    { LLM_TENSOR_DEPTH_MEM_META_WRITE,                   "depth_mem_meta_write" },+    { LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS,                   "depth_mem_entry_bias" },+    { LLM_TENSOR_DEPTH_MEM_QUERY,                        "depth_mem_query" },+    { LLM_TENSOR_DEPTH_MEM_KEY,                          "depth_mem_key" },+    { LLM_TENSOR_DEPTH_MEM_VALUE,                        "depth_mem_value" },+    { LLM_TENSOR_DEPTH_MEM_OUT,                          "depth_mem_out" },+    { LLM_TENSOR_DEPTH_MEM_ROUTE_STATE,                  "depth_mem_route_state" },+    { LLM_TENSOR_DEPTH_MEM_ROUTE_RETR,                   "depth_mem_route_retr" },+    { LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF,                   "depth_mem_route_diff" },+    { LLM_TENSOR_DEPTH_MEM_ROUTE_HIST,                   "depth_mem_route_hist" },+    { LLM_TENSOR_DEPTH_MEM_ROUTE,                        "depth_mem_route" },+    { LLM_TENSOR_DEPTH_MEM_STREAM_GATE,                  "depth_mem_stream_gate" },+    { LLM_TENSOR_CONT_HIDDEN_STATE,                      "cont_hidden_state" },+    { LLM_TENSOR_CONT_HIDDEN_RETR,                       "cont_hidden_retr" },+    { LLM_TENSOR_CONT_HIDDEN_DIFF,                       "cont_hidden_diff" },+    { LLM_TENSOR_CONT_HIDDEN_ROUTE,                      "cont_hidden_route" },+    { LLM_TENSOR_CONT_HIDDEN,                            "cont_hidden" },+    { LLM_TENSOR_CONT_OUTPUT,                            "cont_output" },+    { LLM_TENSOR_HC_ATTN_MIX,                            "blk.%d.hc_attn_mix" },+    { LLM_TENSOR_HC_ATTN_READ,                           "blk.%d.hc_attn_read" },+    { LLM_TENSOR_HC_ATTN_WRITE,                          "blk.%d.hc_attn_write" },+    { LLM_TENSOR_HC_FFN_MIX,                             "blk.%d.hc_ffn_mix" },+    { LLM_TENSOR_HC_FFN_READ,                            "blk.%d.hc_ffn_read" },+    { LLM_TENSOR_HC_FFN_WRITE,                           "blk.%d.hc_ffn_write" },+    { LLM_TENSOR_ATTN_PASS_LORA_DOWN,                    "blk.%d.attn_pass_lora_down" },+    { LLM_TENSOR_ATTN_PASS_LORA_UP,                      "blk.%d.attn_pass_lora_up" },+    { LLM_TENSOR_ATTN_PASS_GATE,                         "blk.%d.attn_pass_gate" },+    { LLM_TENSOR_FFN_EXP_EMBD,                           "blk.%d.ffn_exp_embd" },     { LLM_TENSOR_FFN_PRE_EXP_NORM,                       "blk.%d.ffn_pre_exp_norm" },     { LLM_TENSOR_FFN_POST_EXP_NORM,                      "blk.%d.ffn_post_exp_norm" },     { LLM_TENSOR_FFN_RESID_ALPHA,                        "blk.%d.ffn_resid_alpha" },@@ -1157,6 +1194,36 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {     // latent projections feed ggml_mul_mat, the buft probe must use MUL_MAT to keep them on GPU     {LLM_TENSOR_FFN_LATENT_DOWN,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},     {LLM_TENSOR_FFN_LATENT_UP,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_STREAM_EMBD,                {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_ADD}},+    {LLM_TENSOR_DEPTH_MEM_STATE_WRITE,      {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_DEPTH_MEM_META_WRITE,       {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS,       {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_DEPTH_MEM_QUERY,            {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_DEPTH_MEM_KEY,              {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_DEPTH_MEM_VALUE,            {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_DEPTH_MEM_OUT,              {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_DEPTH_MEM_ROUTE_STATE,      {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_DEPTH_MEM_ROUTE_RETR,       {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF,       {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_DEPTH_MEM_ROUTE_HIST,       {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_DEPTH_MEM_ROUTE,            {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_DEPTH_MEM_STREAM_GATE,      {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_CONT_HIDDEN_STATE,          {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_CONT_HIDDEN_RETR,           {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_CONT_HIDDEN_DIFF,           {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_CONT_HIDDEN_ROUTE,          {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_CONT_HIDDEN,                {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_CONT_OUTPUT,                {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_HC_ATTN_MIX,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_HC_ATTN_READ,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_HC_ATTN_WRITE,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_HC_FFN_MIX,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_HC_FFN_READ,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_HC_FFN_WRITE,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_ATTN_PASS_LORA_DOWN,        {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_ATTN_PASS_LORA_UP,          {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},+    {LLM_TENSOR_ATTN_PASS_GATE,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},+    {LLM_TENSOR_FFN_EXP_EMBD,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_GET_ROWS}},     {LLM_TENSOR_FFN_PRE_EXP_NORM,           {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},     {LLM_TENSOR_FFN_POST_EXP_NORM,          {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},     {LLM_TENSOR_FFN_RESID_ALPHA,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},@@ -1285,6 +1352,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {         case LLM_ARCH_LFM3:         case LLM_ARCH_NEMOTRON_H:         case LLM_ARCH_NEMOTRON_H_MOE:+        case LLM_ARCH_METIS16:         case LLM_ARCH_QWEN3NEXT:         case LLM_ARCH_KIMI_LINEAR:         case LLM_ARCH_BAILINGMOE3:

@@ -105,6 +105,7 @@ enum llm_arch {     LLM_ARCH_NEMOTRON,     LLM_ARCH_NEMOTRON_H,     LLM_ARCH_NEMOTRON_H_MOE,+    LLM_ARCH_METIS16,     LLM_ARCH_EXAONE,     LLM_ARCH_EXAONE4,     LLM_ARCH_EXAONE_MOE,@@ -275,6 +276,10 @@ enum llm_kv {     LLM_KV_HRM_H_CYCLES,     LLM_KV_HRM_L_CYCLES,     LLM_KV_HRM_PREFIX_LM,+    LLM_KV_METIS_PASS_COUNT,+    LLM_KV_METIS_MEMORY_LENGTH,+    LLM_KV_METIS_ROUTE_LENGTH,+    LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT,     LLM_KV_REALTIME_DELAY_TOKENS,     LLM_KV_MUDD_FEED_FORWARD_LENGTH,     LLM_KV_MUDD_TAP_COUNT,@@ -332,6 +337,7 @@ enum llm_kv {     LLM_KV_ATTENTION_KV_SOURCE_LAYERS,     LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT,     LLM_KV_ATTENTION_OUTPUT_LORA_RANK,+    LLM_KV_ATTENTION_PASS_LORA_RANK,     LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE,     LLM_KV_ATTENTION_COMPRESS_RATIOS,     LLM_KV_ATTENTION_SHARED_KV_LAYERS,@@ -390,6 +396,7 @@ enum llm_kv {     LLM_KV_SSM_TIME_STEP_RANK,     LLM_KV_SSM_GROUP_COUNT,     LLM_KV_SSM_DT_B_C_RMS,+    LLM_KV_SSM_NORM_EPS,      LLM_KV_KDA_HEAD_DIM,     LLM_KV_KDA_SAFE_GATE,@@ -565,6 +572,36 @@ enum llm_tensor {     LLM_TENSOR_FFN_EXP_PROBS_B_VL,     LLM_TENSOR_FFN_LATENT_DOWN,     LLM_TENSOR_FFN_LATENT_UP,+    LLM_TENSOR_STREAM_EMBD,+    LLM_TENSOR_DEPTH_MEM_STATE_WRITE,+    LLM_TENSOR_DEPTH_MEM_META_WRITE,+    LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS,+    LLM_TENSOR_DEPTH_MEM_QUERY,+    LLM_TENSOR_DEPTH_MEM_KEY,+    LLM_TENSOR_DEPTH_MEM_VALUE,+    LLM_TENSOR_DEPTH_MEM_OUT,+    LLM_TENSOR_DEPTH_MEM_ROUTE_STATE,+    LLM_TENSOR_DEPTH_MEM_ROUTE_RETR,+    LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF,+    LLM_TENSOR_DEPTH_MEM_ROUTE_HIST,+    LLM_TENSOR_DEPTH_MEM_ROUTE,+    LLM_TENSOR_DEPTH_MEM_STREAM_GATE,+    LLM_TENSOR_CONT_HIDDEN_STATE,+    LLM_TENSOR_CONT_HIDDEN_RETR,+    LLM_TENSOR_CONT_HIDDEN_DIFF,+    LLM_TENSOR_CONT_HIDDEN_ROUTE,+    LLM_TENSOR_CONT_HIDDEN,+    LLM_TENSOR_CONT_OUTPUT,+    LLM_TENSOR_HC_ATTN_MIX,+    LLM_TENSOR_HC_ATTN_READ,+    LLM_TENSOR_HC_ATTN_WRITE,+    LLM_TENSOR_HC_FFN_MIX,+    LLM_TENSOR_HC_FFN_READ,+    LLM_TENSOR_HC_FFN_WRITE,+    LLM_TENSOR_ATTN_PASS_LORA_DOWN,+    LLM_TENSOR_ATTN_PASS_LORA_UP,+    LLM_TENSOR_ATTN_PASS_GATE,+    LLM_TENSOR_FFN_EXP_EMBD,     LLM_TENSOR_FFN_PRE_EXP_NORM,     LLM_TENSOR_FFN_POST_EXP_NORM,     LLM_TENSOR_FFN_RESID_ALPHA,

@@ -2536,6 +2536,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {         res = std::max<uint32_t>(n_tokens * 80, 64u * model.n_tensors());     } else if (model.arch == LLM_ARCH_QWEN3NEXT ||         model.arch == LLM_ARCH_KIMI_LINEAR ||+        model.arch == LLM_ARCH_METIS16 ||         model.arch == LLM_ARCH_BAILINGMOE3 ||         model.arch == LLM_ARCH_QWEN35 ||         model.arch == LLM_ARCH_QWEN35MOE ||

@@ -209,6 +209,7 @@ struct llama_hparams {     uint32_t ssm_d_state = 0;     uint32_t ssm_dt_rank = 0;     uint32_t ssm_n_group = 0;+    float    ssm_norm_eps = 0.0f; // 0 = use f_norm_rms_eps      // for MiniMax-Text-01 linear attention     uint32_t n_embd_head_la = 0;@@ -230,6 +231,13 @@ struct llama_hparams {     uint32_t n_hrm_l_cycles = 0;     bool     hrm_prefix_lm = false; +    // metis16 (passes over shared layers, depth memory)+    uint32_t metis_n_pass         = 0;+    uint32_t metis_n_mem          = 0;+    uint32_t metis_n_route        = 0;+    uint32_t metis_pass_lora_rank = 0;+    float    metis_cont_limit     = 0.0f;+     // limite     float    f_xsa_eps = 0.0f;     std::array<float, 3> final_logit_sigmoid_capping = {}; // scale, shift, temperature

@@ -244,6 +244,10 @@ void llama_model_saver::add_kv_from_model() {     add_kv(LLM_KV_HRM_H_CYCLES,                      hparams.n_hrm_h_cycles);     add_kv(LLM_KV_HRM_L_CYCLES,                      hparams.n_hrm_l_cycles);     add_kv(LLM_KV_HRM_PREFIX_LM,                     hparams.hrm_prefix_lm);+    add_kv(LLM_KV_METIS_PASS_COUNT,                  hparams.metis_n_pass);+    add_kv(LLM_KV_METIS_MEMORY_LENGTH,               hparams.metis_n_mem);+    add_kv(LLM_KV_METIS_ROUTE_LENGTH,                hparams.metis_n_route);+    add_kv(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT,    hparams.metis_cont_limit);     if (hparams.final_logit_sigmoid_capping[0] != 0.0f) {         add_kv(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, std::vector<float>(                 hparams.final_logit_sigmoid_capping.begin(), hparams.final_logit_sigmoid_capping.end()));@@ -316,6 +320,7 @@ void llama_model_saver::add_kv_from_model() {     add_kv(LLM_KV_ATTENTION_RECURRENT_LAYERS,        hparams.is_recr_impl, true);     add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT,      hparams.dsv4_o_group_count);     add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK,        hparams.dsv4_o_lora_rank);+    add_kv(LLM_KV_ATTENTION_PASS_LORA_RANK,          hparams.metis_pass_lora_rank);     add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, hparams.dsv4_compress_rope_base);     if (model->arch == LLM_ARCH_DEEPSEEK4 || hparams.dsv4_hc_mult > 0) {         // the loader requires one compress ratio per layer, including nextn layers@@ -412,6 +417,7 @@ void llama_model_saver::add_kv_from_model() {     add_kv(LLM_KV_SSM_TIME_STEP_RANK,                hparams.ssm_dt_rank);     add_kv(LLM_KV_SSM_GROUP_COUNT,                   hparams.ssm_n_group);     add_kv(LLM_KV_SSM_DT_B_C_RMS,                    hparams.ssm_dt_b_c_rms);+    add_kv(LLM_KV_SSM_NORM_EPS,                      hparams.ssm_norm_eps);      add_kv(LLM_KV_KDA_HEAD_DIM,                      hparams.n_embd_head_kda);     add_kv(LLM_KV_KDA_SAFE_GATE,                     hparams.kda_safe_gate);@@ -514,6 +520,32 @@ void llama_model_saver::add_tensors_from_model() {     add_tensor(model->cls_out_b);     add_tensor(model->cls_norm);     add_tensor(model->hrm_z_l_init);+    add_tensor(model->stream_embd);+    add_tensor(model->depth_mem_state_write);+    add_tensor(model->depth_mem_meta_write);+    add_tensor(model->depth_mem_entry_bias);+    add_tensor(model->depth_mem_query);+    add_tensor(model->depth_mem_query_b);+    add_tensor(model->depth_mem_key);+    add_tensor(model->depth_mem_key_b);+    add_tensor(model->depth_mem_value);+    add_tensor(model->depth_mem_value_b);+    add_tensor(model->depth_mem_out);+    add_tensor(model->depth_mem_out_b);+    add_tensor(model->depth_mem_route_state);+    add_tensor(model->depth_mem_route_retr);+    add_tensor(model->depth_mem_route_diff);+    add_tensor(model->depth_mem_route_hist);+    add_tensor(model->depth_mem_route_b);+    add_tensor(model->depth_mem_stream_gate);+    add_tensor(model->depth_mem_stream_gate_b);+    add_tensor(model->cont_hidden_state);+    add_tensor(model->cont_hidden_retr);+    add_tensor(model->cont_hidden_diff);+    add_tensor(model->cont_hidden_route);+    add_tensor(model->cont_hidden_b);+    add_tensor(model->cont_output);+    add_tensor(model->cont_output_b);     add_tensor(model->hc_head_fn);     add_tensor(model->hc_head_base);     add_tensor(model->hc_head_scale);

@@ -242,6 +242,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params             return new llama_model_nemotron_h(params);         case LLM_ARCH_NEMOTRON_H_MOE:             return new llama_model_nemotron_h_moe(params);+        case LLM_ARCH_METIS16:+            return new llama_model_metis16(params);         case LLM_ARCH_EXAONE:             return new llama_model_exaone(params);         case LLM_ARCH_EXAONE4:@@ -507,7 +509,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str     };      auto get_tensor_config = [&]() -> tensor_config {-        if (ud->model->arch == LLM_ARCH_HRM_TEXT) {+        if (ud->model->arch == LLM_ARCH_HRM_TEXT || ud->model->arch == LLM_ARCH_METIS16) {             // aliased cache slots cannot satisfy the meta-split invariants, so replicate all tensors             return {GGML_BACKEND_SPLIT_AXIS_MIRRORED, tensor, 0, 0};         }@@ -984,6 +986,7 @@ const char * llm_type_name(llm_type type) {         case LLM_TYPE_7B:            return "7B";         case LLM_TYPE_8B:            return "8B";         case LLM_TYPE_9B:            return "9B";+        case LLM_TYPE_10B:           return "10B";         case LLM_TYPE_11B:           return "11B";         case LLM_TYPE_12B:           return "12B";         case LLM_TYPE_13B:           return "13B";@@ -3555,6 +3558,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {         case LLM_ARCH_WAVTOKENIZER_DEC:         case LLM_ARCH_NEMOTRON_H:         case LLM_ARCH_NEMOTRON_H_MOE:+        case LLM_ARCH_METIS16:         case LLM_ARCH_KIMI_LINEAR:         case LLM_ARCH_KIMI_K3:         case LLM_ARCH_GLM5_NEXT:

@@ -76,6 +76,7 @@ enum llm_type {     LLM_TYPE_7B,     LLM_TYPE_8B,     LLM_TYPE_9B,+    LLM_TYPE_10B,     LLM_TYPE_11B,     LLM_TYPE_12B,     LLM_TYPE_13B,@@ -646,6 +647,18 @@ struct llama_layer {     // gemma4 layer output scale, reused for talkie embedding skip scale     struct ggml_tensor * out_scale = nullptr; +    // metis16 per-pass hyper-connection mixes, attention pass LoRA, expert route embeddings+    struct ggml_tensor * hc_attn_mix         = nullptr;+    struct ggml_tensor * hc_attn_read        = nullptr;+    struct ggml_tensor * hc_attn_write       = nullptr;+    struct ggml_tensor * hc_ffn_mix          = nullptr;+    struct ggml_tensor * hc_ffn_read         = nullptr;+    struct ggml_tensor * hc_ffn_write        = nullptr;+    struct ggml_tensor * attn_pass_lora_down = nullptr;+    struct ggml_tensor * attn_pass_lora_up   = nullptr;+    struct ggml_tensor * attn_pass_gate      = nullptr;+    struct ggml_tensor * ffn_exp_embd        = nullptr;+     // limite     struct ggml_tensor * attn_ve_gate     = nullptr;     struct ggml_tensor * attn_xsa_alpha   = nullptr;@@ -779,6 +792,34 @@ struct llama_model {     // hrm-text initial low-cycle state     struct ggml_tensor * hrm_z_l_init = nullptr; +    // metis16 hyper-connection stream embeddings, depth memory and continuation head+    struct ggml_tensor * stream_embd             = nullptr;+    struct ggml_tensor * depth_mem_state_write   = nullptr;+    struct ggml_tensor * depth_mem_meta_write    = nullptr;+    struct ggml_tensor * depth_mem_entry_bias    = nullptr;+    struct ggml_tensor * depth_mem_query         = nullptr;+    struct ggml_tensor * depth_mem_query_b       = nullptr;+    struct ggml_tensor * depth_mem_key           = nullptr;+    struct ggml_tensor * depth_mem_key_b         = nullptr;+    struct ggml_tensor * depth_mem_value         = nullptr;+    struct ggml_tensor * depth_mem_value_b       = nullptr;+    struct ggml_tensor * depth_mem_out           = nullptr;+    struct ggml_tensor * depth_mem_out_b         = nullptr;+    struct ggml_tensor * depth_mem_route_state   = nullptr;+    struct ggml_tensor * depth_mem_route_retr    = nullptr;+    struct ggml_tensor * depth_mem_route_diff    = nullptr;+    struct ggml_tensor * depth_mem_route_hist    = nullptr;+    struct ggml_tensor * depth_mem_route_b       = nullptr;+    struct ggml_tensor * depth_mem_stream_gate   = nullptr;+    struct ggml_tensor * depth_mem_stream_gate_b = nullptr;+    struct ggml_tensor * cont_hidden_state       = nullptr;+    struct ggml_tensor * cont_hidden_retr        = nullptr;+    struct ggml_tensor * cont_hidden_diff        = nullptr;+    struct ggml_tensor * cont_hidden_route       = nullptr;+    struct ggml_tensor * cont_hidden_b           = nullptr;+    struct ggml_tensor * cont_output             = nullptr;+    struct ggml_tensor * cont_output_b           = nullptr;+     // DeepSeek-V4     struct ggml_tensor * hc_head_fn    = nullptr;     struct ggml_tensor * hc_head_base  = nullptr;

@@ -361,6 +361,16 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param     quantize &= name.find("indexer.k_proj.weight") == std::string::npos;     quantize &= name.find("indexer.q_proj.weight") == std::string::npos; +    // metis16: small control tensors that steer every layer; the stream and hc_ tables (ne[0] = 4) would fall back to F16+    if (arch == LLM_ARCH_METIS16) {+        quantize &= name.rfind("depth_mem_", 0) == std::string::npos;+        quantize &= name.rfind("cont_", 0)      == std::string::npos;+        quantize &= name.find("stream_embd")    == std::string::npos;+        quantize &= name.find("attn_pass_lora") == std::string::npos;+        quantize &= name.find("ffn_exp_embd")   == std::string::npos;+        quantize &= name.find("hc_")            == std::string::npos;+    }+     // glm5-next     if (arch == LLM_ARCH_GLM5_NEXT) {         quantize &= name.find("hc_")                     == std::string::npos;

@@ -290,7 +290,11 @@ ggml_tensor * llm_build_mamba_base::build_mamba2_layer(llm_graph_input_rs * inp,         // grouped RMS norm         if (model.layers[il].ssm_norm) {             y = ggml_reshape_4d(ctx0, y, d_inner / n_group, n_group, n_seq_tokens, n_seqs);-            y = build_norm(y, model.layers[il].ssm_norm, NULL, LLM_NORM_RMS, il);+            if (hparams.ssm_norm_eps > 0.0f) {+                y = ggml_mul(ctx0, ggml_rms_norm(ctx0, y, hparams.ssm_norm_eps), model.layers[il].ssm_norm);+            } else {+                y = build_norm(y, model.layers[il].ssm_norm, NULL, LLM_NORM_RMS, il);+            }         }          y = ggml_reshape_2d(ctx0, y, d_inner, n_seq_tokens * n_seqs);

@@ -0,0 +1,401 @@+#include "models.h"++#include <cmath>++void llama_model_metis16::load_arch_hparams(llama_model_loader & ml) {+    ml.get_key(LLM_KV_SSM_CONV_KERNEL,    hparams.ssm_d_conv);+    ml.get_key(LLM_KV_SSM_INNER_SIZE,     hparams.ssm_d_inner);+    ml.get_key(LLM_KV_SSM_STATE_SIZE,     hparams.ssm_d_state);+    ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);+    ml.get_key(LLM_KV_SSM_GROUP_COUNT,    hparams.ssm_n_group);+    ml.get_key(LLM_KV_SSM_NORM_EPS,       hparams.ssm_norm_eps);++    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,       hparams.f_norm_rms_eps);++    ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);+    ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);+    ml.get_key(LLM_KV_MOE_LATENT_SIZE,                   hparams.moe_latent_size);++    ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT,            hparams.dsv4_hc_mult);+    ml.get_key(LLM_KV_METIS_PASS_COUNT,                  hparams.metis_n_pass);+    ml.get_key(LLM_KV_METIS_MEMORY_LENGTH,               hparams.metis_n_mem);+    ml.get_key(LLM_KV_METIS_ROUTE_LENGTH,                hparams.metis_n_route);+    ml.get_key(LLM_KV_ATTENTION_PASS_LORA_RANK,          hparams.metis_pass_lora_rank);+    ml.get_key(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT,    hparams.metis_cont_limit);++    n_pass     = hparams.metis_n_pass;+    n_stream   = hparams.dsv4_hc_mult;+    n_mem      = hparams.metis_n_mem;+    n_route    = hparams.metis_n_route;+    cont_limit = hparams.metis_cont_limit;++    if (n_pass < 1 || hparams.n_layer() % n_pass != 0) {+        throw std::runtime_error(format("metis16: %u layers do not split into %u passes", hparams.n_layer(), n_pass));+    }+    n_layer_phys = hparams.n_layer() / n_pass;++    bool has_attn = false;+    for (uint32_t il = 0; il < hparams.n_layer(); ++il) {+        if (hparams.n_head_kv(il) != hparams.n_head_kv(il % n_layer_phys)) {+            throw std::runtime_error(format("metis16: layer %u does not repeat the layout of layer %u", il, il % n_layer_phys));+        }+        hparams.is_recr_impl[il] = hparams.n_head_kv(il) == 0;+        has_attn = has_attn || !hparams.is_recr_impl[il];+    }+    if (!has_attn) {+        throw std::runtime_error("metis16: at least one attention layer is required");+    }++    switch (n_layer_phys) {+        case 12: type = LLM_TYPE_3B;  break; // Praxis+        case 20: type = LLM_TYPE_10B; break; // Logos+        default: type = LLM_TYPE_UNKNOWN;+    }+}++void llama_model_metis16::load_arch_tensors(llama_model_loader &) {+    LLAMA_LOAD_LOCALS;++    const int64_t d_conv     = hparams.ssm_d_conv;+    const int64_t d_inner    = hparams.ssm_d_inner;+    const int64_t d_state    = hparams.ssm_d_state;+    const int64_t n_ssm_head = hparams.ssm_dt_rank;+    const int64_t n_group    = hparams.ssm_n_group;+    const int64_t d_in_proj  = 2*d_inner + 2*n_group*d_state + n_ssm_head;+    const int64_t n_latent   = hparams.moe_latent_size;+    const int64_t n_ff_shexp = hparams.n_ff_shexp;+    const int64_t n_s = n_stream;+    const int64_t n_p = n_pass;+    const int64_t n_m = n_mem;+    const int64_t n_r = n_route;++    int64_t n_attn = 0;+    for (uint32_t il = 0; il < n_layer_phys; ++il) {+        n_attn += !hparams.is_recr(il);+    }++    tok_embd    = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD,  "weight"), {n_embd, n_vocab}, 0);+    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);+    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);+    if (output == NULL) {+        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);+    }++    stream_embd = create_tensor(tn(LLM_TENSOR_STREAM_EMBD, "weight"), {n_s, n_embd}, 0);++    depth_mem_state_write   = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STATE_WRITE, "weight"), {n_embd, n_m}, 0);+    depth_mem_meta_write    = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_META_WRITE,  "weight"), {n_r + 3, n_m}, 0);+    depth_mem_entry_bias    = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS,  "weight"), {n_m, n_p*(n_attn + 1)}, 0);+    depth_mem_query         = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_QUERY,       "weight"), {n_embd, n_m}, 0);+    depth_mem_query_b       = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_QUERY,       "bias"),   {n_m}, 0);+    depth_mem_key           = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_KEY,         "weight"), {n_m, n_m}, 0);+    depth_mem_key_b         = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_KEY,         "bias"),   {n_m}, 0);+    depth_mem_value         = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_VALUE,       "weight"), {n_m, n_m}, 0);+    depth_mem_value_b       = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_VALUE,       "bias"),   {n_m}, 0);+    depth_mem_out           = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_OUT,         "weight"), {n_m, n_embd}, 0);+    depth_mem_out_b         = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_OUT,         "bias"),   {n_embd}, 0);+    depth_mem_route_state   = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, "weight"), {n_embd, n_r}, 0);+    depth_mem_route_retr    = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_RETR,  "weight"), {n_embd, n_r}, 0);+    depth_mem_route_diff    = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF,  "weight"), {n_embd, n_r}, 0);+    depth_mem_route_hist    = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_HIST,  "weight"), {n_r, n_r}, 0);+    depth_mem_route_b       = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE,       "bias"),   {n_r}, 0);+    depth_mem_stream_gate   = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "weight"), {n_embd, n_s}, 0);+    depth_mem_stream_gate_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "bias"),   {n_s}, 0);++    cont_hidden_state = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_STATE, "weight"), {n_embd, n_r}, 0);+    cont_hidden_retr  = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_RETR,  "weight"), {n_embd, n_r}, 0);+    cont_hidden_diff  = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_DIFF,  "weight"), {n_embd, n_r}, 0);+    cont_hidden_route = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_ROUTE, "weight"), {n_r, n_r}, 0);+    cont_hidden_b     = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN,       "bias"),   {n_r}, 0);+    cont_output       = create_tensor(tn(LLM_TENSOR_CONT_OUTPUT,       "weight"), {n_r, 1}, 0);+    cont_output_b     = create_tensor(tn(LLM_TENSOR_CONT_OUTPUT,       "bias"),   {1}, 0);++    for (int il = 0; il < (int) n_layer_phys; ++il) {+        auto & layer = layers[il];++        layer.attn_norm     = create_tensor(tn(LLM_TENSOR_ATTN_NORM,     "weight", il), {n_embd}, 0);+        layer.hc_attn_mix   = create_tensor(tn(LLM_TENSOR_HC_ATTN_MIX,   "weight", il), {n_s, n_s, n_p}, 0);+        layer.hc_attn_read  = create_tensor(tn(LLM_TENSOR_HC_ATTN_READ,  "weight", il), {n_s, n_p}, 0);+        layer.hc_attn_write = create_tensor(tn(LLM_TENSOR_HC_ATTN_WRITE, "weight", il), {n_s, n_p}, 0);+        layer.hc_ffn_mix    = create_tensor(tn(LLM_TENSOR_HC_FFN_MIX,    "weight", il), {n_s, n_s, n_p}, 0);+        layer.hc_ffn_read   = create_tensor(tn(LLM_TENSOR_HC_FFN_READ,   "weight", il), {n_s, n_p}, 0);+        layer.hc_ffn_write  = create_tensor(tn(LLM_TENSOR_HC_FFN_WRITE,  "weight", il), {n_s, n_p}, 0);+        layer.ffn_norm      = create_tensor(tn(LLM_TENSOR_FFN_NORM,      "weight", il), {n_embd}, 0);++        if (hparams.is_recr(il)) {+            layer.ssm_in       = create_tensor(tn(LLM_TENSOR_SSM_IN,     "weight", il), {n_embd, d_in_proj}, 0);+            layer.ssm_conv1d   = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", il), {d_conv, d_inner + 2*n_group*d_state}, 0);+            layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "bias",   il), {d_inner + 2*n_group*d_state}, 0);+            layer.ssm_dt_b     = create_tensor(tn(LLM_TENSOR_SSM_DT,     "bias",   il), {n_ssm_head}, 0);+            layer.ssm_norm     = create_tensor(tn(LLM_TENSOR_SSM_NORM,   "weight", il), {d_inner / n_group, n_group}, 0);+            layer.ssm_out      = create_tensor(tn(LLM_TENSOR_SSM_OUT,    "weight", il), {d_inner, n_embd}, 0);++            // no "weight" suffix for these+            layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, il), {1, n_ssm_head}, 0);+            layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, il), {1, n_ssm_head}, 0);+        } else {+            const int64_t n_qkv = n_embd_head_k * (hparams.n_head(il) + 2*hparams.n_head_kv(il));+            const int64_t rank  = hparams.metis_pass_lora_rank;++            layer.wqkv                = create_tensor(tn(LLM_TENSOR_ATTN_QKV,            "weight", il), {n_embd, n_qkv}, 0);+            layer.wo                  = create_tensor(tn(LLM_TENSOR_ATTN_OUT,            "weight", il), {n_embd_head_k * hparams.n_head(il), n_embd}, 0);+            layer.attn_pass_lora_down = create_tensor(tn(LLM_TENSOR_ATTN_PASS_LORA_DOWN, "weight", il), {n_embd, rank}, 0);+            layer.attn_pass_lora_up   = create_tensor(tn(LLM_TENSOR_ATTN_PASS_LORA_UP,   "weight", il), {rank, n_qkv}, 0);+            layer.attn_pass_gate      = create_tensor(tn(LLM_TENSOR_ATTN_PASS_GATE,      "weight", il), {n_p}, 0);+        }++        const int64_t n_ff_exp = hparams.n_ff_exp(il);++        layer.ffn_latent_down  = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN,  "weight", il), {n_embd, n_latent}, 0);+        layer.ffn_latent_up    = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP,    "weight", il), {n_latent, n_embd}, 0);+        layer.ffn_gate_inp     = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,     "weight", il), {n_latent + n_r, n_expert}, 0);+        layer.ffn_gate_inp_b   = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,     "bias",   il), {n_expert}, 0);+        layer.ffn_exp_probs_b  = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B,  "bias",   il), {n_expert}, 0);+        layer.ffn_exp_embd     = create_tensor(tn(LLM_TENSOR_FFN_EXP_EMBD,     "weight", il), {n_r, n_expert}, 0);+        layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", il), {n_latent, 2*n_ff_exp, n_expert}, 0);+        layer.ffn_down_exps    = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS,    "weight", il), {n_ff_exp, n_latent, n_expert}, 0);+        layer.ffn_gate_shexp   = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP,   "weight", il), {n_latent, n_ff_shexp}, 0);+        layer.ffn_up_shexp     = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,     "weight", il), {n_latent, n_ff_shexp}, 0);+        layer.ffn_down_shexp   = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP,   "weight", il), {n_ff_shexp, n_latent}, 0);+    }++    // the virtual layers of later passes share the physical layer's weights+    for (int il = n_layer_phys; il < n_layer; ++il) {+        layers[il] = layers[il % n_layer_phys];+    }+}++std::unique_ptr<llm_graph_context> llama_model_metis16::build_arch_graph(const llm_graph_params & params) const {+    return std::make_unique<graph>(*this, params);+}++// streams are kept stream-innermost: [n_stream, n_embd, n_tokens]+ggml_tensor * llama_model_metis16::graph::stream_mean(ggml_tensor * x) const {+    ggml_tensor * mean = ggml_mean(ctx0, ggml_reshape_2d(ctx0, x, x->ne[0], x->ne[1]*x->ne[2]));+    return ggml_reshape_2d(ctx0, mean, x->ne[1], x->ne[2]);+}++// returns the normalized sublayer input+ggml_tensor * llama_model_metis16::graph::hc_read(ggml_tensor * x, ggml_tensor * mix, ggml_tensor * read, ggml_tensor * norm,+        int p, ggml_tensor ** mixed, int il) const {+    const int64_t n_s = x->ne[0];+    ggml_tensor * x2 = ggml_reshape_2d(ctx0, x, n_s, x->ne[1]*x->ne[2]);++    ggml_tensor * m = ggml_view_2d(ctx0, mix, n_s, n_s, mix->nb[1], p*mix->nb[2]);+    *mixed = ggml_mul_mat(ctx0, m, x2);++    ggml_tensor * r   = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, read, n_s, p*read->nb[1]), n_s, 1);+    ggml_tensor * src = ggml_reshape_2d(ctx0, ggml_mul_mat(ctx0, r, x2), x->ne[1], x->ne[2]);+    return build_norm(src, norm, NULL, LLM_NORM_RMS, il);+}++ggml_tensor * llama_model_metis16::graph::hc_write(ggml_tensor * mixed, ggml_tensor * write, ggml_tensor * update, int p) const {+    const int64_t n_s = mixed->ne[0];+    ggml_tensor * w   = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, write, n_s, p*write->nb[1]), 1, n_s);+    ggml_tensor * u   = ggml_reshape_2d(ctx0, update, 1, update->ne[0]*update->ne[1]);+    ggml_tensor * out = ggml_add(ctx0, mixed, ggml_mul_mat(ctx0, w, u));+    return ggml_reshape_3d(ctx0, out, n_s, update->ne[0], update->ne[1]);+}++// blends the retrieved entries into the streams; returns their summary+ggml_tensor * llama_model_metis16::graph::mem_read(const llama_model_metis16 & model, ggml_tensor ** x) const {+    const int64_t n_s = (*x)->ne[0];+    const int64_t n_m = model.n_mem;++    ggml_tensor * x_e = ggml_cont(ctx0, ggml_permute(ctx0, *x, 1, 0, 2, 3));++    ggml_tensor * q = ggml_mul_mat(ctx0, model.depth_mem_query, ggml_reshape_2d(ctx0, x_e, n_embd, n_s*n_tokens));+    q = ggml_reshape_3d(ctx0, ggml_add(ctx0, q, model.depth_mem_query_b), n_m, n_s, n_tokens);++    ggml_tensor * kq = ggml_mul_mat(ctx0, bank_k, q);+    kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f/sqrtf(float(n_m)), 0.0f);++    ggml_tensor * ret  = ggml_mul_mat(ctx0, bank_vt, kq);+    ggml_tensor * proj = ggml_mul_mat(ctx0, model.depth_mem_out, ggml_reshape_2d(ctx0, ret, n_m, n_s*n_tokens));+    proj = ggml_reshape_3d(ctx0, ggml_add(ctx0, proj, model.depth_mem_out_b), n_embd, n_s, n_tokens);++    ggml_tensor * gate = ggml_sum_rows(ctx0, ggml_mul(ctx0, x_e, model.depth_mem_stream_gate));+    gate = ggml_sigmoid(ctx0, ggml_add(ctx0, gate, ggml_reshape_2d(ctx0, model.depth_mem_stream_gate_b, 1, n_s)));++    ggml_tensor * proj_s = ggml_cont(ctx0, ggml_permute(ctx0, proj, 1, 0, 2, 3));+    *x = ggml_add(ctx0, *x, ggml_mul(ctx0, proj_s, ggml_reshape_3d(ctx0, gate, n_s, 1, n_tokens)));++    return stream_mean(proj_s);+}++void llama_model_metis16::graph::mem_write(const llama_model_metis16 & model, ggml_tensor * x, ggml_tensor * summary,+        ggml_tensor * entropy, ggml_tensor * confidence, ggml_tensor * cont, int entry) {+    const int64_t n_m = model.n_mem;++    if (cont == nullptr) {+        cont = ggml_scale(ctx0, confidence, 0.0f);+    }+    ggml_tensor * meta = ggml_concat(ctx0, ggml_concat(ctx0, summary, entropy, 0), ggml_concat(ctx0, confidence, cont, 0), 0);++    ggml_tensor * e = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_state_write, stream_mean(x)), ggml_mul_mat(ctx0, model.depth_mem_meta_write, meta));+    e = ggml_add(ctx0, e, ggml_view_1d(ctx0, model.depth_mem_entry_bias, n_m, entry*model.depth_mem_entry_bias->nb[1]));++    ggml_tensor * k = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_key, e), model.depth_mem_key_b);+    ggml_tensor * v = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_value, e), model.depth_mem_value_b);+    k = ggml_reshape_3d(ctx0, k, n_m, 1, n_tokens);+    v = ggml_reshape_3d(ctx0, v, 1, n_m, n_tokens);+    bank_k  = bank_k  ? ggml_concat(ctx0, bank_k,  k, 1) : k;+    bank_vt = bank_vt ? ggml_concat(ctx0, bank_vt, v, 0) : v;+}++ggml_tensor * llama_model_metis16::graph::route_features(const llama_model_metis16 & model, ggml_tensor * state,+        ggml_tensor * summary, ggml_tensor * diff, ggml_tensor * history) const {+    ggml_tensor * f = ggml_mul_mat(ctx0, model.depth_mem_route_state, state);+    if (summary) {+        f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_retr, summary));+    }+    if (diff) {+        f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_diff, diff));+    }+    if (history) {+        f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_hist, history));+    }+    return ggml_tanh(ctx0, ggml_add(ctx0, f, model.depth_mem_route_b));+}++llama_model_metis16::graph::graph(const llama_model_metis16 & model, const llm_graph_params & params) :+    llm_build_mamba_base(params) {+    const int64_t n_s      = model.n_stream;+    const int     n_p      = model.n_pass;+    const int     n_phys   = model.n_layer_phys;+    const int64_t n_r      = model.n_route;+    const int64_t n_hd     = hparams.n_embd_head_k();+    const int64_t n_exp    = hparams.n_expert;+    const int64_t n_used   = hparams.n_expert_used();++    ggml_tensor * inpL = build_inp_embd(model.tok_embd);+    auto * inp = build_inp_mem_hybrid();+    ggml_tensor * inp_out_ids = build_inp_out_ids();++    ggml_tensor * x = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, inpL, 1, n_embd, n_tokens), n_s, n_embd, n_tokens, 1);+    x = ggml_add(ctx0, x, model.stream_embd);+    cb(x, "streams_in", -1);++    ggml_tensor * prev_state = stream_mean(x);+    ggml_tensor * history    = nullptr; // routing history, [n_route, n_tokens]+    ggml_tensor * cont       = nullptr; // depth head reading of the last pass, [1, n_tokens]+    int n_entry = 0;++    for (int p = 0; p < n_p; ++p) {+        ggml_tensor * summ = nullptr;+        ggml_tensor * ent  = nullptr;+        ggml_tensor * conf = nullptr;++        for (int il_p = 0; il_p < n_phys; ++il_p) {+            const int il = p*n_phys + il_p;+            const auto & layer = model.layers[il];+            const bool is_attn  = !hparams.is_recr(il);+            const bool writes   = is_attn || il_p == n_phys - 1;++            ggml_tensor * retrieved = bank_k ? mem_read(model, &x) : nullptr;+            ggml_tensor * feats = route_features(model, stream_mean(x), retrieved, nullptr, history);++            ggml_tensor * mixed = nullptr;+            ggml_tensor * cur = hc_read(x, layer.hc_attn_mix, layer.hc_attn_read, layer.attn_norm, p, &mixed, il);+            if (is_attn) {+                ggml_tensor * qkv  = build_lora_mm(layer.wqkv, cur);+                ggml_tensor * lora = ggml_mul_mat(ctx0, layer.attn_pass_lora_up, ggml_mul_mat(ctx0, layer.attn_pass_lora_down, cur));+                qkv = ggml_add(ctx0, qkv, ggml_mul(ctx0, lora, ggml_view_1d(ctx0, layer.attn_pass_gate, 1, p*layer.attn_pass_gate->nb[0])));++                const int64_t n_head    = hparams.n_head(il);+                const int64_t n_head_kv = hparams.n_head_kv(il);+                const size_t  es        = ggml_element_size(qkv);+                ggml_tensor * Q = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head,    n_tokens, n_hd*es, qkv->nb[1], 0));+                ggml_tensor * K = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head_kv, n_tokens, n_hd*es, qkv->nb[1], n_hd*n_head*es));+                ggml_tensor * V = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head_kv, n_tokens, n_hd*es, qkv->nb[1], n_hd*(n_head + n_head_kv)*es));++                cur = build_attn(inp->get_attn(), layer.wo, nullptr, nullptr, Q, K, V, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_hd)), il);+            } else {+                cur = build_mamba2_layer(inp->get_recr(), cur, model, ubatch, il);+            }+            cb(cur, "mixer_out", il);+            x = hc_write(mixed, layer.hc_attn_write, cur, p);++            cur = hc_read(x, layer.hc_ffn_mix, layer.hc_ffn_read, layer.ffn_norm, p, &mixed, il);+            {+                ggml_tensor * latent = ggml_mul_mat(ctx0, layer.ffn_latent_down, cur);+                ggml_tensor * logits = ggml_mul_mat(ctx0, layer.ffn_gate_inp, ggml_concat(ctx0, latent, feats, 0));+                logits = ggml_add(ctx0, logits, layer.ffn_gate_inp_b);++                // the balance bias picks the experts; the weights come from the unbiased logits+                ggml_tensor * selected = ggml_argsort_top_k(ctx0, ggml_add(ctx0, logits, layer.ffn_exp_probs_b), n_used);++                ggml_tensor * moe_out = build_moe_ffn(latent, nullptr, nullptr, nullptr, layer.ffn_down_exps, nullptr,+                        n_exp, n_used, LLM_FFN_SILU, false, 0.0f, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT, il_p,+                        logits, layer.ffn_gate_up_exps, nullptr, nullptr, nullptr, selected);+                ggml_tensor * shexp = build_ffn(latent,+                        layer.ffn_up_shexp,   NULL, NULL,+                        layer.ffn_gate_shexp, NULL, NULL,+                        layer.ffn_down_shexp, NULL, NULL,+                        NULL, LLM_FFN_SILU, LLM_FFN_PAR, il_p);+                cur = ggml_mul_mat(ctx0, layer.ffn_latent_up, ggml_add(ctx0, moe_out, shexp));+                cb(cur, "ffn_out", il);++                // routing summary: the chosen experts' identity vectors blended by their weights+                ggml_tensor * logits3 = ggml_reshape_3d(ctx0, logits, 1, n_exp, n_tokens);+                ggml_tensor * weights = ggml_soft_max(ctx0, ggml_reshape_2d(ctx0, ggml_get_rows(ctx0, logits3, selected), n_used, n_tokens));+                ggml_tensor * rows    = ggml_get_rows(ctx0, layer.ffn_exp_embd, ggml_reshape_1d(ctx0, ggml_cont(ctx0, selected), n_used*n_tokens));+                rows = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, rows, n_r, n_used, n_tokens), 1, 0, 2, 3));+                summ = ggml_reshape_2d(ctx0, ggml_mul_mat(ctx0, rows, ggml_reshape_3d(ctx0, weights, n_used, 1, n_tokens)), n_r, n_tokens);++                if (writes) {+                    ggml_tensor * probs = ggml_soft_max(ctx0, logits);+                    ent  = ggml_sum_rows(ctx0, ggml_mul(ctx0, probs, ggml_log(ctx0, ggml_clamp(ctx0, probs, 1.0e-9f, 1.0f))));+                    ent  = ggml_scale(ctx0, ent, -1.0f);+                    conf = ggml_get_rows(ctx0, ggml_reshape_3d(ctx0, probs, 1, n_exp, n_tokens),+                            ggml_reshape_2d(ctx0, ggml_argmax(ctx0, probs), 1, n_tokens));+                    conf = ggml_reshape_2d(ctx0, conf, 1, n_tokens);+                }+            }+            x = hc_write(mixed, layer.hc_ffn_write, cur, p);+            cb(x, "streams", il);++            history = history ? ggml_add(ctx0, ggml_scale(ctx0, history, 0.5f), ggml_scale(ctx0, summ, 0.5f)) : ggml_scale(ctx0, summ, 0.5f);++            if (is_attn) {+                mem_write(model, x, summ, ent, conf, cont, n_entry++);+            }+        }++        // end of the pass: one more memory read, the depth head, and an entry recording where the pass left the token+        ggml_tensor * retrieved = mem_read(model, &x);+        ggml_tensor * state = stream_mean(x);+        ggml_tensor * diff  = ggml_sub(ctx0, state, prev_state);+        ggml_tensor * feats = route_features(model, state, retrieved, diff, history);++        ggml_tensor * h = ggml_mul_mat(ctx0, model.cont_hidden_state, state);+        h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_retr,  retrieved));+        h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_diff,  diff));+        h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_route, feats));+        h = ggml_silu(ctx0, ggml_add(ctx0, h, model.cont_hidden_b));++        ggml_tensor * logit = ggml_add(ctx0, ggml_mul_mat(ctx0, model.cont_output, h), model.cont_output_b);+        ggml_tensor * scale = ggml_clamp(ctx0, ggml_sqrt(ctx0, ggml_mean(ctx0, ggml_sqr(ctx0, state))), 1.0e-6f, INFINITY);+        ggml_tensor * innov = ggml_sqrt(ctx0, ggml_mean(ctx0, ggml_sqr(ctx0, diff)));+        logit = ggml_add(ctx0, logit, ggml_scale_bias(ctx0, ggml_tanh(ctx0, ggml_div(ctx0, innov, scale)), 1.0f, -0.5f));+        cont  = ggml_sigmoid(ctx0, ggml_clamp(ctx0, logit, -model.cont_limit, model.cont_limit));+        cb(cont, "continuation", p);++        mem_write(model, x, summ, ent, conf, cont, n_entry++);+        prev_state = state;+    }++    ggml_tensor * cur = stream_mean(x);+    if (inp_out_ids) {+        cur = ggml_get_rows(ctx0, cur, inp_out_ids);+    }++    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);+    cb(cur, "result_norm", -1);+    res->t_embd = cur;++    cur = build_lora_mm(model.output, cur);+    cb(cur, "result_output", -1);+    res->t_logits = cur;++}

@@ -1799,6 +1799,39 @@ struct llama_model_nemotron_h_moe : public llama_model_nemotron_h { };  +struct llama_model_metis16 : public llama_model_base {+    llama_model_metis16(const struct llama_model_params & params) : llama_model_base(params) {}+    void load_arch_hparams(llama_model_loader & ml) override;+    void load_arch_tensors(llama_model_loader & ml) override;++    uint32_t n_pass       = 1;+    uint32_t n_stream     = 1;+    uint32_t n_mem        = 0;+    uint32_t n_route      = 0;+    uint32_t n_layer_phys = 0;+    float    cont_limit   = 0.0f;++    struct graph : public llm_build_mamba_base {+        graph(const llama_model_metis16 & model, const llm_graph_params & params);++        // memory bank of the current forward: keys [n_mem, n_entry, n_tokens], values [n_entry, n_mem, n_tokens]+        ggml_tensor * bank_k  = nullptr;+        ggml_tensor * bank_vt = nullptr;++        ggml_tensor * stream_mean(ggml_tensor * x) const;+        ggml_tensor * hc_read(ggml_tensor * x, ggml_tensor * mix, ggml_tensor * read, ggml_tensor * norm, int p, ggml_tensor ** mixed, int il) const;+        ggml_tensor * hc_write(ggml_tensor * mixed, ggml_tensor * write, ggml_tensor * update, int p) const;+        ggml_tensor * mem_read(const llama_model_metis16 & model, ggml_tensor ** x) const;+        void          mem_write(const llama_model_metis16 & model, ggml_tensor * x, ggml_tensor * summary,+                                ggml_tensor * entropy, ggml_tensor * confidence, ggml_tensor * cont, int entry);+        ggml_tensor * route_features(const llama_model_metis16 & model, ggml_tensor * state, ggml_tensor * summary,+                                     ggml_tensor * diff, ggml_tensor * history) const;+    };++    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;+};++ struct llama_model_exaone : public llama_model_base {     llama_model_exaone(const struct llama_model_params & params) : llama_model_base(params) {}     void load_arch_hparams(llama_model_loader & ml) override;

@@ -179,6 +179,8 @@ maple               ,0   ,any     ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, mellum              ,0   ,any     ,MUL+ADD                     ,      4 mellum              ,0   ,any     ,RMS_NORM+MUL                ,      9 mellum              ,0   ,any     ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV,      2+metis16             ,0   ,any     ,MUL+ADD                     ,      8+metis16             ,0   ,any     ,RMS_NORM+MUL                ,     11 mimo2               ,0   ,any     ,MUL+ADD                     ,      4 mimo2               ,0   ,any     ,RMS_NORM+MUL                ,      5 minicpm             ,0   ,any     ,RMS_NORM+MUL                ,      5

@@ -291,6 +291,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {         }     } else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {         n_layer = 3;+    } else if (arch == LLM_ARCH_METIS16) {+        n_layer = 4; // 2 passes over 2 physical layers, one attention and one Mamba-2     } else if (arch == LLM_ARCH_CHAMELEON) {         n_vocab = 10240;     } else if (arch == LLM_ARCH_QWEN3TTS) {@@ -361,6 +363,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {             ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head_per_layer);         }         ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer);+    } else if (arch == LLM_ARCH_METIS16) {+        // the layer types repeat in every pass+        std::vector<uint32_t> n_head_kv_per_layer;+        n_head_kv_per_layer.reserve(n_layer);+        for (uint32_t il = 0; il < n_layer; il++) {+            n_head_kv_per_layer.push_back(il % 2 == 0 ? n_head_kv : 0);+        }+        ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);+        ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_kv_per_layer);     } else {         ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);         ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) ? uint32_t(1) : n_head_kv);@@ -610,6 +621,17 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {         ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f);     } +    if (arch == LLM_ARCH_METIS16) {+        ms.add_kv(LLM_KV_METIS_PASS_COUNT,               uint32_t(2));+        ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT,         uint32_t(4));+        ms.add_kv(LLM_KV_METIS_MEMORY_LENGTH,            uint32_t(64));+        ms.add_kv(LLM_KV_METIS_ROUTE_LENGTH,             uint32_t(32));+        ms.add_kv(LLM_KV_ATTENTION_PASS_LORA_RANK,       uint32_t(8));+        ms.add_kv(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, 4.0f);+        ms.add_kv(LLM_KV_SSM_NORM_EPS,                   1e-5f);+        ms.add_kv(LLM_KV_MOE_LATENT_SIZE,                n_embd/2);+    }+     if (arch == LLM_ARCH_LIMITE) {         ms.add_kv(LLM_KV_ATTENTION_SCALE,             0.1f);         ms.add_kv(LLM_KV_ATTENTION_XSA_EPS,           1e-4f);@@ -965,6 +987,7 @@ static bool moe_mandatory(const llm_arch arch) {         case LLM_ARCH_MAPLE:         case LLM_ARCH_MUSE_SPARK:         case LLM_ARCH_KOLIBRI1:+        case LLM_ARCH_METIS16:             return true;         default:             return false;