test-llama-archs : generate dummy test vocab

@@ -77,6 +77,7 @@ extern "C" {         LLAMA_VOCAB_TYPE_UGM    = 4, // T5 tokenizer based on Unigram         LLAMA_VOCAB_TYPE_RWKV   = 5, // RWKV tokenizer based on greedy tokenization         LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming+        LLAMA_VOCAB_TYPE_TEST   = 7, // Dummy tokenizer for testing: rolling hash of fixed-size chunks -> tokens, tokens -> hex     };      enum llama_rope_type {

@@ -395,13 +395,13 @@ void llama_model_saver::add_kv_from_model() {     add_kv(LLM_KV_TOKENIZER_SCORES,                  scores);     add_kv(LLM_KV_TOKENIZER_MERGES,                  vocab.get_bpe_merges());     // FIXME llama_token is type i32 but when reading in a GGUF file u32 is expected, not an issue for writing though-    add_kv(LLM_KV_TOKENIZER_BOS_ID,                  uint32_t(vocab.token_bos()));-    add_kv(LLM_KV_TOKENIZER_EOS_ID,                  uint32_t(vocab.token_eos()));-    add_kv(LLM_KV_TOKENIZER_EOT_ID,                  uint32_t(vocab.token_eot()));-    add_kv(LLM_KV_TOKENIZER_EOM_ID,                  uint32_t(vocab.token_eom()));-    add_kv(LLM_KV_TOKENIZER_UNK_ID,                  uint32_t(vocab.token_unk()));-    add_kv(LLM_KV_TOKENIZER_SEP_ID,                  uint32_t(vocab.token_sep()));-    add_kv(LLM_KV_TOKENIZER_PAD_ID,                  uint32_t(vocab.token_pad()));+    if (vocab.token_bos()  != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_BOS_ID, uint32_t(vocab.token_bos()));  }+    if (vocab.token_eos()  != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOS_ID, uint32_t(vocab.token_eos()));  }+    if (vocab.token_eot()  != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOT_ID, uint32_t(vocab.token_eot()));  }+    if (vocab.token_eom()  != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOM_ID, uint32_t(vocab.token_eom()));  }+    if (vocab.token_unk()  != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_UNK_ID, uint32_t(vocab.token_unk()));  }+    if (vocab.token_sep()  != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_SEP_ID, uint32_t(vocab.token_sep()));  }+    if (vocab.token_pad()  != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_PAD_ID, uint32_t(vocab.token_pad()));  }     // add_kv(LLM_KV_TOKENIZER_CLS_ID,                  uint32_t(vocab.token_bos())); // deprecated     // add_kv(LLM_KV_TOKENIZER_MASK_ID,                 ???);     add_kv(LLM_KV_TOKENIZER_ADD_BOS,                 vocab.get_add_bos());@@ -412,12 +412,12 @@ void llama_model_saver::add_kv_from_model() {     add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP,    vocab.get_precompiled_charsmap());     // add_kv(LLM_KV_TOKENIZER_HF_JSON,                 ???);     // add_kv(LLM_KV_TOKENIZER_RWKV,                    ???);-    add_kv(LLM_KV_TOKENIZER_FIM_PRE_ID,              uint32_t(vocab.token_fim_pre()));-    add_kv(LLM_KV_TOKENIZER_FIM_SUF_ID,              uint32_t(vocab.token_fim_suf()));-    add_kv(LLM_KV_TOKENIZER_FIM_MID_ID,              uint32_t(vocab.token_fim_mid()));-    add_kv(LLM_KV_TOKENIZER_FIM_PAD_ID,              uint32_t(vocab.token_fim_pad()));-    add_kv(LLM_KV_TOKENIZER_FIM_REP_ID,              uint32_t(vocab.token_fim_rep()));-    add_kv(LLM_KV_TOKENIZER_FIM_SEP_ID,              uint32_t(vocab.token_fim_sep()));+    if (vocab.token_fim_pre() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_PRE_ID, uint32_t(vocab.token_fim_pre())); }+    if (vocab.token_fim_suf() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_SUF_ID, uint32_t(vocab.token_fim_suf())); }+    if (vocab.token_fim_mid() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_MID_ID, uint32_t(vocab.token_fim_mid())); }+    if (vocab.token_fim_pad() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_PAD_ID, uint32_t(vocab.token_fim_pad())); }+    if (vocab.token_fim_rep() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_REP_ID, uint32_t(vocab.token_fim_rep())); }+    if (vocab.token_fim_sep() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_SEP_ID, uint32_t(vocab.token_fim_sep())); }      // TODO: implement LoRA support     // add_kv(LLM_KV_ADAPTER_TYPE,                      ???);

@@ -2087,6 +2087,16 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {             special_unk_id = LLAMA_TOKEN_NULL;             special_sep_id = LLAMA_TOKEN_NULL;             special_pad_id = LLAMA_TOKEN_NULL;+        } else if (tokenizer_model == "test") {+            type = LLAMA_VOCAB_TYPE_TEST;++            // default special tokens+            special_bos_id  = LLAMA_TOKEN_NULL;+            special_eos_id  = LLAMA_TOKEN_NULL;+            special_unk_id  = LLAMA_TOKEN_NULL;+            special_sep_id  = LLAMA_TOKEN_NULL;+            special_pad_id  = LLAMA_TOKEN_NULL;+            special_mask_id = LLAMA_TOKEN_NULL;         } else if (tokenizer_model == "plamo2") {             type = LLAMA_VOCAB_TYPE_PLAMO2; @@ -3134,6 +3144,7 @@ std::string llama_vocab::impl::type_name() const{         case LLAMA_VOCAB_TYPE_UGM:    return "UGM";         case LLAMA_VOCAB_TYPE_RWKV:   return "RWKV";         case LLAMA_VOCAB_TYPE_PLAMO2: return "PLaMo2";+        case LLAMA_VOCAB_TYPE_TEST:   return "TEST";         default:                      return "unknown";     } }@@ -3222,6 +3233,9 @@ void llama_vocab::impl::init_tokenizer(enum llama_vocab_type type) {         case LLAMA_VOCAB_TYPE_PLAMO2:             tokenizer = std::make_unique<llm_tokenizer_plamo2>(vocab);             break;+        case LLAMA_VOCAB_TYPE_TEST:+            tokenizer = std::make_unique<llm_tokenizer>();+            break;         default:             GGML_ABORT("unsupported vocab type");     }@@ -3595,6 +3609,42 @@ std::vector<llama_token> llama_vocab::impl::tokenize(                     }                 }             } break;+        case LLAMA_VOCAB_TYPE_TEST:+            {+                const uint32_t n_vocab = vocab.n_tokens();+                constexpr size_t chunk_size = 5;++                // reserve output to avoid repeated reallocations+                size_t n_tokens = 0;+                for (const auto & fragment : fragment_buffer) {+                    if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {+                        n_tokens += (fragment.length + chunk_size - 1) / chunk_size;+                    } else {+                        ++n_tokens;+                    }+                }+                output.reserve(output.size() + n_tokens);++                for (const auto & fragment : fragment_buffer) {+                    if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {+                        const auto & text = fragment.raw_text;+                        const size_t begin = fragment.offset;+                        const size_t end   = begin + fragment.length;+                        size_t pos = begin;+                        while (pos < end) {+                            const size_t n = std::min(chunk_size, end - pos);+                            uint64_t hash = 0;+                            for (size_t i = 0; i < n; ++i) {+                                hash = hash*31 + (uint8_t) text[pos + i];+                            }+                            output.push_back((llama_token)(hash % n_vocab));+                            pos += n;+                        }+                    } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)+                        output.push_back(fragment.token);+                    }+                }+            } break;         case LLAMA_VOCAB_TYPE_NONE:             GGML_ABORT("fatal error");     }@@ -3693,6 +3743,11 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t                 memcpy(buf, result.data(), result.size());                 return (int)result.size();             }+            case LLAMA_VOCAB_TYPE_TEST: {+                // tokens -> text: simply stringify the token id in hex+                std::string result = format("%x", token);+                return _try_copy(result.data(), result.size());+            }             case LLAMA_VOCAB_TYPE_PLAMO2: {                 // PLaMo-2 uses similar token handling as BPE/SPM                 if (vocab.is_byte(token)) {@@ -3963,6 +4018,9 @@ llama_token llama_vocab::byte_to_token(uint8_t ch) const {             snprintf(hex_str, sizeof(hex_str), "<0x%02X>", ch);             return pimpl->token_to_id.at(hex_str);         }+        case LLAMA_VOCAB_TYPE_TEST:+            // TEST tokens have no byte-level mapping+            return LLAMA_TOKEN_NULL;         default:             GGML_ABORT("fatal error");     }

@@ -338,7 +338,19 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {         ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f);     } -    ms.add_kv(LLM_KV_TOKENIZER_MODEL,         "no_vocab");+    // dummy tokenizer: token ids are derived from fixed-size chunks and detokenized as hex ids+    {+        std::vector<std::string> tokenizer_list(n_vocab);+        std::vector<float>       tokenizer_scores(n_vocab, 0.0f);++        ms.add_kv(LLM_KV_TOKENIZER_MODEL,         "test");+        for (uint32_t i = 0; i < n_vocab; i++) {+            tokenizer_list[i] = "tok_" + std::to_string(i);+        }+        ms.add_kv(LLM_KV_TOKENIZER_LIST,   tokenizer_list);+        ms.add_kv(LLM_KV_TOKENIZER_SCORES, tokenizer_scores);+    }+     // ms.add_kv(LLM_KV_DENSE_2_FEAT_OUT,     n_embd);     // ms.add_kv(LLM_KV_DENSE_3_FEAT_IN,      n_embd);