ggml-org/llama.cpp · #32053
chat: allow TTS model to provide custom text normalizer via jinja
common/CMakeLists.txt2 + / 0 −
@@ -66,6 +66,8 @@ add_library(${TARGET} chat-diff-analyzer.cpp chat-peg-parser.cpp chat-peg-parser.h+ chat-tts-norm.cpp+ chat-tts-norm.h chat.cpp chat.h common.cppcommon/chat-tts-norm.cppadded864 + / 0 −
@@ -0,0 +1,864 @@+#include "chat-tts-norm.h"++#include "jinja/runtime.h"+#include "jinja/value.h"++#include <algorithm>+#include <climits>+#include <stdexcept>+#include <string>+#include <vector>++// TTS normalization patterns, see docs/tts-norm-patterns.md for the specification+// keep this file in sync with the python reference: scripts/chat_tts_norm.py++using namespace jinja;++//+// character classes+// they are defined with explicit codepoint ranges, independent of any unicode database version,+// so that the C++ and python implementations agree on every codepoint+//++struct tts_norm_range {+ uint32_t lo;+ uint32_t hi;+};++static const tts_norm_range TTS_NORM_SPACE[] = {+ {0x0009, 0x000D}, {0x0020, 0x0020}, {0x0085, 0x0085}, {0x00A0, 0x00A0}, {0x1680, 0x1680},+ {0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F}, {0x205F, 0x205F}, {0x3000, 0x3000},+};++static const tts_norm_range TTS_NORM_PUNCT[] = {+ {0x0021, 0x002F}, {0x003A, 0x0040}, {0x005B, 0x0060}, {0x007B, 0x007E},+ {0x00A1, 0x00A9}, {0x00AB, 0x00B4}, {0x00B6, 0x00B9}, {0x00BB, 0x00BF}, {0x00D7, 0x00D7}, {0x00F7, 0x00F7},+ {0x2010, 0x2027}, {0x2030, 0x205E}, {0x20A0, 0x20CF}, {0x2100, 0x218F}, {0x2190, 0x2BFF},+ {0x3001, 0x3003}, {0x3008, 0x3020}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30FB, 0x30FB},+ {0xFE10, 0xFE19}, {0xFE30, 0xFE4F}, {0xFE50, 0xFE6B},+ {0xFF01, 0xFF0F}, {0xFF1A, 0xFF20}, {0xFF3B, 0xFF40}, {0xFF5B, 0xFF65},+ {0x1F000, 0x1FAFF},+};++static const tts_norm_range TTS_NORM_CONTROL[] = {+ {0x0000, 0x001F}, {0x007F, 0x009F}, {0x200B, 0x200F}, {0x2060, 0x206F}, {0xFEFF, 0xFEFF},+};++static const tts_norm_range TTS_NORM_HAN[] = {+ {0x2E80, 0x2FDF}, {0x3005, 0x3005}, {0x3007, 0x3007}, {0x3021, 0x3029}, {0x3038, 0x303B},+ {0x3400, 0x4DBF}, {0x4E00, 0x9FFF}, {0xF900, 0xFAFF}, {0x20000, 0x323AF},+};++template<size_t N>+static bool tts_norm_in(const tts_norm_range (&ranges)[N], uint32_t c) {+ for (const auto & r : ranges) {+ if (c >= r.lo && c <= r.hi) {+ return true;+ }+ }+ return false;+}++static bool tts_norm_is_alpha(uint32_t c) {+ if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {+ return true;+ }+ if (c == 0xAA || c == 0xB5 || c == 0xBA) {+ return true;+ }+ return c >= 0xC0 && !tts_norm_in(TTS_NORM_SPACE, c) && !tts_norm_in(TTS_NORM_PUNCT, c) && !tts_norm_in(TTS_NORM_CONTROL, c);+}++// kind is one of "dspawulhc"+static bool tts_norm_class_match(char kind, uint32_t c) {+ switch (kind) {+ case 'd': return c >= '0' && c <= '9';+ case 's': return tts_norm_in(TTS_NORM_SPACE, c);+ case 'p': return tts_norm_in(TTS_NORM_PUNCT, c);+ case 'a': return tts_norm_is_alpha(c);+ case 'w': return tts_norm_is_alpha(c) || (c >= '0' && c <= '9');+ case 'u': return c >= 'A' && c <= 'Z';+ case 'l': return c >= 'a' && c <= 'z';+ case 'h': return tts_norm_in(TTS_NORM_HAN, c);+ case 'c': return tts_norm_in(TTS_NORM_CONTROL, c);+ }+ return false;+}++static bool tts_norm_is_class_letter(uint32_t c) {+ if (c >= 'A' && c <= 'Z') {+ c += 'a' - 'A';+ }+ return c < 0x80 && std::string("dspawulhc").find(static_cast<char>(c)) != std::string::npos;+}++static uint32_t tts_norm_ascii_lower(uint32_t c) {+ return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c;+}++static uint32_t tts_norm_ascii_swapcase(uint32_t c) {+ if (c >= 'A' && c <= 'Z') return c + ('a' - 'A');+ if (c >= 'a' && c <= 'z') return c - ('a' - 'A');+ return c;+}++static bool tts_norm_is_ascii_alnum(uint32_t c) {+ return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');+}++// invalid bytes are decoded as U+FFFD, one per byte; offs[i] is the byte offset of codepoint i, offs[n] = text size+static void tts_norm_decode_utf8(const std::string & s, std::vector<uint32_t> & cpts, std::vector<size_t> & offs) {+ cpts.clear();+ offs.clear();+ size_t i = 0;+ const size_t n = s.size();+ while (i < n) {+ offs.push_back(i);+ const unsigned char c = static_cast<unsigned char>(s[i]);+ uint32_t cp = 0;+ size_t len = 0;+ if (c < 0x80) {+ cp = c; len = 1;+ } else if ((c & 0xE0) == 0xC0) {+ cp = c & 0x1F; len = 2;+ } else if ((c & 0xF0) == 0xE0) {+ cp = c & 0x0F; len = 3;+ } else if ((c & 0xF8) == 0xF0) {+ cp = c & 0x07; len = 4;+ }+ bool ok = len > 0 && i + len <= n;+ for (size_t k = 1; ok && k < len; k++) {+ const unsigned char cc = static_cast<unsigned char>(s[i + k]);+ if ((cc & 0xC0) != 0x80) {+ ok = false;+ } else {+ cp = (cp << 6) | (cc & 0x3F);+ }+ }+ if (!ok) {+ cpts.push_back(0xFFFD);+ i += 1;+ } else {+ cpts.push_back(cp);+ i += len;+ }+ }+ offs.push_back(n);+}++//+// AST+//++enum class tts_norm_op {+ LITERAL, // one codepoint+ SET, // one codepoint from a set; class escapes like %d are sets with a single class+ ANY, // any codepoint+ START, // ^+ END, // $+ BOUNDARY, // %b+ GROUP, // (...), (?:...), (?<name>...)+ LOOKAHEAD, // (?=...), (?!...)+ LOOKBEHIND, // (?<=c), (?<!c) - exactly one codepoint+};++struct tts_norm_class {+ char kind;+ bool neg;+};++struct tts_norm_set {+ bool neg = false;+ std::vector<std::pair<uint32_t, uint32_t>> ranges;+ std::vector<tts_norm_class> classes;+};++struct tts_norm_item;+using tts_norm_seq = std::vector<tts_norm_item>;++struct tts_norm_pattern_node {+ tts_norm_op op = tts_norm_op::GROUP;+ uint32_t cp = 0; // LITERAL+ tts_norm_set set; // SET+ std::vector<tts_norm_seq> alts; // GROUP, LOOKAHEAD; LOOKBEHIND holds a single item+ int capture = -1; // GROUP: capture index, -1 for non-capturing+ bool neg = false; // LOOKAHEAD, LOOKBEHIND+};++static constexpr int TTS_NORM_INF = INT_MAX;+static constexpr int TTS_NORM_MAX_REPEAT = 1000;+static constexpr size_t TTS_NORM_MAX_PATTERN_LEN = 4096;++struct tts_norm_item {+ tts_norm_pattern_node atom;+ int min = 1;+ int max = 1;+};++static bool tts_norm_is_zero_width(tts_norm_op op) {+ return op == tts_norm_op::START || op == tts_norm_op::END || op == tts_norm_op::BOUNDARY+ || op == tts_norm_op::LOOKAHEAD || op == tts_norm_op::LOOKBEHIND;+}++//+// parser+//++struct tts_norm_parser {+ std::vector<uint32_t> s;+ size_t p = 0;+ size_t & n_caps;++ tts_norm_parser(std::vector<uint32_t> src, size_t & n_caps) : s(std::move(src)), n_caps(n_caps) {}++ [[noreturn]] void fail(const std::string & msg) const {+ throw std::runtime_error("invalid TTS norm pattern: " + msg + " at position " + std::to_string(p));+ }++ bool eof() const { return p >= s.size(); }+ uint32_t peek() const { return s[p]; }+ bool peek_is(uint32_t c) const { return !eof() && s[p] == c; }++ void expect(uint32_t c, const char * what) {+ if (!peek_is(c)) {+ fail(std::string("expected ") + what);+ }+ p++;+ }++ std::vector<tts_norm_seq> parse_alts() {+ std::vector<tts_norm_seq> alts;+ alts.push_back(parse_seq());+ while (peek_is('|')) {+ p++;+ alts.push_back(parse_seq());+ }+ return alts;+ }++ tts_norm_seq parse_seq() {+ tts_norm_seq seq;+ while (!eof() && peek() != '|' && peek() != ')') {+ tts_norm_item item;+ item.atom = parse_atom();+ parse_quantifier(item);+ seq.push_back(std::move(item));+ }+ return seq;+ }++ int parse_int() {+ if (eof() || peek() < '0' || peek() > '9') {+ fail("expected number in quantifier");+ }+ int v = 0;+ while (!eof() && peek() >= '0' && peek() <= '9') {+ v = v * 10 + static_cast<int>(peek() - '0');+ if (v > TTS_NORM_MAX_REPEAT) {+ fail("repetition count exceeds " + std::to_string(TTS_NORM_MAX_REPEAT));+ }+ p++;+ }+ return v;+ }++ void parse_quantifier(tts_norm_item & item) {+ if (eof()) {+ return;+ }+ const uint32_t c = peek();+ if (c != '?' && c != '*' && c != '+' && c != '{') {+ return;+ }+ if (tts_norm_is_zero_width(item.atom.op)) {+ fail("zero-width assertion cannot be repeated");+ }+ p++;+ if (c == '?') {+ item.min = 0; item.max = 1;+ } else if (c == '*') {+ item.min = 0; item.max = TTS_NORM_INF;+ } else if (c == '+') {+ item.min = 1; item.max = TTS_NORM_INF;+ } else {+ item.min = parse_int();+ item.max = item.min;+ if (peek_is(',')) {+ p++;+ item.max = peek_is('}') ? TTS_NORM_INF : parse_int();+ }+ expect('}', "'}' to close quantifier");+ if (item.max < item.min) {+ fail("quantifier max is smaller than min");+ }+ }+ if (!eof() && (peek() == '?' || peek() == '*' || peek() == '+' || peek() == '{')) {+ fail("multiple quantifiers");+ }+ }++ tts_norm_pattern_node parse_group() {+ // '(' already consumed+ tts_norm_pattern_node node;+ node.op = tts_norm_op::GROUP;+ if (peek_is('?')) {+ p++;+ if (eof()) {+ fail("unterminated group");+ }+ const uint32_t c = peek();+ if (c == ':') {+ p++;+ } else if (c == '=' || c == '!') {+ p++;+ node.op = tts_norm_op::LOOKAHEAD;+ node.neg = c == '!';+ } else if (c == '<' && p + 1 < s.size() && (s[p + 1] == '=' || s[p + 1] == '!')) {+ node.op = tts_norm_op::LOOKBEHIND;+ node.neg = s[p + 1] == '!';+ p += 2;+ tts_norm_item item;+ if (eof() || peek() == ')') {+ fail("lookbehind must contain exactly one codepoint matcher");+ }+ item.atom = parse_atom();+ if (item.atom.op != tts_norm_op::LITERAL && item.atom.op != tts_norm_op::SET && item.atom.op != tts_norm_op::ANY) {+ fail("lookbehind must contain exactly one codepoint matcher");+ }+ if (!peek_is(')')) {+ fail("lookbehind must contain exactly one codepoint matcher");+ }+ p++;+ node.alts.push_back(tts_norm_seq{});+ node.alts[0].push_back(std::move(item));+ return node;+ } else if (c == 'i') {+ fail("(?i) is only allowed at the start of the pattern");+ } else {+ fail("unknown group type");+ }+ } else {+ node.capture = static_cast<int>(n_caps++);+ }+ node.alts = parse_alts();+ expect(')', "')' to close group");+ return node;+ }++ // after '%' inside or outside of a set+ // returns true if it is a class (stored in cls), false if it is a literal (stored in lit)+ bool parse_escape(bool in_set, tts_norm_class & cls, uint32_t & lit, bool & boundary) {+ boundary = false;+ if (eof()) {+ fail("dangling '%'");+ }+ const uint32_t e = s[p++];+ if (tts_norm_is_class_letter(e)) {+ cls.kind = static_cast<char>(tts_norm_ascii_lower(e));+ cls.neg = e >= 'A' && e <= 'Z';+ return true;+ }+ if (e == 'b' && !in_set) {+ boundary = true;+ return false;+ }+ if (tts_norm_is_ascii_alnum(e)) {+ p--;+ fail(std::string("unknown escape '%") + static_cast<char>(e) + "'");+ }+ lit = e;+ return false;+ }++ tts_norm_pattern_node parse_set() {+ // '[' already consumed+ tts_norm_pattern_node node;+ node.op = tts_norm_op::SET;+ if (peek_is('^')) {+ node.set.neg = true;+ p++;+ }+ bool empty = true;+ while (true) {+ if (eof()) {+ fail("unterminated set");+ }+ if (peek() == ']') {+ p++;+ break;+ }+ empty = false;+ uint32_t lo = 0;+ if (peek() == '%') {+ p++;+ tts_norm_class cls{};+ bool boundary = false;+ if (parse_escape(true, cls, lo, boundary)) {+ node.set.classes.push_back(cls);+ continue;+ }+ } else {+ lo = s[p++];+ }+ uint32_t hi = lo;+ if (peek_is('-') && p + 1 < s.size() && s[p + 1] != ']') {+ p++;+ if (peek() == '%') {+ p++;+ tts_norm_class cls{};+ bool boundary = false;+ if (parse_escape(true, cls, hi, boundary)) {+ fail("character class cannot be used in a range");+ }+ } else {+ hi = s[p++];+ }+ if (hi < lo) {+ fail("invalid range in set");+ }+ }+ node.set.ranges.emplace_back(lo, hi);+ }+ if (empty) {+ fail("empty set");+ }+ return node;+ }++ tts_norm_pattern_node parse_atom() {+ const uint32_t c = s[p++];+ tts_norm_pattern_node node;+ switch (c) {+ case '(':+ return parse_group();+ case '[':+ return parse_set();+ case '.':+ node.op = tts_norm_op::ANY;+ return node;+ case '^':+ node.op = tts_norm_op::START;+ return node;+ case '$':+ node.op = tts_norm_op::END;+ return node;+ case '%': {+ tts_norm_class cls{};+ uint32_t lit = 0;+ bool boundary = false;+ if (parse_escape(false, cls, lit, boundary)) {+ node.op = tts_norm_op::SET;+ node.set.classes.push_back(cls);+ } else if (boundary) {+ node.op = tts_norm_op::BOUNDARY;+ } else {+ node.op = tts_norm_op::LITERAL;+ node.cp = lit;+ }+ return node;+ }+ case '?':+ case '*':+ case '+':+ case '{':+ p--;+ fail("nothing to repeat");+ case ']':+ case '}':+ p--;+ fail(std::string("unescaped '") + static_cast<char>(c) + "', use '%" + static_cast<char>(c) + "'");+ default:+ node.op = tts_norm_op::LITERAL;+ node.cp = c;+ return node;+ }+ }+};++//+// matcher+// PEG-style: alternatives are ordered (the first one that matches wins) and quantifiers are greedy and possessive+// (they never give back what they consumed), so there is no backtracking+//++struct tts_norm_matcher {+ const std::vector<uint32_t> & t;+ bool icase;+ std::vector<std::pair<int64_t, int64_t>> caps;++ bool is_w(size_t i) const {+ return i < t.size() && tts_norm_class_match('w', t[i]);+ }++ bool set_match(const tts_norm_set & st, uint32_t c) const {+ auto test = [&](uint32_t x) {+ for (const auto & r : st.ranges) {+ if (x >= r.first && x <= r.second) {+ return true;+ }+ }+ for (const auto & k : st.classes) {+ if (tts_norm_class_match(k.kind, x) != k.neg) {+ return true;+ }+ }+ return false;+ };+ bool m = test(c);+ if (!m && icase) {+ const uint32_t o = tts_norm_ascii_swapcase(c);+ if (o != c) {+ m = test(o);+ }+ }+ return m != st.neg;+ }++ bool match_alts(const std::vector<tts_norm_seq> & alts, size_t i, size_t & end) {+ for (const auto & seq : alts) {+ auto saved = caps;+ if (match_seq(seq, i, end)) {+ return true;+ }+ caps = std::move(saved);+ }+ return false;+ }++ bool match_seq(const tts_norm_seq & seq, size_t i, size_t & end) {+ size_t pos = i;+ for (const auto & it : seq) {+ int count = 0;+ while (count < it.max) {+ const bool is_group = it.atom.op == tts_norm_op::GROUP;+ std::vector<std::pair<int64_t, int64_t>> saved;+ if (is_group) {+ saved = caps;+ }+ size_t e = pos;+ if (!match_once(it.atom, pos, e)) {+ if (is_group) {+ caps = std::move(saved);+ }+ break;+ }+ if (e == pos) {+ // an empty iteration would repeat forever, it satisfies any remaining minimum+ count = std::max(count + 1, it.min);+ break;+ }+ pos = e;+ count++;+ }+ if (count < it.min) {+ return false;+ }+ }+ end = pos;+ return true;+ }++ bool match_once(const tts_norm_pattern_node & node, size_t i, size_t & end) {+ const size_t n = t.size();+ switch (node.op) {+ case tts_norm_op::LITERAL:+ if (i < n && (t[i] == node.cp || (icase && tts_norm_ascii_lower(t[i]) == tts_norm_ascii_lower(node.cp)))) {+ end = i + 1;+ return true;+ }+ return false;+ case tts_norm_op::SET:+ if (i < n && set_match(node.set, t[i])) {+ end = i + 1;+ return true;+ }+ return false;+ case tts_norm_op::ANY:+ if (i < n) {+ end = i + 1;+ return true;+ }+ return false;+ case tts_norm_op::START:+ end = i;+ return i == 0;+ case tts_norm_op::END:+ end = i;+ return i == n;+ case tts_norm_op::BOUNDARY:+ end = i;+ return (i > 0 && is_w(i - 1)) != is_w(i);+ case tts_norm_op::GROUP: {+ size_t e = i;+ if (!match_alts(node.alts, i, e)) {+ return false;+ }+ if (node.capture >= 0) {+ caps[node.capture] = {static_cast<int64_t>(i), static_cast<int64_t>(e)};+ }+ end = e;+ return true;+ }+ case tts_norm_op::LOOKAHEAD: {+ // captures made inside a lookahead are discarded+ auto saved = caps;+ size_t e = i;+ const bool ok = match_alts(node.alts, i, e);+ caps = std::move(saved);+ end = i;+ return ok != node.neg;+ }+ case tts_norm_op::LOOKBEHIND: {+ size_t e = 0;+ const bool ok = i > 0 && match_once(node.alts[0][0].atom, i - 1, e) && e == i;+ end = i;+ return ok != node.neg;+ }+ }+ return false;+ }+};++//+// common_tts_norm_pattern+//++common_tts_norm_pattern::common_tts_norm_pattern(const std::string & src) {+ if (src.size() > TTS_NORM_MAX_PATTERN_LEN) {+ throw std::runtime_error("invalid TTS norm pattern: pattern is longer than " + std::to_string(TTS_NORM_MAX_PATTERN_LEN) + " bytes");+ }+ std::vector<uint32_t> cpts;+ std::vector<size_t> offs;+ tts_norm_decode_utf8(src, cpts, offs);++ tts_norm_parser parser(std::move(cpts), n_caps);+ static const std::vector<uint32_t> icase_prefix = {'(', '?', 'i', ')'};+ if (parser.s.size() >= 4 && std::equal(icase_prefix.begin(), icase_prefix.end(), parser.s.begin())) {+ icase = true;+ parser.p = 4;+ }++ root = std::make_unique<tts_norm_pattern_node>();+ root->op = tts_norm_op::GROUP;+ root->alts = parser.parse_alts();+ if (!parser.eof()) {+ parser.fail("unbalanced ')'");+ }+}++common_tts_norm_pattern::~common_tts_norm_pattern() = default;+common_tts_norm_pattern::common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept = default;+common_tts_norm_pattern & common_tts_norm_pattern::operator=(common_tts_norm_pattern &&) noexcept = default;++bool common_tts_norm_pattern::match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const {+ tts_norm_matcher m{cpts, icase, std::vector<std::pair<int64_t, int64_t>>(n_caps, {-1, -1})};+ size_t e = pos;+ if (!m.match_alts(root->alts, pos, e)) {+ return false;+ }+ end = e;+ groups = std::move(m.caps);+ return true;+}++size_t common_tts_norm_pattern::n_groups() const {+ return n_caps;+}++//+// scan / sub+//++std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules) {+ std::vector<uint32_t> cpts;+ std::vector<size_t> offs;+ tts_norm_decode_utf8(text, cpts, offs);++ std::vector<common_tts_norm_segment> out;+ const size_t n = cpts.size();+ size_t i = 0;+ size_t pending = 0; // start of the current unmatched run+ std::vector<std::pair<int64_t, int64_t>> groups;++ while (i < n) {+ bool matched = false;+ for (size_t r = 0; r < rules.size(); r++) {+ size_t end = i;+ if (rules[r]->match_at(cpts, i, end, groups) && end > i) {+ if (pending < i) {+ out.push_back({-1, offs[pending], offs[i], {}});+ }+ common_tts_norm_segment seg;+ seg.rule = static_cast<int>(r);+ seg.start = offs[i];+ seg.end = offs[end];+ for (const auto & g : groups) {+ if (g.first < 0) {+ seg.groups.emplace_back(std::string::npos, std::string::npos);+ } else {+ seg.groups.emplace_back(offs[g.first], offs[g.second]);+ }+ }+ out.push_back(std::move(seg));+ i = end;+ pending = i;+ matched = true;+ break;+ }+ }+ if (!matched) {+ i++;+ }+ }+ if (pending < n) {+ out.push_back({-1, offs[pending], offs[n], {}});+ }+ return out;+}++struct tts_norm_repl_piece {+ std::string literal;+ int group = -1; // 0 = whole match, 1.. = groups+};++static std::vector<tts_norm_repl_piece> tts_norm_parse_repl(const std::string & repl, const common_tts_norm_pattern & pattern) {+ std::vector<tts_norm_repl_piece> pieces;+ std::string lit;+ auto flush = [&]() {+ if (!lit.empty()) {+ pieces.push_back({lit, -1});+ lit.clear();+ }+ };+ for (size_t i = 0; i < repl.size(); i++) {+ if (repl[i] != '%') {+ lit += repl[i];+ continue;+ }+ if (i + 1 >= repl.size()) {+ throw std::runtime_error("invalid TTS norm replacement: dangling '%'");+ }+ const char c = repl[++i];+ if (c == '%') {+ lit += '%';+ } else if (c >= '0' && c <= '9') {+ const int g = c - '0';+ if (static_cast<size_t>(g) > pattern.n_groups()) {+ throw std::runtime_error("invalid TTS norm replacement: group %" + std::string(1, c) + " does not exist");+ }+ flush();+ pieces.push_back({"", g});+ } else {+ throw std::runtime_error("invalid TTS norm replacement: unknown escape '%" + std::string(1, c) + "'");+ }+ }+ flush();+ return pieces;+}++std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl) {+ const auto pieces = tts_norm_parse_repl(repl, pattern);+ std::string out;+ for (const auto & seg : common_tts_norm_scan(text, {&pattern})) {+ if (seg.rule < 0) {+ out.append(text, seg.start, seg.end - seg.start);+ continue;+ }+ for (const auto & piece : pieces) {+ if (piece.group < 0) {+ out += piece.literal;+ } else if (piece.group == 0) {+ out.append(text, seg.start, seg.end - seg.start);+ } else {+ const auto & g = seg.groups[piece.group - 1];+ if (g.first != std::string::npos) {+ out.append(text, g.first, g.second - g.first);+ }+ }+ }+ }+ return out;+}++//+// template builtins+//++// rules can be: a pattern string (rule name "match"), a dict {name: pattern}, or a list of [name, pattern] pairs+static std::vector<std::pair<std::string, std::string>> tts_norm_rules_from_value(const value & v) {+ std::vector<std::pair<std::string, std::string>> rules;+ if (is_val<value_string>(v)) {+ rules.emplace_back("match", v->as_string().str());+ } else if (is_val<value_object>(v)) {+ for (const auto & kv : v->as_ordered_object()) {+ if (!is_val<value_string>(kv.second)) {+ throw std::runtime_error("tts_norm_scan: rule pattern must be a string");+ }+ rules.emplace_back(kv.first->as_string().str(), kv.second->as_string().str());+ }+ } else if (is_val<value_array>(v)) {+ for (const auto & item : v->as_array()) {+ if (!is_val<value_array>(item) || item->as_array().size() != 2+ || !is_val<value_string>(item->as_array()[0]) || !is_val<value_string>(item->as_array()[1])) {+ throw std::runtime_error("tts_norm_scan: each rule must be a [name, pattern] pair of strings");+ }+ rules.emplace_back(item->as_array()[0]->as_string().str(), item->as_array()[1]->as_string().str());+ }+ } else {+ throw std::runtime_error("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs, got " + v->type());+ }+ return rules;+}++static value tts_norm_builtin_scan(const func_args & args) {+ args.ensure_count(2, 2);+ args.ensure_val<value_string>(args.get_pos(0));+ const string text_val = args.get_pos(0)->as_string();+ const std::string text = text_val.str();+ const bool is_input = text_val.all_parts_are_input();++ const auto rule_srcs = tts_norm_rules_from_value(args.get_pos(1));+ std::vector<common_tts_norm_pattern> patterns;+ patterns.reserve(rule_srcs.size());+ std::vector<const common_tts_norm_pattern *> ptrs;+ for (const auto & r : rule_srcs) {+ patterns.emplace_back(r.second);+ }+ for (const auto & p : patterns) {+ ptrs.push_back(&p);+ }++ auto mk_str = [&](size_t start, size_t end) -> value {+ return mk_val<value_string>(string(text.substr(start, end - start), is_input));+ };++ auto out = mk_val<value_array>();+ for (const auto & seg : common_tts_norm_scan(text, ptrs)) {+ auto groups = mk_val<value_array>();+ for (const auto & g : seg.groups) {+ // groups that did not participate are empty strings, so they render as nothing and are falsy+ groups->push_back(g.first == std::string::npos ? mk_str(0, 0) : mk_str(g.first, g.second));+ }+ auto obj = mk_val<value_object>();+ obj->insert("rule", seg.rule < 0 ? value(mk_val<value_none>()) : value(mk_val<value_string>(rule_srcs[seg.rule].first)));+ obj->insert("text", mk_str(seg.start, seg.end));+ obj->insert("groups", groups);+ out->push_back(obj);+ }+ return out;+}++static value tts_norm_builtin_sub(const func_args & args) {+ args.ensure_count(3, 3);+ args.ensure_vals<value_string, value_string, value_string>();+ const string text_val = args.get_pos(0)->as_string();+ const common_tts_norm_pattern pattern(args.get_pos(1)->as_string().str());+ const std::string res = common_tts_norm_sub(text_val.str(), pattern, args.get_pos(2)->as_string().str());+ return mk_val<value_string>(string(res, text_val.all_parts_are_input()));+}++void common_chat_tts_norm_add_builtins(context & ctx) {+ ctx.set_val("tts_norm_scan", mk_val<value_func>("tts_norm_scan", tts_norm_builtin_scan));+ ctx.set_val("tts_norm_sub", mk_val<value_func>("tts_norm_sub", tts_norm_builtin_sub));+}+common/chat-tts-norm.hadded62 + / 0 −
@@ -0,0 +1,62 @@+#pragma once++#include <cstdint>+#include <memory>+#include <string>+#include <utility>+#include <vector>++// TTS normalization patterns: a small, fully specified matching language for TTS text normalization templates+//+// it is NOT a regex engine: the semantics are defined by llama.cpp (see docs/tts-norm-patterns.md), so that this+// implementation and the python reference (scripts/chat_tts_norm.py) give the same result for every input+//+// two functions are exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():+// tts_norm_scan(text, rules) -> list of segments {rule, text, groups}+// tts_norm_sub(text, pattern, repl) -> string++namespace jinja {+struct context;+}++struct tts_norm_pattern_node;++class common_tts_norm_pattern {+public:+ // throws std::runtime_error on invalid pattern+ explicit common_tts_norm_pattern(const std::string & src);+ ~common_tts_norm_pattern();+ common_tts_norm_pattern(common_tts_norm_pattern &&) noexcept;+ common_tts_norm_pattern & operator=(common_tts_norm_pattern &&) noexcept;++ // match anchored at codepoint index pos; on success, end is the codepoint index after the match+ // groups[i] holds [start, end) codepoint indices of capture i+1, or {-1, -1} if the group did not participate+ bool match_at(const std::vector<uint32_t> & cpts, size_t pos, size_t & end, std::vector<std::pair<int64_t, int64_t>> & groups) const;++ size_t n_groups() const;++private:+ std::unique_ptr<tts_norm_pattern_node> root;+ size_t n_caps = 0;+ bool icase = false;+};++// segment produced by common_tts_norm_scan(); byte offsets into the input text+struct common_tts_norm_segment {+ int rule = -1; // index of the matched rule, -1 for unmatched text+ size_t start = 0;+ size_t end = 0;+ std::vector<std::pair<size_t, size_t>> groups; // byte offsets; {npos, npos} for groups that did not participate+};++// split text into matched and unmatched segments+// at each position, rules are tried in order and the first non-empty match wins; otherwise one codepoint is skipped+std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & text, const std::vector<const common_tts_norm_pattern *> & rules);++// replace every non-overlapping match of pattern with repl+// repl references: %0 (whole match), %1..%9 (groups), %% (literal percent)+std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl);++// add tts_norm_scan() and tts_norm_sub() to the context of a task template+// they are not jinja global builtins on purpose: chat templates are not affected+void common_chat_tts_norm_add_builtins(jinja::context & ctx);common/chat.cpp3 + / 0 −
@@ -3,6 +3,7 @@ #include "chat-auto-parser-helpers.h" #include "chat-auto-parser.h" #include "chat-peg-parser.h"+#include "chat-tts-norm.h" #include "common.h" #include "ggml.h" #include "json-schema-to-grammar.h"@@ -807,6 +808,8 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp // renders a task template (e.g. "tts") with the given variables static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) { jinja::context ctx(tmpl.source());+ // text normalization helpers: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md+ common_chat_tts_norm_add_builtins(ctx); jinja::global_from_json(ctx, vars, false); ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token())); ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));docs/tts-norm-patterns.mdadded125 + / 0 −
@@ -0,0 +1,125 @@+# TTS normalization patterns++Text-to-speech models often need their input normalized before tokenization: numbers, currencies, dates and symbols spelled out, punctuation mapped, spacing fixed. The exact rules are model-specific (they must match what the model was trained on), so they live in the model's `tts` task template rather than in C++.++Jinja has no regex support, and exposing `std::regex` / Python `re` would not help: the two differ in many details (`\d`, `\w`, `\b`, lookbehind, case folding, backtracking limits), so a template could give different results in llama.cpp and in Python. Instead, task templates get two functions with a small matching language whose semantics are defined here:++- `tts_norm_scan(text, rules)`: split text into matched and unmatched segments+- `tts_norm_sub(text, pattern, repl)`: replace every match++Implementations:+- C++: [common/chat-tts-norm.cpp](../common/chat-tts-norm.cpp), injected into task templates by `common_chat_template_render()` in [common/chat.cpp](../common/chat.cpp). They are not available to chat templates.+- Python reference: [scripts/chat_tts_norm.py](../scripts/chat_tts_norm.py), for use with `jinja2` (`env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan`)+- Both are checked against each other by `tests/test-jinja.cpp` (`test-jinja -py "tts norm patterns.*"`)++## Example++```jinja+{%- for s in tts_norm_scan(text, [+ ['money', '%$(%d+)(?:%.(%d%d))?'],+ ['percent', '(%d+)%%'],+ ['time', '(%d%d?):(%d%d)(?!%d)'],+ ['neg', '(?<!%w)-(%d+)'],+ ['number', '%d+'],+]) -%}+ {%- if s.rule == 'money' -%}{%- set whole, cents = s.groups -%}{{ num(whole) }} dollars{{ ' and ' ~ num(cents) ~ ' cents' if cents else '' }}+ {%- elif s.rule == 'percent' -%}{{ num(s.groups[0]) }} percent+ {%- elif s.rule == 'time' -%}{%- set h, m = s.groups -%}{{ num(h) }} {{ num(m) }}+ {%- elif s.rule == 'neg' -%}minus {{ num(s.groups[0]) }}+ {%- elif s.rule == 'number' -%}{{ num(s.text) }}+ {%- else -%}{{ s.text }}+ {%- endif -%}+{%- endfor -%}+```++`It costs $3.50, 20% off, at 10:45, -5 degrees, 7 cats.` becomes `It costs three dollars and fifty cents, twenty percent off, at ten forty-five, minus five degrees, seven cats.` (with a `num` macro that spells numbers, see the "mini text normalizer" test in `tests/test-jinja.cpp`).++## Functions++### `tts_norm_scan(text, rules)`++`rules` is one of:+- a list of `[name, pattern]` pairs (recommended, the order is explicit)+- a dict `{name: pattern}`, in insertion order+- a single pattern string, the rule is named `match`++Scanning starts at the first codepoint. At each position, the rules are tried in order and the first one that produces a **non-empty** match wins; the scan continues after the match. If no rule matches, one codepoint is skipped. Matches never overlap.++The result is a list of segments covering the whole text, in order. Consecutive unmatched codepoints are merged into one segment. Each segment is a dict:++| key | matched segment | unmatched segment |+|---|---|---|+| `rule` | rule name | `none` |+| `text` | matched text | unmatched text |+| `groups` | list of capture groups, in order of their `(` | `[]` |++There are no named groups, unpack `groups` instead: `{% set whole, cents = s.groups %}`.++A group that did not participate in the match (e.g. inside a `?` that matched nothing) is an empty string, so it renders as nothing and is falsy.++### `tts_norm_sub(text, pattern, repl)`++Replaces every match of `pattern`, with the same scanning rules as `tts_norm_scan`. `repl` is a literal string with references:++| reference | value |+|---|---|+| `%0` | whole match |+| `%1` .. `%9` | group |+| `%%` | a literal `%` |++Any other `%` sequence in `repl`, or a reference to a group that does not exist, is an error.++## Pattern syntax++Patterns operate on Unicode codepoints. Invalid UTF-8 bytes (C++ only) are seen as U+FFFD, one per byte, and preserved in the output.++| syntax | meaning |+|---|---|+| `x` | the literal codepoint `x` |+| `%x` | escape: `%` followed by any non-alphanumeric codepoint is that literal codepoint (`%.`, `%$`, `%%`, `%(`, ...) |+| `.` | any codepoint, including newlines |+| `%d` `%s` `%p` `%a` `%w` `%u` `%l` `%h` `%c` | character classes, see below; the uppercase form (`%D`, ...) is the negation |+| `[...]` | set: literal codepoints, ranges `a-z`, classes `%d`; `[^...]` is the negation |+| `(...)` | capture group |+| `(?:...)` | group without capture |+| `a\|b` | ordered alternation |+| `?` `*` `+` `{m}` `{m,}` `{m,n}` | quantifiers, greedy and possessive; `m`, `n` up to 1000 |+| `^` `$` | start / end of the whole text |+| `%b` | boundary: between a `%w` codepoint and a non-`%w` codepoint (the edges of the text count as non-`%w`) |+| `(?=...)` `(?!...)` | lookahead, positive / negative |+| `(?<=c)` `(?<!c)` | lookbehind of exactly one codepoint: `c` is a literal, a set, a class or `.` |+| `(?i)` | at the very start of the pattern only: ignore ASCII case |++These characters must be escaped with `%` to be matched literally: `( ) [ ] { } . | ? * + ^ $ %`. Inside a set, only `]`, `%`, `-` (between two codepoints) and `^` (first position) are special.++### Matching semantics++The matcher is a PEG, not a backtracking regex:++- **Ordered alternation**: alternatives are tried left to right and the first one that matches is committed to. If what follows fails, the other alternatives are not retried. `(?:a|ab)c` never matches `abc`; write `(?:ab|a)c`.+- **Possessive quantifiers**: quantifiers match as many repetitions as possible and never give any back. `%d+0` never matches; `.*x` never matches. Write patterns that describe exactly what to consume.+- A repetition that matches the empty string ends the loop.+- Captures inside lookaheads are discarded. A capture inside a repeated group holds its last repetition.++This gives a simple and linear-time implementation with identical results in C++ and Python.++### Character classes++Classes are defined by explicit codepoint ranges, not by a Unicode database, so the result does not depend on the Unicode version of the platform.++| class | definition |+|---|---|+| `%d` | ASCII digits `0-9` only |+| `%s` | U+0009..U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000..U+200A, U+2028, U+2029, U+202F, U+205F, U+3000 |+| `%p` | punctuation and symbols: ASCII punctuation, U+00A1..U+00BF (except U+00AA, U+00B5, U+00BA), U+00D7, U+00F7, U+2010..U+2027, U+2030..U+205E, U+20A0..U+20CF (currency), U+2100..U+2BFF (letterlike, number forms, arrows, math, technical, box drawing, shapes, misc symbols, dingbats), CJK punctuation (U+3001..U+3003, U+3008..U+3020, U+3030, U+303D, U+30FB), U+FE10..U+FE19, U+FE30..U+FE6B, fullwidth punctuation (U+FF01..U+FF0F, U+FF1A..U+FF20, U+FF3B..U+FF40, U+FF5B..U+FF65), U+1F000..U+1FAFF (emoji) |+| `%c` | control and invisible format characters: U+0000..U+001F, U+007F..U+009F, U+200B..U+200F, U+2060..U+206F, U+FEFF |+| `%a` | ASCII letters, U+00AA, U+00B5, U+00BA, and every codepoint from U+00C0 that is not in `%s`, `%p` or `%c` |+| `%w` | `%a` or `%d` |+| `%u` / `%l` | ASCII uppercase / lowercase letters only |+| `%h` | Han: U+2E80..U+2FDF, U+3005, U+3007, U+3021..U+3029, U+3038..U+303B, U+3400..U+4DBF, U+4E00..U+9FFF, U+F900..U+FAFF, U+20000..U+323AF |++Classes may overlap (e.g. U+0085 is both `%s` and `%c`). `%a` intentionally includes all scripts, combining marks and CJK, so `(?<!%w)` and `%b` work as "not inside a word" for any language.++## Errors++Invalid patterns are rejected when the function is called, with a message and a position, for example: unbalanced parentheses, a quantifier with nothing to repeat, multiple quantifiers (`a+?`), repeating a zero-width assertion (`%b+`), unknown escapes (`%q`), unescaped `]` or `}`, an empty set, a lookbehind wider than one codepoint and `(?i)` not at the start.scripts/chat_tts_norm.pyadded556 + / 0 −
@@ -0,0 +1,556 @@+#!/usr/bin/env python3+"""Reference implementation of TTS normalization patterns, the tts_norm_scan() and tts_norm_sub() template functions.++The specification is in docs/tts-norm-patterns.md. This file must give the same result as the C++ implementation+(common/chat-tts-norm.cpp) for every input; tests/test-jinja.cpp checks this in python mode (test-jinja -py).++Usage with jinja2:++ env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan+ env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub+"""++from __future__ import annotations++from dataclasses import dataclass, field+from typing import Any++# character classes, explicit codepoint ranges (keep in sync with common/chat-tts-norm.cpp)++TTS_NORM_SPACE = [+ (0x0009, 0x000D), (0x0020, 0x0020), (0x0085, 0x0085), (0x00A0, 0x00A0), (0x1680, 0x1680),+ (0x2000, 0x200A), (0x2028, 0x2029), (0x202F, 0x202F), (0x205F, 0x205F), (0x3000, 0x3000),+]++TTS_NORM_PUNCT = [+ (0x0021, 0x002F), (0x003A, 0x0040), (0x005B, 0x0060), (0x007B, 0x007E),+ (0x00A1, 0x00A9), (0x00AB, 0x00B4), (0x00B6, 0x00B9), (0x00BB, 0x00BF), (0x00D7, 0x00D7), (0x00F7, 0x00F7),+ (0x2010, 0x2027), (0x2030, 0x205E), (0x20A0, 0x20CF), (0x2100, 0x218F), (0x2190, 0x2BFF),+ (0x3001, 0x3003), (0x3008, 0x3020), (0x3030, 0x3030), (0x303D, 0x303D), (0x30FB, 0x30FB),+ (0xFE10, 0xFE19), (0xFE30, 0xFE4F), (0xFE50, 0xFE6B),+ (0xFF01, 0xFF0F), (0xFF1A, 0xFF20), (0xFF3B, 0xFF40), (0xFF5B, 0xFF65),+ (0x1F000, 0x1FAFF),+]++TTS_NORM_CONTROL = [+ (0x0000, 0x001F), (0x007F, 0x009F), (0x200B, 0x200F), (0x2060, 0x206F), (0xFEFF, 0xFEFF),+]++TTS_NORM_HAN = [+ (0x2E80, 0x2FDF), (0x3005, 0x3005), (0x3007, 0x3007), (0x3021, 0x3029), (0x3038, 0x303B),+ (0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), (0x20000, 0x323AF),+]++CLASS_LETTERS = "dspawulhc"+INF = 1 << 31+MAX_REPEAT = 1000+MAX_PATTERN_LEN = 4096+++def _in(ranges: list[tuple[int, int]], c: int) -> bool:+ return any(lo <= c <= hi for lo, hi in ranges)+++def _is_alpha(c: int) -> bool:+ if ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z"):+ return True+ if c in (0xAA, 0xB5, 0xBA):+ return True+ return c >= 0xC0 and not _in(TTS_NORM_SPACE, c) and not _in(TTS_NORM_PUNCT, c) and not _in(TTS_NORM_CONTROL, c)+++def _class_match(kind: str, c: int) -> bool:+ if kind == "d":+ return ord("0") <= c <= ord("9")+ if kind == "s":+ return _in(TTS_NORM_SPACE, c)+ if kind == "p":+ return _in(TTS_NORM_PUNCT, c)+ if kind == "a":+ return _is_alpha(c)+ if kind == "w":+ return _is_alpha(c) or ord("0") <= c <= ord("9")+ if kind == "u":+ return ord("A") <= c <= ord("Z")+ if kind == "l":+ return ord("a") <= c <= ord("z")+ if kind == "h":+ return _in(TTS_NORM_HAN, c)+ if kind == "c":+ return _in(TTS_NORM_CONTROL, c)+ return False+++def _ascii_lower(c: int) -> int:+ return c + 32 if ord("A") <= c <= ord("Z") else c+++def _ascii_swapcase(c: int) -> int:+ if ord("A") <= c <= ord("Z"):+ return c + 32+ if ord("a") <= c <= ord("z"):+ return c - 32+ return c+++def _is_ascii_alnum(c: int) -> bool:+ return ord("0") <= c <= ord("9") or ord("a") <= c <= ord("z") or ord("A") <= c <= ord("Z")+++# AST++LITERAL, SET, ANY, START, END, BOUNDARY, GROUP, LOOKAHEAD, LOOKBEHIND = range(9)+ZERO_WIDTH = (START, END, BOUNDARY, LOOKAHEAD, LOOKBEHIND)+++@dataclass+class _Set:+ neg: bool = False+ ranges: list[tuple[int, int]] = field(default_factory=list)+ classes: list[tuple[str, bool]] = field(default_factory=list)+++@dataclass+class _Node:+ op: int = GROUP+ cp: int = 0+ set: _Set = field(default_factory=_Set)+ alts: list[list[_Item]] = field(default_factory=list)+ capture: int = -1+ neg: bool = False+++@dataclass+class _Item:+ atom: _Node+ min: int = 1+ max: int = 1+++class PatternError(ValueError):+ pass+++class _Parser:+ def __init__(self, s: list[int]):+ self.s = s+ self.p = 0+ self.n_caps = 0++ def fail(self, msg: str):+ raise PatternError(f"invalid TTS norm pattern: {msg} at position {self.p}")++ def eof(self) -> bool:+ return self.p >= len(self.s)++ def peek_is(self, c: str) -> bool:+ return not self.eof() and self.s[self.p] == ord(c)++ def expect(self, c: str, what: str):+ if not self.peek_is(c):+ self.fail(f"expected {what}")+ self.p += 1++ def parse_alts(self) -> list[list[_Item]]:+ alts = [self.parse_seq()]+ while self.peek_is("|"):+ self.p += 1+ alts.append(self.parse_seq())+ return alts++ def parse_seq(self) -> list[_Item]:+ seq: list[_Item] = []+ while not self.eof() and self.s[self.p] not in (ord("|"), ord(")")):+ item = _Item(self.parse_atom())+ self.parse_quantifier(item)+ seq.append(item)+ return seq++ def parse_int(self) -> int:+ if self.eof() or not (ord("0") <= self.s[self.p] <= ord("9")):+ self.fail("expected number in quantifier")+ v = 0+ while not self.eof() and ord("0") <= self.s[self.p] <= ord("9"):+ v = v * 10 + self.s[self.p] - ord("0")+ if v > MAX_REPEAT:+ self.fail(f"repetition count exceeds {MAX_REPEAT}")+ self.p += 1+ return v++ def parse_quantifier(self, item: _Item):+ if self.eof():+ return+ c = chr(self.s[self.p])+ if c not in "?*+{":+ return+ if item.atom.op in ZERO_WIDTH:+ self.fail("zero-width assertion cannot be repeated")+ self.p += 1+ if c == "?":+ item.min, item.max = 0, 1+ elif c == "*":+ item.min, item.max = 0, INF+ elif c == "+":+ item.min, item.max = 1, INF+ else:+ item.min = self.parse_int()+ item.max = item.min+ if self.peek_is(","):+ self.p += 1+ item.max = INF if self.peek_is("}") else self.parse_int()+ self.expect("}", "'}' to close quantifier")+ if item.max < item.min:+ self.fail("quantifier max is smaller than min")+ if not self.eof() and chr(self.s[self.p]) in "?*+{":+ self.fail("multiple quantifiers")++ def parse_group(self) -> _Node:+ node = _Node(op=GROUP)+ if self.peek_is("?"):+ self.p += 1+ if self.eof():+ self.fail("unterminated group")+ c = chr(self.s[self.p])+ if c == ":":+ self.p += 1+ elif c in "=!":+ self.p += 1+ node.op = LOOKAHEAD+ node.neg = c == "!"+ elif c == "<" and self.p + 1 < len(self.s) and self.s[self.p + 1] in (ord("="), ord("!")):+ node.op = LOOKBEHIND+ node.neg = self.s[self.p + 1] == ord("!")+ self.p += 2+ if self.eof() or self.s[self.p] == ord(")"):+ self.fail("lookbehind must contain exactly one codepoint matcher")+ atom = self.parse_atom()+ if atom.op not in (LITERAL, SET, ANY):+ self.fail("lookbehind must contain exactly one codepoint matcher")+ if not self.peek_is(")"):+ self.fail("lookbehind must contain exactly one codepoint matcher")+ self.p += 1+ node.alts = [[_Item(atom)]]+ return node+ elif c == "i":+ self.fail("(?i) is only allowed at the start of the pattern")+ else:+ self.fail("unknown group type")+ else:+ node.capture = self.n_caps+ self.n_caps += 1+ node.alts = self.parse_alts()+ self.expect(")", "')' to close group")+ return node++ # returns ("class", (kind, neg)) | ("boundary", None) | ("literal", cp)+ def parse_escape(self, in_set: bool) -> tuple[str, Any]:+ if self.eof():+ self.fail("dangling '%'")+ e = self.s[self.p]+ self.p += 1+ if e < 0x80 and chr(e).lower() in CLASS_LETTERS:+ return "class", (chr(e).lower(), ord("A") <= e <= ord("Z"))+ if e == ord("b") and not in_set:+ return "boundary", None+ if _is_ascii_alnum(e):+ self.p -= 1+ self.fail(f"unknown escape '%{chr(e)}'")+ return "literal", e++ def parse_set(self) -> _Node:+ node = _Node(op=SET)+ if self.peek_is("^"):+ node.set.neg = True+ self.p += 1+ empty = True+ while True:+ if self.eof():+ self.fail("unterminated set")+ if self.s[self.p] == ord("]"):+ self.p += 1+ break+ empty = False+ if self.s[self.p] == ord("%"):+ self.p += 1+ kind, val = self.parse_escape(True)+ if kind == "class":+ node.set.classes.append(val)+ continue+ lo = val+ else:+ lo = self.s[self.p]+ self.p += 1+ hi = lo+ if self.peek_is("-") and self.p + 1 < len(self.s) and self.s[self.p + 1] != ord("]"):+ self.p += 1+ if self.s[self.p] == ord("%"):+ self.p += 1+ kind, val = self.parse_escape(True)+ if kind == "class":+ self.fail("character class cannot be used in a range")+ hi = val+ else:+ hi = self.s[self.p]+ self.p += 1+ if hi < lo:+ self.fail("invalid range in set")+ node.set.ranges.append((lo, hi))+ if empty:+ self.fail("empty set")+ return node++ def parse_atom(self) -> _Node:+ c = chr(self.s[self.p])+ self.p += 1+ if c == "(":+ return self.parse_group()+ if c == "[":+ return self.parse_set()+ if c == ".":+ return _Node(op=ANY)+ if c == "^":+ return _Node(op=START)+ if c == "$":+ return _Node(op=END)+ if c == "%":+ kind, val = self.parse_escape(False)+ if kind == "class":+ node = _Node(op=SET)+ node.set.classes.append(val)+ return node+ if kind == "boundary":+ return _Node(op=BOUNDARY)+ return _Node(op=LITERAL, cp=val)+ if c in "?*+{":+ self.p -= 1+ self.fail("nothing to repeat")+ if c in "]}":+ self.p -= 1+ self.fail(f"unescaped '{c}', use '%{c}'")+ return _Node(op=LITERAL, cp=ord(c))+++class _Matcher:+ def __init__(self, t: list[int], icase: bool, n_groups: int):+ self.t = t+ self.icase = icase+ self.caps: list[tuple[int, int]] = [(-1, -1)] * n_groups++ def is_w(self, i: int) -> bool:+ return i < len(self.t) and _class_match("w", self.t[i])++ def set_match(self, st: _Set, c: int) -> bool:+ def test(x: int) -> bool:+ if any(lo <= x <= hi for lo, hi in st.ranges):+ return True+ return any(_class_match(kind, x) != neg for kind, neg in st.classes)++ m = test(c)+ if not m and self.icase:+ o = _ascii_swapcase(c)+ if o != c:+ m = test(o)+ return m != st.neg++ def match_alts(self, alts: list[list[_Item]], i: int) -> int:+ """returns the end position, or -1 if no alternative matches"""+ for seq in alts:+ saved = list(self.caps)+ end = self.match_seq(seq, i)+ if end >= 0:+ return end+ self.caps = saved+ return -1++ def match_seq(self, seq: list[_Item], i: int) -> int:+ pos = i+ for it in seq:+ count = 0+ while count < it.max:+ is_group = it.atom.op == GROUP+ saved = list(self.caps) if is_group else None+ e = self.match_once(it.atom, pos)+ if e < 0:+ if saved is not None:+ self.caps = saved+ break+ if e == pos:+ # an empty iteration would repeat forever, it satisfies any remaining minimum+ count = max(count + 1, it.min)+ break+ pos = e+ count += 1+ if count < it.min:+ return -1+ return pos++ def match_once(self, node: _Node, i: int) -> int:+ """returns the end position, or -1 on failure"""+ n = len(self.t)+ op = node.op+ if op == LITERAL:+ if i < n and (self.t[i] == node.cp or (self.icase and _ascii_lower(self.t[i]) == _ascii_lower(node.cp))):+ return i + 1+ return -1+ if op == SET:+ return i + 1 if i < n and self.set_match(node.set, self.t[i]) else -1+ if op == ANY:+ return i + 1 if i < n else -1+ if op == START:+ return i if i == 0 else -1+ if op == END:+ return i if i == n else -1+ if op == BOUNDARY:+ return i if (i > 0 and self.is_w(i - 1)) != self.is_w(i) else -1+ if op == GROUP:+ e = self.match_alts(node.alts, i)+ if e < 0:+ return -1+ if node.capture >= 0:+ self.caps[node.capture] = (i, e)+ return e+ if op == LOOKAHEAD:+ # captures made inside a lookahead are discarded+ saved = list(self.caps)+ ok = self.match_alts(node.alts, i) >= 0+ self.caps = saved+ return i if ok != node.neg else -1+ if op == LOOKBEHIND:+ ok = i > 0 and self.match_once(node.alts[0][0].atom, i - 1) == i+ return i if ok != node.neg else -1+ return -1+++class Pattern:+ def __init__(self, src: str):+ if len(src.encode("utf-8")) > MAX_PATTERN_LEN:+ raise PatternError(f"invalid TTS norm pattern: pattern is longer than {MAX_PATTERN_LEN} bytes")+ self.icase = False+ parser = _Parser([ord(c) for c in src])+ if src.startswith("(?i)"):+ self.icase = True+ parser.p = 4+ self.root = _Node(op=GROUP, alts=parser.parse_alts())+ if not parser.eof():+ parser.fail("unbalanced ')'")+ self.n_caps = parser.n_caps++ def match_at(self, t: list[int], pos: int) -> tuple[int, list[tuple[int, int]]] | None:+ m = _Matcher(t, self.icase, self.n_caps)+ end = m.match_alts(self.root.alts, pos)+ if end < 0:+ return None+ return end, m.caps+++# (rule index or -1, start, end, groups) with codepoint offsets+Segment = tuple[int, int, int, list[tuple[int, int]]]+++def scan_segments(text: str, rules: list[Pattern]) -> list[Segment]:+ t = [ord(c) for c in text]+ n = len(t)+ out: list[Segment] = []+ i = 0+ pending = 0+ while i < n:+ matched = False+ for r, rule in enumerate(rules):+ res = rule.match_at(t, i)+ if res is not None and res[0] > i:+ end, groups = res+ if pending < i:+ out.append((-1, pending, i, []))+ out.append((r, i, end, groups))+ i = end+ pending = i+ matched = True+ break+ if not matched:+ i += 1+ if pending < n:+ out.append((-1, pending, n, []))+ return out+++def _rules_from_value(rules: Any) -> list[tuple[str, str]]:+ if isinstance(rules, str):+ return [("match", rules)]+ if isinstance(rules, dict):+ out = []+ for k, v in rules.items():+ if not isinstance(v, str):+ raise TypeError("tts_norm_scan: rule pattern must be a string")+ out.append((str(k), v))+ return out+ if isinstance(rules, (list, tuple)):+ out = []+ for item in rules:+ if not isinstance(item, (list, tuple)) or len(item) != 2 or not isinstance(item[0], str) or not isinstance(item[1], str):+ raise TypeError("tts_norm_scan: each rule must be a [name, pattern] pair of strings")+ out.append((item[0], item[1]))+ return out+ raise TypeError("tts_norm_scan: rules must be a string, a dict or a list of [name, pattern] pairs")+++def tts_norm_scan(text: str, rules: Any) -> list[dict[str, Any]]:+ rule_srcs = _rules_from_value(rules)+ patterns = [Pattern(src) for _, src in rule_srcs]+ out: list[dict[str, Any]] = []+ for r, start, end, groups in scan_segments(text, patterns):+ # groups that did not participate are empty strings, so they render as nothing and are falsy+ vals = ["" if gs < 0 else text[gs:ge] for gs, ge in groups]+ out.append({"rule": rule_srcs[r][0] if r >= 0 else None, "text": text[start:end], "groups": vals})+ return out+++def _parse_repl(repl: str, pattern: Pattern) -> list[str | int]:+ pieces: list[str | int] = []+ lit = ""+ i = 0+ while i < len(repl):+ c = repl[i]+ if c != "%":+ lit += c+ i += 1+ continue+ if i + 1 >= len(repl):+ raise PatternError("invalid TTS norm replacement: dangling '%'")+ i += 1+ c = repl[i]+ if c == "%":+ lit += "%"+ elif "0" <= c <= "9":+ g = ord(c) - ord("0")+ if g > pattern.n_caps:+ raise PatternError(f"invalid TTS norm replacement: group %{c} does not exist")+ if lit:+ pieces.append(lit)+ lit = ""+ pieces.append(g)+ else:+ raise PatternError(f"invalid TTS norm replacement: unknown escape '%{c}'")+ i += 1+ if lit:+ pieces.append(lit)+ return pieces+++def tts_norm_sub(text: str, pattern: str, repl: str) -> str:+ pat = Pattern(pattern)+ pieces = _parse_repl(repl, pat)+ out = []+ for r, start, end, groups in scan_segments(text, [pat]):+ if r < 0:+ out.append(text[start:end])+ continue+ for piece in pieces:+ if isinstance(piece, str):+ out.append(piece)+ elif piece == 0:+ out.append(text[start:end])+ else:+ gs, ge = groups[piece - 1]+ if gs >= 0:+ out.append(text[gs:ge])+ return "".join(out)tests/test-jinja.cpp157 + / 11 −
@@ -3,6 +3,7 @@ #include <random> #include <cstdlib> +#include "chat-tts-norm.h" #include "json.h" #include "subproc.h" @@ -16,8 +17,8 @@ using json = common_json; -static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect);-static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect);+static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm = false);+static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm = false); static void test_whitespace_control(testing & t); static void test_conditionals(testing & t);@@ -39,6 +40,7 @@ static void test_stats(testing & t); static void test_caps(testing & t); static void test_string_parts(testing & t); static void test_fuzzing(testing & t);+static void test_tts_norm_patterns(testing & t); static bool g_python_mode = false; @@ -75,6 +77,7 @@ int main(int argc, char *argv[]) { t.test("string methods", test_string_methods); t.test("array methods", test_array_methods); t.test("object methods", test_object_methods);+ t.test("tts norm patterns", test_tts_norm_patterns); if (!g_python_mode) { t.test("hasher", test_hasher); t.test("stats", test_stats);@@ -2713,13 +2716,134 @@ static void test_string_parts(testing & t) { } -static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect) {- t.test(name, [&tmpl, &expect](testing & t) {+// TTS normalization patterns: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md+// these functions are only available to task templates (e.g. "tts"), so every test here passes tts_norm = true+static void test_tts_norm_patterns(testing & t) {+ test_template(t, "not available without tts norm builtins",+ "{{ tts_norm_scan is defined }}",+ json::object(),+ "False");++ test_template(t, "tts_norm_scan splits text into matched and unmatched segments",+ "{%- for s in tts_norm_scan(text, [['num', '%d+'], ['word', '%a+']]) -%}[{{ s.rule or '-' }}:{{ s.text }}]{%- endfor -%}",+ {{"text", "abc 123, x9"}},+ "[word:abc][-: ][num:123][-:, ][word:x][num:9]", true);++ test_template(t, "tts_norm_scan groups, optional group that did not match is empty",+ "{%- for s in tts_norm_scan(text, [['money', '%$(%d+)(?:%.(%d%d))?']]) -%}"+ "{%- if s.rule -%}{%- set whole, cents = s.groups -%}{{ whole }}|{{ cents }}|{{ s.groups | length }};{%- endif -%}"+ "{%- endfor -%}",+ {{"text", "$3.50 and $7"}},+ "3|50|2;7||2;", true);++ test_template(t, "tts_norm_scan rules as dict, the first rule that matches wins",+ "{%- for s in tts_norm_scan(text, {'ordinal': '%d+(?:st|nd|rd|th)%b', 'number': '%d+'}) -%}"+ "{%- if s.rule -%}<{{ s.rule }}:{{ s.text }}>{%- else -%}{{ s.text }}{%- endif -%}"+ "{%- endfor -%}",+ {{"text", "the 3rd of 10 or 4three"}},+ "the <ordinal:3rd> of <number:10> or <number:4>three", true);++ // unlike regex, a failed sequence does not retry the other branches: (?:a|ab)c never matches "abc"+ test_template(t, "ordered alternation commits to the first branch that matches",+ "{%- for s in tts_norm_scan(text, '(?:a|ab)c') -%}[{{ s.rule or '-' }}:{{ s.text }}]{%- endfor -%}",+ {{"text", "abc ac"}},+ "[-:abc ][match:ac]", true);++ // unlike regex, quantifiers never give back what they consumed: %d+0 never matches+ test_template(t, "quantifiers are possessive",+ "{{ tts_norm_sub(text, '%d+0', 'X') }}|{{ tts_norm_sub(text, '%d+', 'X') }}",+ {{"text", "100 20"}},+ "100 20|X X", true);++ test_template(t, "quantifier ranges",+ "{{ tts_norm_sub(text, '%d{3}-%d{2,}', '#') }}",+ {{"text", "123-4 123-45 123-456789"}},+ "123-4 # #", true);++ test_template(t, "lookbehind: a minus sign is not part of a word",+ "{{ tts_norm_sub(text, '(?<!%w)-(%d+)', 'minus %1') }}",+ {{"text", "GPT-5 is -5, (-3)"}},+ "GPT-5 is minus 5, (minus 3)", true);++ test_template(t, "lookahead: clock time not followed by a digit",+ "{{ tts_norm_sub(text, '(%d%d?):(%d%d)(?!%d)', '%1h%2') }}",+ {{"text", "at 10:30 or 1:234"}},+ "at 10h30 or 1:234", true);++ test_template(t, "tts_norm_sub references: group, whole match and literal percent",+ "{{ tts_norm_sub(text, '(%d+)%%', '%1 percent (%0) 100%%') }}",+ {{"text", "up 50%"}},+ "up 50 percent (50%) 100%", true);++ test_template(t, "split letters and digits, non-ASCII letters included",+ "{{ tts_norm_sub(tts_norm_sub(text, '(%a)(%d)', '%1 %2'), '(%d)(%a)', '%1 %2') }}",+ {{"text", "R2D2 café2 4K"}},+ "R 2 D 2 café 2 4 K", true);++ test_template(t, "case-insensitive pattern and word boundary",+ "{{ tts_norm_sub(text, '(?i)%bdr%.', 'doctor') }}",+ {{"text", "Dr. Who and dr. No, not Mdr."}},+ "doctor Who and doctor No, not Mdr.", true);++ test_template(t, "sets with multi-byte codepoints, negated sets with classes",+ "{{ tts_norm_sub(text, '[$€£]%s*(%d+)', '<%1>') }}|{{ tts_norm_sub(text, '[^%w%s]', '') }}",+ {{"text", "€ 5 & $6!"}},+ "<5> & <6>!| 5 6", true);++ test_template(t, "remove spaces between Han characters",+ "{{ tts_norm_sub(text, '(%h) +(%h)', '%1%2') }}",+ {{"text", "你 好 world 世界"}},+ "你好 world 世界", true);++ test_template(t, "mini text normalizer",+ "{%- set ones = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten',"+ " 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'] -%}\n"+ "{%- set tens = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'] -%}\n"+ "{%- macro num(n) -%}{{ ones[n] if n < 20 else tens[(n / 10) | int] ~ ('-' ~ ones[n % 10] if n % 10 else '') }}{%- endmacro -%}\n"+ "{%- for s in tts_norm_scan(text, [\n"+ " ['money', '%$(%d+)(?:%.(%d%d))?'],\n"+ " ['percent', '(%d+)%%'],\n"+ " ['time', '(%d%d?):(%d%d)(?!%d)'],\n"+ " ['neg', '(?<!%w)-(%d+)'],\n"+ " ['number', '%d+'],\n"+ "]) -%}\n"+ "{%- if s.rule == 'money' -%}{%- set whole, cents = s.groups -%}{{ num(whole | int) }} dollars{{ ' and ' ~ num(cents | int) ~ ' cents' if cents else '' }}\n"+ "{%- elif s.rule == 'percent' -%}{{ num(s.groups[0] | int) }} percent\n"+ "{%- elif s.rule == 'time' -%}{%- set h, m = s.groups -%}{{ num(h | int) }} {{ num(m | int) }}\n"+ "{%- elif s.rule == 'neg' -%}minus {{ num(s.groups[0] | int) }}\n"+ "{%- elif s.rule == 'number' -%}{{ num(s.text | int) }}\n"+ "{%- else -%}{{ s.text }}\n"+ "{%- endif -%}\n"+ "{%- endfor -%}",+ {{"text", "It costs $3.50, 20% off, at 10:45, -5 degrees, 7 cats."}},+ "It costs three dollars and fifty cents, twenty percent off, at ten forty-five, minus five degrees, seven cats.", true);++ if (!g_python_mode) {+ test_template_error(t, "error: unclosed group", "{{ tts_norm_scan('x', '(a') }}", "expected ')' to close group", true);+ test_template_error(t, "error: unbalanced parenthesis", "{{ tts_norm_scan('x', 'a)') }}", "unbalanced ')'", true);+ test_template_error(t, "error: nothing to repeat", "{{ tts_norm_scan('x', '*a') }}", "nothing to repeat", true);+ test_template_error(t, "error: multiple quantifiers", "{{ tts_norm_scan('x', 'a+?') }}", "multiple quantifiers", true);+ test_template_error(t, "error: bad quantifier range", "{{ tts_norm_scan('x', 'a{2,1}') }}", "quantifier max is smaller than min", true);+ test_template_error(t, "error: repeated assertion", "{{ tts_norm_scan('x', '%b+') }}", "zero-width assertion cannot be repeated", true);+ test_template_error(t, "error: unknown class", "{{ tts_norm_scan('x', '%q') }}", "unknown escape '%q'", true);+ test_template_error(t, "error: unescaped bracket", "{{ tts_norm_scan('x', 'a]') }}", "unescaped ']'", true);+ test_template_error(t, "error: no named groups", "{{ tts_norm_scan('x', '(?<n>a)') }}", "unknown group type", true);+ test_template_error(t, "error: wide lookbehind", "{{ tts_norm_scan('x', '(?<=ab)c') }}", "lookbehind must contain exactly one codepoint matcher", true);+ test_template_error(t, "error: missing group in replacement", "{{ tts_norm_sub('x', '(a)', '%2') }}", "group %2 does not exist", true);+ test_template_error(t, "error: bad rules", "{{ tts_norm_scan('x', 1) }}", "rules must be a string, a dict or a list", true);+ }+}++static void test_template_error(testing & t, const std::string & name, const std::string & tmpl, const std::string & expect, bool tts_norm) {+ t.test(name, [&tmpl, &expect, tts_norm](testing & t) { std::string error; try { jinja::lexer lexer; jinja::program ast = jinja::parse_from_tokens(lexer.tokenize(tmpl)); jinja::context ctx(tmpl);+ if (tts_norm) {+ common_chat_tts_norm_add_builtins(ctx);+ } jinja::runtime runtime(ctx); runtime.gather_string_parts(runtime.execute(ast)); } catch (const std::exception & e) {@@ -2729,14 +2853,17 @@ static void test_template_error(testing & t, const std::string & name, const std }); } -static void test_template_cpp(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) {- t.test(name, [&tmpl, &vars, &expect](testing & t) {+static void test_template_cpp(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) {+ t.test(name, [&tmpl, &vars, &expect, tts_norm](testing & t) { jinja::lexer lexer; auto lexer_res = lexer.tokenize(tmpl); jinja::program ast = jinja::parse_from_tokens(lexer_res); jinja::context ctx(tmpl);+ if (tts_norm) {+ common_chat_tts_norm_add_builtins(ctx);+ } jinja::global_from_json(ctx, vars, true); jinja::runtime runtime(ctx);@@ -2788,17 +2915,36 @@ env.filters["tojson"] = lambda x, ensure_ascii=False, indent=None, separators=No env.globals["strftime_now"] = lambda format: datetime.now().strftime(format) env.globals["raise_exception"] = raise_exception +if merged_input.get("tts_norm_ref"):+ import importlib.util+ spec = importlib.util.spec_from_file_location("chat_tts_norm", merged_input["tts_norm_ref"])+ chat_tts_norm = importlib.util.module_from_spec(spec)+ sys.modules["chat_tts_norm"] = chat_tts_norm # required by dataclasses+ spec.loader.exec_module(chat_tts_norm)+ env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan+ env.globals["tts_norm_sub"] = chat_tts_norm.tts_norm_sub+ template = env.from_string(tmpl) result = template.render(**vars_json) sys.stdout.buffer.write(result.encode()) )"; -static void test_template_py(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) {- t.test(name, [&tmpl, &vars, &expect](testing & t) {+// python reference of the TTS normalization patterns, relative to this file+static std::string tts_norm_ref_path() {+ std::string path = __FILE__;+ const size_t pos = path.find_last_of("/\\");+ return (pos == std::string::npos ? std::string(".") : path.substr(0, pos)) + "/../scripts/chat_tts_norm.py";+}++static void test_template_py(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) {+ t.test(name, [&tmpl, &vars, &expect, tts_norm](testing & t) { // Prepare arguments json merged; merged["tmpl"] = json(tmpl); merged["vars"] = vars;+ if (tts_norm) {+ merged["tts_norm_ref"] = tts_norm_ref_path();+ } #ifdef _WIN32 const char * python_executable = "python.exe";@@ -2859,11 +3005,11 @@ static void test_template_py(testing & t, const std::string & name, const std::s }); } -static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) {+static void test_template(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect, bool tts_norm) { if (g_python_mode) {- test_template_py(t, name, tmpl, vars, expect);+ test_template_py(t, name, tmpl, vars, expect, tts_norm); } else {- test_template_cpp(t, name, tmpl, vars, expect);+ test_template_cpp(t, name, tmpl, vars, expect, tts_norm); } }