ggml-org/llama.cpp · #31808
mtmd: move log-mel computation to cgraph
tests/test-mtmd-impl.cpp177 + / 12 −
@@ -1,10 +1,13 @@ #include "testing.h" +#include "ggml-backend.h"+#include "ggml-cpp.h" #include "mtmd-audio.h" #include "mtmd-helper.h" #include "mtmd-image.h" #include "mtmd-internal.h" +#include <algorithm> #include <cmath> #include <cstdint> #include <cstdlib>@@ -479,15 +482,6 @@ MAKE_TEST(test_audio_preprocessor_conformer) { } } - mtmd_audio_preprocessor_conformer preproc(hparams);- preproc.initialize();- std::vector<mtmd_audio_mel> mels;- if (!t.assert_true("preprocess", preproc.preprocess(samples.data(), samples.size(), mels))) {- return;- }- const mtmd_audio_mel & mel = mels[0];- t.assert_equal("n_len", (int64_t) 101, mel.n_len);- // reference values from NeMo AudioToMelSpectrogramPreprocessor (liquid-audio 1.3.0) // { mel bin, frame, value } const std::vector<std::tuple<int, int, float>> cases = {@@ -496,12 +490,180 @@ MAKE_TEST(test_audio_preprocessor_conformer) { { 24, 60, -0.0476f }, // silence -> quiet tone { 100, 20, -0.0913f }, // tones, high bin };- for (const auto & [bin, frame, expected] : cases) {- const float actual = mel.data[bin * mel.n_len + frame];- t.assert_true("mel[" + std::to_string(bin) + "][" + std::to_string(frame) + "] = " + std::to_string(actual) + ", expected " + std::to_string(expected), std::fabs(actual - expected) < 1e-3f);++ ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr));+ ggml_backend_t backends[] = { backend.get() };+ ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true));++ // the result must not depend on the number of threads+ for (int n_threads : { 1, 4, 7 }) {+ mtmd_audio_preprocessor_conformer preproc(hparams, sched.get());+ preproc.mel_config.n_threads = n_threads;+ preproc.initialize();+ mtmd_audio_preproc_out out = preproc.preprocess(samples.data(), samples.size());+ if (!t.assert_true("preprocess", !out.entries.empty() && preproc.compute(out))) {+ return;+ }+ const clip_image_f32 & mel = out.entries[0];+ t.assert_equal("n_len", 101, mel.nx());++ for (const auto & [bin, frame, expected] : cases) {+ const float actual = mel.get_ro_buf()[bin * mel.nx() + frame];+ t.assert_true("mel[" + std::to_string(bin) + "][" + std::to_string(frame) + "] = " + std::to_string(actual) + ", expected " + std::to_string(expected), std::fabs(actual - expected) < 1e-3f);+ } } } +MAKE_TEST(test_audio_preprocessor_threads) {+ clip_hparams hparams;+ hparams.n_mel_bins = 128;+ hparams.audio_sample_rate = 16000;+ hparams.audio_n_fft = 512;+ hparams.audio_window_len = 400;+ hparams.audio_hop_len = 160;++ // chirp: every frame has a different content+ std::vector<float> samples(hparams.audio_sample_rate * 5);+ for (size_t i = 0; i < samples.size(); i++) {+ samples[i] = (float) (0.3 * std::sin(0.05 * i + 2e-6 * i * i));+ }++ ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr));+ ggml_backend_t backends[] = { backend.get() };+ ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true));++ mtmd_audio_preprocessor_conformer preproc(hparams, sched.get());+ preproc.initialize();+ // tokenize only gets the shapes, encode must get the same ones+ mtmd_audio_preproc_out ref = preproc.preprocess(samples.data(), samples.size());+ if (!t.assert_true("preprocess", ref.entries.size() == 1 && ref.needs_compute())) {+ return;+ }+ const clip_image_size shape = ref.entries[0].get_size();+ if (!t.assert_true("compute", preproc.compute(ref))) {+ return;+ }+ t.assert_true("shape", ref.entries.size() == 1 && ref.entries[0].get_size() == shape);+ t.assert_true("computed", !ref.needs_compute());++ // frames are independent, so the whole output must be bit-exact with the single-threaded one+ for (int n_threads : { 4, 12 }) {+ preproc.mel_config.n_threads = n_threads;+ mtmd_audio_preproc_out out = preproc.preprocess(samples.data(), samples.size());+ const bool ok = preproc.compute(out);+ t.assert_true("n_threads = " + std::to_string(n_threads), ok && out.entries[0].get_ro_buf() == ref.entries[0].get_ro_buf());+ }+}++MAKE_TEST(test_audio_log_mel_precision) {+ const int n_mel = 80;+ const int n_fft = 400;+ const int hop = 160;+ const int n_frames = 200;++ mtmd_audio_cache cache;+ cache.fill_sin_cos_table(n_fft);+ cache.fill_hann_window(n_fft, true);+ cache.fill_mel_filterbank_matrix(n_mel, n_fft, 16000);++ ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr));+ ggml_backend_t backends[] = { backend.get() };+ ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true));++ mtmd_audio_mel_config config;+ config.sched = sched.get();+ config.n_threads = 2;+ config.n_mel = n_mel;+ config.n_fft_bins = 1 + n_fft / 2;+ config.hop_length = hop;+ config.cache = &cache;++ // a loud tone next to a 74 dB weaker one: rounding the DFT inputs to F16 or TF32 loses the weak one+ const double pi = 3.14159265358979323846;+ std::vector<float> samples((size_t) (n_frames - 1) * hop + n_fft);+ for (size_t i = 0; i < samples.size(); i++) {+ samples[i] = (float) (0.5 * std::sin(2 * pi * 200 * i / 16000) + 1e-4 * std::sin(2 * pi * 6000 * i / 16000));+ }++ std::vector<float> out((size_t) n_frames * n_mel);+ if (!t.assert_true("log mel", clip_audio_log_mel(config, samples.data(), cache.hann_window.data(), n_frames, out.data()))) {+ return;+ }++ const int n_bins = n_fft / 2 + 1;+ double max_diff = 0.0;+ std::vector<double> power(n_bins);+ for (int i = 0; i < n_frames; i++) {+ for (int k = 0; k < n_bins; k++) {+ double re = 0.0, im = 0.0;+ for (int j = 0; j < n_fft; j++) {+ const double x = (double) cache.hann_window[j] * samples[(size_t) i * hop + j];+ re += x * std::cos(2 * pi * j * k / n_fft);+ im -= x * std::sin(2 * pi * j * k / n_fft);+ }+ power[k] = re * re + im * im;+ }+ for (int m = 0; m < n_mel; m++) {+ double sum = 0.0;+ for (int k = 0; k < n_bins; k++) {+ sum += power[k] * cache.filters.data[(size_t) m * n_bins + k];+ }+ const double expected = std::log10(std::max(sum, (double) config.mel_floor));+ max_diff = std::max(max_diff, std::fabs(expected - out[(size_t) m * n_frames + i]));+ }+ }+ t.assert_true("max_diff = " + std::to_string(max_diff), max_diff < 0.05);+}++MAKE_TEST(test_audio_log_mel_blocks) {+ const int n_mel = 80;+ const int n_fft = 400;+ const int hop = 160;+ const int n_frames = 8192 + 100; // long audio is split into blocks of 8192 frames++ mtmd_audio_cache cache;+ cache.fill_sin_cos_table(n_fft);+ cache.fill_hann_window(n_fft, true);+ cache.fill_mel_filterbank_matrix(n_mel, n_fft, 16000);++ ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr));+ ggml_backend_t backends[] = { backend.get() };+ ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true));++ mtmd_audio_mel_config config;+ config.sched = sched.get();+ config.n_threads = 2;+ config.n_mel = n_mel;+ config.n_fft_bins = 1 + n_fft / 2;+ config.hop_length = hop;+ config.cache = &cache;++ std::vector<float> samples((size_t) (n_frames - 1) * hop + n_fft);+ for (size_t i = 0; i < samples.size(); i++) {+ samples[i] = (float) (0.3 * std::sin(0.05 * i + 1e-6 * i * i));+ }++ // frames are independent: the frames around the seam must not change when computed on their own+ // not bit-exact, matmul kernels may round differently depending on the shape+ const int off = 8192 - 50;+ const int n_sub = n_frames - off;+ std::vector<float> all((size_t) n_frames * n_mel);+ std::vector<float> sub((size_t) n_sub * n_mel);+ const float * window = cache.hann_window.data();+ if (!t.assert_true("all", clip_audio_log_mel(config, samples.data(), window, n_frames, all.data())) ||+ !t.assert_true("sub", clip_audio_log_mel(config, samples.data() + (size_t) off * hop, window, n_sub, sub.data()))) {+ return;+ }++ float max_diff = 0.0f;+ for (int j = 0; j < n_mel; j++) {+ for (int i = 0; i < n_sub; i++) {+ max_diff = std::max(max_diff, std::fabs(all[(size_t) j * n_frames + off + i] - sub[(size_t) j * n_sub + i]));+ }+ }+ t.assert_true("max_diff = " + std::to_string(max_diff), max_diff < 1e-5f);+}+ // // main //@@ -510,6 +672,9 @@ int main(int argc, char ** argv) { testing t(std::cout); t.verbose = true; + // the audio preprocessors run on the CPU backend+ ggml_backend_load_all();+ // the video helper logs every frame at debug level mtmd_helper_log_set([](ggml_log_level level, const char * text, void *) { if (level != GGML_LOG_LEVEL_DEBUG) {tools/mtmd/clip.cpp190 + / 13 −
@@ -3,11 +3,13 @@ #include "clip-model.h" #include "clip-graph.h" #include "models/models.h"+#include "mtmd-audio.h" #include "ggml.h" #include "ggml-cpp.h" #include "ggml-alloc.h" #include "ggml-backend.h"+#include "ggml-cpu.h" #include "gguf.h" #include <algorithm>@@ -17,6 +19,7 @@ #include <cstring> #include <fstream> #include <map>+#include <memory> #include <random> #include <stdexcept> #include <unordered_set>@@ -144,6 +147,38 @@ static void clip_image_convert_f32_to_u8(const clip_image_f32& src, clip_image_u #endif +// ggml_backend_cpu_set_n_threads(backend_cpu, n_threads);+static void clip_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) {+ ggml_backend_dev_t dev = ggml_backend_get_device(backend_cpu);+ ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;+ if (reg) {+ auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");+ if (ggml_backend_set_n_threads_fn) {+ ggml_backend_set_n_threads_fn(backend_cpu, n_threads);+ }+ }+}++static std::shared_ptr<ggml_threadpool> clip_threadpool_init(int n_threads) {+ ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);+ ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;+ if (!reg || n_threads < 2) {+ return nullptr;+ }+ auto * new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new");+ auto * free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free");+ if (!new_fn || !free_fn) {+ return nullptr;+ }+ ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads);+ tpp.poll = 0; // the pool is idle most of the time, don't take CPU away from the text model+ ggml_threadpool_t threadpool = new_fn(&tpp);+ if (!threadpool) {+ throw std::runtime_error("failed to create threadpool");+ }+ return std::shared_ptr<ggml_threadpool>(threadpool, free_fn);+}+ struct clip_ctx { clip_model model; @@ -155,6 +190,8 @@ struct clip_ctx { std::vector<ggml_backend_t> backend_ptrs; std::vector<ggml_backend_buffer_type_t> backend_buft; + std::shared_ptr<ggml_threadpool> threadpool; // shared by the contexts of a model+ ggml_backend_t backend = nullptr; ggml_backend_t backend_cpu = nullptr; ggml_backend_buffer_ptr buf;@@ -174,13 +211,20 @@ struct clip_ctx { bool support_batch = false; - clip_ctx(clip_context_params & ctx_params) {+ clip_ctx(clip_context_params & ctx_params, std::shared_ptr<ggml_threadpool> threadpool) : threadpool(std::move(threadpool)) { flash_attn_type = ctx_params.flash_attn_type; no_alloc = ctx_params.no_alloc; backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); if (!backend_cpu) { throw std::runtime_error("failed to initialize CPU backend"); }+ if (this->threadpool) {+ ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu));+ auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool");+ if (set_threadpool_fn) {+ set_threadpool_fn(backend_cpu, this->threadpool.get());+ }+ } if (ctx_params.use_gpu) { if (ctx_params.device != nullptr) { backend = ggml_backend_dev_init(ctx_params.device, nullptr);@@ -5100,15 +5144,20 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params clip_ctx * ctx_audio = nullptr; clip_ctx * ctx_gen_audio = nullptr; + std::shared_ptr<ggml_threadpool> threadpool;+ try {+ // no_alloc only measures memory, nothing is computed+ threadpool = clip_threadpool_init(ctx_params.no_alloc ? 1 : ctx_params.n_threads);+ clip_model_loader loader(fname, /* skip_tensors */ false, ctx_params.progress_callback, ctx_params.progress_callback_user_data); bool skip_audio = false; if (loader.has_vision) {- ctx_vision = new clip_ctx(ctx_params);+ ctx_vision = new clip_ctx(ctx_params, threadpool); loader.load_hparams(ctx_vision->model, CLIP_MODALITY_VISION); loader.load_tensors(*ctx_vision); loader.init_ctx(*ctx_vision);@@ -5122,7 +5171,7 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params } if (loader.has_audio && !skip_audio) {- ctx_audio = new clip_ctx(ctx_params);+ ctx_audio = new clip_ctx(ctx_params, threadpool); loader.load_hparams(ctx_audio->model, CLIP_MODALITY_AUDIO); loader.load_tensors(*ctx_audio); loader.init_ctx(*ctx_audio);@@ -5132,7 +5181,7 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params } if (loader.has_gen_audio) {- ctx_gen_audio = new clip_ctx(ctx_params);+ ctx_gen_audio = new clip_ctx(ctx_params, threadpool); loader.load_hparams(ctx_gen_audio->model, CLIP_MODALITY_GEN_AUDIO); // TODO: CUDA has no flash attention kernel for the depthformer head dim 32, enable it once it does if (ctx_gen_audio->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN && ctx_gen_audio->flash_attn_type == CLIP_FLASH_ATTN_TYPE_AUTO) {@@ -5608,6 +5657,142 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { return n_patches; } +ggml_backend_sched_t clip_get_sched(const struct clip_ctx * ctx) {+ return ctx->sched.get();+}++// TODO: remove this and use ggml_mul_mat with GGML_PREC_F32 once all backends honor it for F32 x F32+// ggml_mul_mat for 2D F32 tensors, always in F32 on CPU: some GPU backends round the inputs to F16 or TF32, which loses the quiet bins of a spectrum+static void mtmd_mul_mat_prec_op(ggml_tensor * dst, int ith, int nth, void * /*userdata*/) {+ const ggml_tensor * a = dst->src[0];+ const ggml_tensor * b = dst->src[1];+ const int64_t n_k = a->ne[0];++ for (int64_t n = ith; n < b->ne[1]; n += nth) {+ const float * b_row = (const float *) ((const char *) b->data + n * b->nb[1]);+ float * dst_row = (float *) ((char *) dst->data + n * dst->nb[1]);+ for (int64_t m = 0; m < a->ne[1]; m++) {+ const float * a_row = (const float *) ((const char *) a->data + m * a->nb[1]);+ // independent partial sums, so that the compiler can vectorize the loop+ float acc[16] = {};+ int64_t k = 0;+ for (; k + 16 <= n_k; k += 16) {+ for (int l = 0; l < 16; l++) {+ acc[l] += a_row[k + l] * b_row[k + l];+ }+ }+ float sum = 0.0f;+ for (; k < n_k; k++) {+ sum += a_row[k] * b_row[k];+ }+ for (int l = 0; l < 16; l++) {+ sum += acc[l];+ }+ dst_row[m] = sum;+ }+ }+}++static ggml_tensor * mtmd_mul_mat_prec(ggml_context * ctx, ggml_tensor * a, ggml_tensor * b) {+ GGML_ASSERT(a->type == GGML_TYPE_F32 && b->type == GGML_TYPE_F32);+ GGML_ASSERT(ggml_n_dims(a) <= 2 && ggml_n_dims(b) <= 2 && a->ne[0] == b->ne[0]);+ GGML_ASSERT(a->nb[0] == sizeof(float) && b->nb[0] == sizeof(float));+ ggml_tensor * args[] = { a, b };+ return ggml_custom_4d(ctx, GGML_TYPE_F32, a->ne[1], b->ne[1], 1, 1, args, 2, mtmd_mul_mat_prec_op, GGML_N_TASKS_MAX, nullptr);+}++bool clip_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out) {+ ggml_backend_sched_t sched = config.sched;+ ggml_backend_t backend_cpu = ggml_backend_sched_get_backend(sched, ggml_backend_sched_get_n_backends(sched) - 1);+ clip_backend_cpu_set_n_threads(backend_cpu, std::max(1, config.n_threads));++ const auto & cache = *config.cache;+ const auto & filters = cache.filters.data;+ const int64_t n_fft_bins = config.n_fft_bins;+ const int64_t frame_size = (n_fft_bins - 1) * 2;+ const int64_t n_mel = config.n_mel;+ GGML_ASSERT(filters.size() == (size_t) (n_mel * n_fft_bins));+ GGML_ASSERT(cache.cos_vals.size() == (size_t) frame_size && cache.sin_vals.size() == (size_t) frame_size);++ // DFT matrix with the window folded in, the first n_fft_bins rows give the real part+ std::vector<float> dft((size_t) 2 * n_fft_bins * frame_size);+ for (int64_t k = 0; k < n_fft_bins; k++) {+ for (int64_t j = 0; j < frame_size; j++) {+ const int64_t q = (j * k) % frame_size;+ dft[k * frame_size + j] = window[j] * cache.cos_vals[q];+ dft[(k + n_fft_bins) * frame_size + j] = window[j] * cache.sin_vals[q];+ }+ }++ // the spectrum is several times larger than the result, so long audio is processed in blocks+ const int64_t n_frames_max = 8192;++ for (int64_t off = 0; off < n_frames; off += n_frames_max) {+ const int64_t n_cur = std::min(n_frames_max, n_frames - off);++ ggml_init_params params = {+ /*.mem_size =*/ ggml_tensor_overhead() * 32 + ggml_graph_overhead(),+ /*.mem_buffer =*/ nullptr,+ /*.no_alloc =*/ true,+ };+ ggml_context_ptr ctx_ptr(ggml_init(params));+ ggml_context * ctx0 = ctx_ptr.get();++ ggml_tensor * inp_samples = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, (n_cur - 1) * config.hop_length + frame_size);+ ggml_tensor * inp_dft = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, frame_size, 2 * n_fft_bins);+ ggml_tensor * inp_filters = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_fft_bins, n_mel);+ ggml_set_input(inp_samples);+ ggml_set_input(inp_dft);+ ggml_set_input(inp_filters);++ ggml_tensor * cur = ggml_cont(ctx0, ggml_view_2d(ctx0, inp_samples, frame_size, n_cur, config.hop_length * ggml_element_size(inp_samples), 0));+ cur = mtmd_mul_mat_prec(ctx0, inp_dft, cur); // [2 * n_fft_bins, n_cur]++ cur = ggml_sqr(ctx0, cur);+ cur = ggml_add(ctx0,+ ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], 0),+ ggml_view_2d(ctx0, cur, n_fft_bins, n_cur, cur->nb[1], n_fft_bins * ggml_element_size(cur)));+ if (config.use_magnitude) {+ cur = ggml_sqrt(ctx0, cur);+ }++ cur = mtmd_mul_mat_prec(ctx0, cur, inp_filters); // [n_cur, n_mel]+ cur = config.mel_floor_add+ ? ggml_scale_bias(ctx0, cur, 1.0f, config.mel_floor)+ : ggml_clamp(ctx0, cur, config.mel_floor, FLT_MAX);+ cur = ggml_log(ctx0, cur);+ if (!config.use_natural_log) {+ cur = ggml_scale(ctx0, cur, 1.0f / logf(10.0f));+ }+ ggml_set_output(cur);++ ggml_cgraph * gf = ggml_new_graph(ctx0);+ ggml_build_forward_expand(gf, cur);++ ggml_backend_sched_reset(sched);+ if (!ggml_backend_sched_alloc_graph(sched, gf)) {+ LOG_ERR("%s: failed to allocate compute graph\n", __func__);+ return false;+ }++ ggml_backend_tensor_set(inp_samples, samples + off * config.hop_length, 0, ggml_nbytes(inp_samples));+ ggml_backend_tensor_set(inp_dft, dft.data(), 0, ggml_nbytes(inp_dft));+ ggml_backend_tensor_set(inp_filters, filters.data(), 0, ggml_nbytes(inp_filters));++ auto status = ggml_backend_sched_graph_compute(sched, gf);+ if (status != GGML_STATUS_SUCCESS) {+ LOG_ERR("%s: ggml_backend_sched_graph_compute failed with error %d\n", __func__, status);+ return false;+ }++ for (int64_t j = 0; j < n_mel; j++) {+ ggml_backend_tensor_get(cur, out + j * n_frames + off, j * cur->nb[1], n_cur * ggml_element_size(cur));+ }+ }++ return true;+}+ bool clip_image_encode(struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector<float> & out_vec) { clip_image_f32_batch imgs; clip_image_f32 img_copy = *img;@@ -7520,15 +7705,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { GGML_ABORT("Unknown projector type"); } - // ggml_backend_cpu_set_n_threads(ctx->backend_cpu, n_threads);- ggml_backend_dev_t dev = ggml_backend_get_device(ctx->backend_cpu);- ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;- if (reg) {- auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");- if (ggml_backend_set_n_threads_fn) {- ggml_backend_set_n_threads_fn(ctx->backend_cpu, params->n_threads);- }- }+ clip_backend_cpu_set_n_threads(ctx->backend_cpu, params->n_threads); auto status = ggml_backend_sched_graph_compute(ctx->sched.get(), gf); if (status != GGML_STATUS_SUCCESS) {tools/mtmd/clip.h8 + / 0 −
@@ -60,6 +60,7 @@ struct clip_context_params { bool no_alloc; mtmd_progress_callback progress_callback; void * progress_callback_user_data;+ int n_threads; }; struct clip_init_result {@@ -88,6 +89,13 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img); // this should be equal to the embedding dimension of the text model int clip_n_mmproj_embd(const struct clip_ctx * ctx); +ggml_backend_sched_t clip_get_sched(const struct clip_ctx * ctx);++// compute the log-mel of n_frames frames, the waveform must already be padded and window must hold one whole frame+// out has the layout [n_frames, n_mel]; like the encoders, must not run at the same time as another pass on config.sched+struct mtmd_audio_mel_config;+bool clip_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out);+ // TODO: remove clip_image_encode() and always use batched version bool clip_image_encode (struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector<float> & out_vec); bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, std::vector<float> & out_batch_embd);tools/mtmd/mtmd-audio.cpp237 + / 395 −
@@ -4,7 +4,6 @@ #include <cmath> #include <cstdint> #include <cstring>-#include <thread> #include <vector> #include <fstream> #include <algorithm>@@ -270,100 +269,44 @@ static void ifft(const mtmd_audio_cache & cache, float * in, int N, float * out) fft_impl<true, false>(cache, in, N, out); } -struct filter_params {- int64_t n_mel;- int64_t n_fft_bins;- int32_t hann_window_size;- int32_t hop_length;- int32_t sample_rate;- bool no_padding = false;- bool center_padding = false;- float preemph = 0.f;- bool use_natural_log = false;- bool norm_per_feature = false;- bool use_magnitude = false; // |X| instead of |X|^2- float mel_floor = 5.960464477539063e-08f;- bool mel_floor_add = false; // log(x + floor) instead of log(max(x, floor))- bool std_eps_after_sqrt = false; // std + eps instead of sqrt(var + eps)-};--static void log_mel_spectrogram_worker_thread(int ith,- const float * hann,- const std::vector<float> & samples,- int n_samples,- int frame_size,- int frame_step,- int n_threads,- const filter_params & params,- const mtmd_audio_cache & cache,- mtmd_audio_mel & out) {- std::vector<float> fft_in(frame_size * 2, 0.0);- std::vector<float> fft_out(frame_size * 2 * 2 * 2);-- int64_t n_fft_bins = params.n_fft_bins;- int64_t i = ith;-- const auto & filters = cache.filters;-- // make sure n_fft == 1 + (WHISPER_N_FFT / 2), bin_0 to bin_nyquist- GGML_ASSERT(n_fft_bins == 1 + (frame_size / 2));- GGML_ASSERT(cache.sin_vals.size() == cache.cos_vals.size());- // calculate FFT only when fft_in are not all zero- for (; i < std::min((int64_t)(n_samples / frame_step + 1), out.n_len); i += n_threads) {- const int64_t offset = i * frame_step;-- // apply Hann window (~10% faster)- const int valid_len = std::min(frame_size, std::max(0, n_samples - (int)offset));- for (int j = 0; j < valid_len; j++) {- fft_in[j] = hann[j] * samples[offset + j];- }-- // fill the rest with zeros- if (valid_len < frame_size) {- std::fill(fft_in.begin() + valid_len, fft_in.end(), 0.0);- }-- // FFT- fft(cache, fft_in.data(), frame_size, fft_out.data());-- // Calculate modulus^2 (power) or modulus (magnitude)- for (int j = 0; j < n_fft_bins; j++) {- float power = (fft_out[2 * j + 0] * fft_out[2 * j + 0] + fft_out[2 * j + 1] * fft_out[2 * j + 1]);- fft_out[j] = params.use_magnitude ? sqrtf(power) : power;- }-- // mel spectrogram- for (int64_t j = 0; j < out.n_mel; j++) {- double sum = 0.0;- // unroll loop (suggested by GH user @lunixbochs)- int k = 0;- for (k = 0; k < n_fft_bins - 3; k += 4) {- size_t idx = size_t(j) * size_t(n_fft_bins) + size_t(k);- sum +=- fft_out[k + 0] * filters.data[idx + 0] +- fft_out[k + 1] * filters.data[idx + 1] +- fft_out[k + 2] * filters.data[idx + 2] +- fft_out[k + 3] * filters.data[idx + 3];- }- // handle n_fft remainder- for (; k < n_fft_bins; k++) {- sum += fft_out[k] * filters.data[(size_t)j * n_fft_bins + k];- }- sum = params.mel_floor_add ? sum + (double)params.mel_floor : std::max(sum, (double)params.mel_floor);- sum = params.use_natural_log- ? log(sum)- : log10(sum);- out.data[(size_t)j * out.n_len + i] = sum;+static void mtmd_audio_fill_entries(std::vector<mtmd_audio_mel> & mels, bool shape_only, std::vector<clip_image_f32> & entries) {+ entries.clear();+ for (auto & mel : mels) {+ GGML_ASSERT(mel.n_len <= INT32_MAX && mel.n_len >= 0);+ GGML_ASSERT(mel.n_mel <= INT32_MAX && mel.n_mel >= 0);+ clip_image_f32 entry;+ entry.set_size({(int) mel.n_len, (int) mel.n_mel}, shape_only || mel.data.empty(), /* is_audio */ true);+ if (!entry.is_placeholder()) {+ entry.cpy_buf(mel.data); }+ entries.push_back(std::move(entry)); }+} - // Otherwise fft_out are all zero- double sum = params.use_natural_log ? log(1e-10) : log10(1e-10);- for (; i < out.n_len; i += n_threads) {- for (int64_t j = 0; j < out.n_mel; j++) {- out.data[(size_t)j * out.n_len + i] = sum;+mtmd_audio_preproc_out mtmd_audio_preprocessor::preprocess(const float * samples, size_t n_samples) const {+ mtmd_audio_preproc_out out;+ const bool deferred = mel_config.cache != nullptr;+ std::vector<mtmd_audio_mel> mels;+ if (process(samples, n_samples, mels, deferred)) {+ mtmd_audio_fill_entries(mels, deferred, out.entries);+ if (deferred) {+ out.samples.assign(samples, samples + n_samples); } }+ return out;+}++bool mtmd_audio_preprocessor::compute(mtmd_audio_preproc_out & out) const {+ if (!out.needs_compute()) {+ return true;+ }+ std::vector<mtmd_audio_mel> mels;+ if (!process(out.samples.data(), out.samples.size(), mels, false)) {+ return false;+ }+ mtmd_audio_fill_entries(mels, false, out.entries);+ out.samples = {};+ return true; } void mtmd_audio_log_mel_frame(const mtmd_audio_cache & cache, const float * samples, int n_fft, int n_mel, float * out) {@@ -395,15 +338,16 @@ void mtmd_audio_log_mel_frame(const mtmd_audio_cache & cache, const float * samp static bool log_mel_spectrogram( const float * samples, const int n_samples_in,- const int n_threads,- const filter_params & params,- const mtmd_audio_cache & cache,+ const mtmd_audio_mel_config & params,+ const bool shape_only, mtmd_audio_mel & out) { //const int64_t t_start_us = ggml_time_us(); out.n_len_org = n_samples_in; int n_samples = n_samples_in; + const auto & cache = *params.cache;+ // Hann window const float * hann = cache.hann_window.data(); const int frame_size = (params.n_fft_bins - 1) * 2;@@ -486,20 +430,8 @@ static bool log_mel_spectrogram( } out.data.resize(total_size); - {- std::vector<std::thread> workers(n_threads - 1);- for (int iw = 0; iw < n_threads - 1; ++iw) {- workers[iw] =- std::thread(log_mel_spectrogram_worker_thread, iw + 1, hann, std::cref(samples_padded), n_samples,- frame_size, frame_step, n_threads, std::cref(params), std::cref(cache), std::ref(out));- }-- // main thread- log_mel_spectrogram_worker_thread(0, hann, samples_padded, n_samples, frame_size, frame_step, n_threads, params,- cache, out);- for (int iw = 0; iw < n_threads - 1; ++iw) {- workers[iw].join();- }+ if (!shape_only && !clip_audio_log_mel(params, samples_padded.data(), hann, out.n_len, out.data.data())) {+ return false; } const int64_t effective_n_len = n_samples_in / frame_step;@@ -572,11 +504,23 @@ void mtmd_audio_preprocessor_whisper::initialize() { cache.fill_sin_cos_table(hparams.audio_n_fft); cache.fill_hann_window(hparams.audio_window_len, true); cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);++ mel_config.n_mel = hparams.n_mel_bins;+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);+ mel_config.hann_window_size = hparams.audio_window_len;+ mel_config.hop_length = hparams.audio_hop_len;+ mel_config.sample_rate = hparams.audio_sample_rate;+ mel_config.center_padding = false;+ mel_config.preemph = 0.0f; // disabled+ mel_config.use_natural_log = false;+ mel_config.norm_per_feature = false;+ mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_whisper::preprocess(const float * samples,- size_t n_samples,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_whisper::process(const float * samples,+ size_t n_samples,+ std::vector<mtmd_audio_mel> & output,+ bool shape_only) const { if (n_samples == 0) { // empty audio return false;@@ -592,26 +536,13 @@ bool mtmd_audio_preprocessor_whisper::preprocess(const float * s n_samples = smpl.size(); } - filter_params params;- params.n_mel = hparams.n_mel_bins;- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);- params.hann_window_size = hparams.audio_window_len;- params.hop_length = hparams.audio_hop_len;- params.sample_rate = hparams.audio_sample_rate;- params.center_padding = false;- params.preemph = 0.0f; // disabled- params.use_natural_log = false;- params.norm_per_feature = false;- // make sure the cache is initialized GGML_ASSERT(!cache.sin_vals.empty()); GGML_ASSERT(!cache.cos_vals.empty()); GGML_ASSERT(!cache.filters.data.empty()); mtmd_audio_mel out_full;- bool ok = log_mel_spectrogram(samples, n_samples,- 4, // n_threads- params, cache, out_full);+ bool ok = log_mel_spectrogram(samples, n_samples, mel_config, shape_only, out_full); if (!ok) { return false; }@@ -660,11 +591,21 @@ void mtmd_audio_preprocessor_qwen3a::initialize() { cache.fill_sin_cos_table(hparams.audio_n_fft); cache.fill_hann_window(hparams.audio_window_len, true); cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);++ mel_config.n_mel = hparams.n_mel_bins;+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);+ mel_config.hann_window_size = hparams.audio_window_len;+ mel_config.hop_length = hparams.audio_hop_len;+ mel_config.sample_rate = hparams.audio_sample_rate;+ mel_config.no_padding = true; // reflection padding applied in preprocess()+ mel_config.use_natural_log = false; // log10+ mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * samples,- size_t n_samples,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_qwen3a::process(const float * samples,+ size_t n_samples,+ std::vector<mtmd_audio_mel> & output,+ bool shape_only) const { if (n_samples == 0) { return false; }@@ -689,17 +630,8 @@ bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * sa padded[n_samples + pad + i] = (src >= 0) ? samples[src] : 0.0f; } - filter_params params;- params.n_mel = hparams.n_mel_bins;- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);- params.hann_window_size = hparams.audio_window_len;- params.hop_length = hparams.audio_hop_len;- params.sample_rate = hparams.audio_sample_rate;- params.no_padding = true; // reflection padding already applied above- params.use_natural_log = false; // log10- mtmd_audio_mel mel_full;- bool ok = log_mel_spectrogram(padded.data(), (int)padded.size(), 4, params, cache, mel_full);+ bool ok = log_mel_spectrogram(padded.data(), (int)padded.size(), mel_config, shape_only, mel_full); if (!ok) { return false; }@@ -762,11 +694,21 @@ void mtmd_audio_preprocessor_dots3note::initialize() { cache.fill_sin_cos_table(hparams.audio_n_fft); cache.fill_hann_window(hparams.audio_window_len, true); cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);++ mel_config.n_mel = hparams.n_mel_bins;+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);+ mel_config.hann_window_size = hparams.audio_window_len;+ mel_config.hop_length = hparams.audio_hop_len;+ mel_config.sample_rate = hparams.audio_sample_rate;+ mel_config.no_padding = true; // padding applied in preprocess()+ mel_config.use_natural_log = false;+ mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_dots3note::preprocess(const float * samples,- size_t n_samples,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_dots3note::process(const float * samples,+ size_t n_samples,+ std::vector<mtmd_audio_mel> & output,+ bool shape_only) const { if (n_samples == 0) { return false; }@@ -803,17 +745,8 @@ bool mtmd_audio_preprocessor_dots3note::preprocess(const float * } } - filter_params params;- params.n_mel = hparams.n_mel_bins;- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);- params.hann_window_size = hparams.audio_window_len;- params.hop_length = hop;- params.sample_rate = hparams.audio_sample_rate;- params.no_padding = true; // padding already applied above- params.use_natural_log = false;- mtmd_audio_mel mel_full;- if (!log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, mel_full)) {+ if (!log_mel_spectrogram(padded.data(), (int) padded.size(), mel_config, shape_only, mel_full)) { return false; } GGML_ASSERT(mel_full.n_len >= n_valid);@@ -862,11 +795,24 @@ void mtmd_audio_preprocessor_mimo_audio::initialize() { /*scale=*/ 1.0f, /*use_htk=*/ true );++ mel_config.n_mel = hparams.n_mel_bins;+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);+ mel_config.hann_window_size = hparams.audio_window_len;+ mel_config.hop_length = hparams.audio_hop_len;+ mel_config.sample_rate = hparams.audio_sample_rate;+ mel_config.no_padding = true; // reflect padding applied in preprocess()+ mel_config.use_natural_log = true;+ mel_config.use_magnitude = true;+ mel_config.mel_floor = 1e-7f;+ mel_config.norm_per_feature = false;+ mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_mimo_audio::preprocess(const float * samples,- size_t n_samples,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_mimo_audio::process(const float * samples,+ size_t n_samples,+ std::vector<mtmd_audio_mel> & output,+ bool shape_only) const { if (n_samples == 0) { return false; }@@ -888,20 +834,8 @@ bool mtmd_audio_preprocessor_mimo_audio::preprocess(const float * padded[n_samples + pad + i] = (src >= 0) ? samples[src] : 0.0f; } - filter_params params;- params.n_mel = hparams.n_mel_bins;- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);- params.hann_window_size = hparams.audio_window_len;- params.hop_length = hparams.audio_hop_len;- params.sample_rate = hparams.audio_sample_rate;- params.no_padding = true; // reflect padding already applied above- params.use_natural_log = true;- params.use_magnitude = true;- params.mel_floor = 1e-7f;- params.norm_per_feature = false;- mtmd_audio_mel out;- bool ok = log_mel_spectrogram(padded.data(), (int)padded.size(), 4, params, cache, out);+ bool ok = log_mel_spectrogram(padded.data(), (int)padded.size(), mel_config, shape_only, out); if (!ok) { return false; }@@ -921,11 +855,23 @@ void mtmd_audio_preprocessor_qwen3tts_spk::initialize() { cache.fill_sin_cos_table(hparams.audio_n_fft); cache.fill_hann_window(hparams.audio_window_len, true); cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);++ mel_config.n_mel = hparams.n_mel_bins;+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);+ mel_config.hann_window_size = hparams.audio_window_len;+ mel_config.hop_length = hparams.audio_hop_len;+ mel_config.sample_rate = hparams.audio_sample_rate;+ mel_config.no_padding = true; // reflect padding applied in preprocess()+ mel_config.use_natural_log = true;+ mel_config.use_magnitude = true;+ mel_config.mel_floor = 1e-5f;+ mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float * samples,- size_t n_samples,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_qwen3tts_spk::process(const float * samples,+ size_t n_samples,+ std::vector<mtmd_audio_mel> & output,+ bool shape_only) const { if (n_samples == 0) { return false; }@@ -949,19 +895,8 @@ bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float * padded[n_samples + pad + i] = samples[n_samples - 2 - i]; } - filter_params params;- params.n_mel = hparams.n_mel_bins;- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);- params.hann_window_size = hparams.audio_window_len;- params.hop_length = hparams.audio_hop_len;- params.sample_rate = hparams.audio_sample_rate;- params.no_padding = true; // reflect padding already applied above- params.use_natural_log = true;- params.use_magnitude = true;- params.mel_floor = 1e-5f;- mtmd_audio_mel out;- bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, out);+ bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), mel_config, shape_only, out); if (!ok) { return false; }@@ -1294,9 +1229,10 @@ void mtmd_audio_preprocessor_chatterbox_ref::initialize() { matcha_cache.fill_mel_filterbank_matrix(80, 1920, 24000, 0.0f, 8000.0f); } -bool mtmd_audio_preprocessor_chatterbox_ref::preprocess(const float * samples,- size_t n_samples,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_chatterbox_ref::process(const float * samples,+ size_t n_samples,+ std::vector<mtmd_audio_mel> & output,+ bool /*shape_only*/) const { output.clear(); const int sr = (int) hparams.audio_sample_rate; @@ -1453,38 +1389,37 @@ void mtmd_audio_preprocessor_conformer::initialize() { // NeMo uses a symmetric window: torch.hann_window(periodic=False) cache.fill_hann_window(hparams.audio_window_len, false); cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);++ mel_config.n_mel = hparams.n_mel_bins;+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);+ mel_config.hann_window_size = hparams.audio_window_len;+ mel_config.hop_length = hparams.audio_hop_len;+ mel_config.sample_rate = hparams.audio_sample_rate;+ mel_config.center_padding = true;+ mel_config.preemph = 0.97f;+ mel_config.use_natural_log = true;+ mel_config.norm_per_feature = true;+ mel_config.mel_floor_add = true;+ mel_config.std_eps_after_sqrt = true;+ mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_conformer::preprocess(const float * samples,- size_t n_samples,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_conformer::process(const float * samples,+ size_t n_samples,+ std::vector<mtmd_audio_mel> & output,+ bool shape_only) const { // empty audio if (n_samples == 0) { return false; } - filter_params params;- params.n_mel = hparams.n_mel_bins;- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);- params.hann_window_size = hparams.audio_window_len;- params.hop_length = hparams.audio_hop_len;- params.sample_rate = hparams.audio_sample_rate;- params.center_padding = true;- params.preemph = 0.97f;- params.use_natural_log = true;- params.norm_per_feature = true;- params.mel_floor_add = true;- params.std_eps_after_sqrt = true;- // make sure the cache is initialized GGML_ASSERT(!cache.sin_vals.empty()); GGML_ASSERT(!cache.cos_vals.empty()); GGML_ASSERT(!cache.filters.data.empty()); mtmd_audio_mel out_full;- bool ok = log_mel_spectrogram(samples, n_samples,- 4, // n_threads- params, cache, out_full);+ bool ok = log_mel_spectrogram(samples, n_samples, mel_config, shape_only, out_full); if (!ok) { return false; }@@ -1497,9 +1432,10 @@ bool mtmd_audio_preprocessor_conformer::preprocess(const float * // mtmd_audio_preprocessor_d1omni // -bool mtmd_audio_preprocessor_d1omni::preprocess(const float * samples,- size_t n_samples,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_d1omni::process(const float * samples,+ size_t n_samples,+ std::vector<mtmd_audio_mel> & output,+ bool shape_only) const { if (n_samples == 0) { return false; }@@ -1509,7 +1445,7 @@ bool mtmd_audio_preprocessor_d1omni::preprocess(const float * sa std::vector<float> buf(samples, samples + std::min(n_samples, n_max)); buf.resize(std::max(buf.size(), n_min), 0.0f); - if (!mtmd_audio_preprocessor_conformer::preprocess(buf.data(), buf.size(), output)) {+ if (!mtmd_audio_preprocessor_conformer::process(buf.data(), buf.size(), output, shape_only)) { return false; } @@ -1537,11 +1473,21 @@ void mtmd_audio_preprocessor_lfm3a::initialize() { cache.fill_sin_cos_table(hparams.audio_n_fft); cache.fill_hann_window(hparams.audio_window_len, true); cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);++ mel_config.n_mel = hparams.n_mel_bins;+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);+ mel_config.hann_window_size = hparams.audio_window_len;+ mel_config.hop_length = hparams.audio_hop_len;+ mel_config.sample_rate = hparams.audio_sample_rate;+ mel_config.no_padding = true;+ mel_config.mel_floor = 1e-12f; // log10 floor of -12+ mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_lfm3a::preprocess(const float * samples,- size_t n_samples,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_lfm3a::process(const float * samples,+ size_t n_samples,+ std::vector<mtmd_audio_mel> & output,+ bool shape_only) const { if (n_samples == 0) { return false; }@@ -1557,21 +1503,12 @@ bool mtmd_audio_preprocessor_lfm3a::preprocess(const float * sam std::vector<float> padded(samples, samples + n_samples); padded.resize(n_padded, 0.0f); - filter_params params;- params.n_mel = hparams.n_mel_bins;- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);- params.hann_window_size = hparams.audio_window_len;- params.hop_length = hparams.audio_hop_len;- params.sample_rate = hparams.audio_sample_rate;- params.no_padding = true;- params.mel_floor = 1e-12f; // log10 floor of -12- GGML_ASSERT(!cache.sin_vals.empty()); GGML_ASSERT(!cache.cos_vals.empty()); GGML_ASSERT(!cache.filters.data.empty()); mtmd_audio_mel out;- bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, out);+ bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), mel_config, shape_only, out); if (!ok) { return false; }@@ -1594,11 +1531,25 @@ void mtmd_audio_preprocessor_granite_speech::initialize() { cache.fill_mel_filterbank_matrix( hparams.n_mel_bins / 2, hparams.audio_n_fft, hparams.audio_sample_rate, 0.0f, -1.0f, false, 1.0f, true);++ mel_config.n_mel = hparams.n_mel_bins / 2;+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);+ mel_config.hann_window_size = hparams.audio_window_len;+ mel_config.hop_length = hparams.audio_hop_len;+ mel_config.sample_rate = hparams.audio_sample_rate;+ mel_config.no_padding = true;+ mel_config.center_padding = false;+ mel_config.preemph = 0.0f;+ mel_config.use_natural_log = false;+ mel_config.norm_per_feature = false;+ mel_config.mel_floor = 1e-10f;+ mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_granite_speech::preprocess(const float * samples,- size_t n_samples,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_granite_speech::process(const float * samples,+ size_t n_samples,+ std::vector<mtmd_audio_mel> & output,+ bool shape_only) const { if (n_samples == 0) { return false; }@@ -1629,21 +1580,8 @@ bool mtmd_audio_preprocessor_granite_speech::preprocess(const float * padded[pad + (int)n_samples + i] = samples[src]; } - filter_params params;- params.n_mel = hparams.n_mel_bins / 2;- params.n_fft_bins = 1 + (n_fft / 2);- params.hann_window_size = hparams.audio_window_len;- params.hop_length = hparams.audio_hop_len;- params.sample_rate = hparams.audio_sample_rate;- params.no_padding = true;- params.center_padding = false;- params.preemph = 0.0f;- params.use_natural_log = false;- params.norm_per_feature = false;- params.mel_floor = 1e-10f;- mtmd_audio_mel mel;- if (!log_mel_spectrogram(padded.data(), n_padded, 4, params, cache, mel)) {+ if (!log_mel_spectrogram(padded.data(), n_padded, mel_config, shape_only, mel)) { return false; } @@ -1708,11 +1646,26 @@ void mtmd_audio_preprocessor_gemma4a::initialize() { /*scale=*/ 1.0f, /*use_htk=*/ true );++ mel_config.n_mel = hparams.n_mel_bins;+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);+ mel_config.hann_window_size = hparams.audio_n_fft; // window is zero-padded to FFT size+ mel_config.hop_length = hparams.audio_hop_len;+ mel_config.sample_rate = hparams.audio_sample_rate;+ mel_config.no_padding = true;+ mel_config.center_padding = false;+ mel_config.preemph = 0.0f;+ mel_config.use_natural_log = true;+ mel_config.use_magnitude = true;+ mel_config.mel_floor = 0.001f;+ mel_config.norm_per_feature = false;+ mel_config.cache = &cache; } -bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * samples,- size_t n_samples,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_gemma4a::process(const float * samples,+ size_t n_samples,+ std::vector<mtmd_audio_mel> & output,+ bool shape_only) const { if (n_samples == 0) { return false; }@@ -1721,20 +1674,6 @@ bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * s GGML_ASSERT(!cache.cos_vals.empty()); GGML_ASSERT(!cache.filters.data.empty()); - filter_params params;- params.n_mel = hparams.n_mel_bins;- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);- params.hann_window_size = hparams.audio_n_fft; // window is zero-padded to FFT size- params.hop_length = hparams.audio_hop_len;- params.sample_rate = hparams.audio_sample_rate;- params.no_padding = true;- params.center_padding = false;- params.preemph = 0.0f;- params.use_natural_log = true;- params.use_magnitude = true;- params.mel_floor = 0.001f;- params.norm_per_feature = false;- // Split into 30-second chunks (model context limit, ~750 tokens each) const size_t chunk_samples = 30 * hparams.audio_sample_rate; for (size_t off = 0; off < n_samples; off += chunk_samples) {@@ -1754,7 +1693,7 @@ bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * s std::copy(chunk_ptr, chunk_ptr + chunk_len, padded_samples.data() + pad_left); mtmd_audio_mel out_chunk;- bool ok = log_mel_spectrogram(padded_samples.data(), padded_samples.size(), 4, params, cache, out_chunk);+ bool ok = log_mel_spectrogram(padded_samples.data(), padded_samples.size(), mel_config, shape_only, out_chunk); if (!ok) { return false; }@@ -1772,79 +1711,6 @@ bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * s // mtmd_audio_preprocessor_parakeet implementation // -void mtmd_audio_preprocessor_parakeet::worker_thread(- int ith,- const float * window_func,- int window_size,- const std::vector<float> & samples,- int n_samples,- int frame_size,- int frame_step,- int n_threads,- int n_fft_bins,- const mtmd_audio_cache & cache,- mtmd_audio_mel & mel) {- std::vector<float> fft_in(frame_size * 2, 0.0);- std::vector<float> fft_out(frame_size * 2 * 2 * 2);-- int n_fb = n_fft_bins;- int i = ith;-- GGML_ASSERT(n_fb == 1 + (frame_size / 2));-- const double eps = 5.960464477539063e-08;-- for (; i < std::min(n_samples / frame_step + 1, (int) mel.n_len); i += n_threads) {- const int offset = i * frame_step;- const int window_pad_left = (frame_size - window_size) / 2;-- // Zero-pad left.- std::fill(fft_in.begin(), fft_in.begin() + window_pad_left, 0.0f);-- // Apply windowed samples in the center.- const int n_to_process = std::min({window_size, n_samples - offset});- for (int j = 0; j < n_to_process; j++) {- fft_in[window_pad_left + j] = window_func[j] * samples[offset + window_pad_left + j];- }-- // Zero-pad right.- std::fill(fft_in.begin() + window_pad_left + n_to_process, fft_in.begin() + frame_size, 0.0f);-- // FFT.- fft(cache, fft_in.data(), frame_size, fft_out.data());-- // Calculate modulus^2 of complex numbers.- for (int j = 0; j < n_fb; j++) {- fft_out[j] = (fft_out[2 * j + 0] * fft_out[2 * j + 0] + fft_out[2 * j + 1] * fft_out[2 * j + 1]);- }-- // mel spectrogram.- for (int j = 0; j < mel.n_mel; j++) {- double sum = 0.0;- int k = 0;- for (k = 0; k < n_fb - 3; k += 4) {- sum +=- fft_out[k + 0] * cache.filters.data[j * n_fb + k + 0] +- fft_out[k + 1] * cache.filters.data[j * n_fb + k + 1] +- fft_out[k + 2] * cache.filters.data[j * n_fb + k + 2] +- fft_out[k + 3] * cache.filters.data[j * n_fb + k + 3];- }- for (; k < n_fb; k++) {- sum += fft_out[k] * cache.filters.data[j * n_fb + k];- }- mel.data[j * mel.n_len + i] = std::log(sum + eps);- }- }-- // Otherwise fft_out are all zero.- const double empty_sum = std::log(eps);- for (; i < mel.n_len; i += n_threads) {- for (int j = 0; j < mel.n_mel; j++) {- mel.data[j * mel.n_len + i] = empty_sum;- }- }-}- void mtmd_audio_preprocessor_parakeet::initialize() { cache.fill_sin_cos_table(hparams.audio_n_fft); @@ -1857,25 +1723,25 @@ void mtmd_audio_preprocessor_parakeet::initialize() { GGML_ASSERT(hparams.window.size() == (size_t)hparams.audio_window_len); GGML_ASSERT(hparams.window.size() <= (size_t) hparams.audio_n_fft); cache.hann_window = hparams.window;++ mel_config.n_mel = hparams.n_mel_bins;+ mel_config.n_fft_bins = 1 + (hparams.audio_n_fft / 2);+ mel_config.hann_window_size = hparams.audio_window_len;+ mel_config.hop_length = hparams.audio_hop_len;+ mel_config.sample_rate = hparams.audio_sample_rate;+ mel_config.no_padding = true; // preemphasis and padding already applied in log_mel()+ mel_config.use_natural_log = true;+ mel_config.mel_floor_add = true;+ mel_config.cache = &cache; } // log-mel without normalization-void mtmd_audio_preprocessor_parakeet::log_mel(const float * samples, size_t n_samples_in, mtmd_audio_mel & out_full) const {- filter_params params;- params.n_mel = hparams.n_mel_bins;- params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);- params.hann_window_size = hparams.audio_window_len;- params.hop_length = hparams.audio_hop_len;- params.sample_rate = hparams.audio_sample_rate;-+bool mtmd_audio_preprocessor_parakeet::log_mel(const float * samples, size_t n_samples_in, mtmd_audio_mel & out_full, bool shape_only) const { GGML_ASSERT(!cache.sin_vals.empty()); GGML_ASSERT(!cache.cos_vals.empty()); GGML_ASSERT(!cache.filters.data.empty()); - const float * window_func = cache.hann_window.data();- const int window_size = params.hann_window_size;- const int frame_size = (params.n_fft_bins - 1) * 2;- const int frame_step = params.hop_length;+ const int frame_size = (mel_config.n_fft_bins - 1) * 2; // Apply preemphasis filter (high-pass): x[i] = x[i] - 0.97 * x[i-1] std::vector<float> samples_preprocessed(samples, samples + n_samples_in);@@ -1891,55 +1757,25 @@ void mtmd_audio_preprocessor_parakeet::log_mel(const float * samples, size_t n_s std::vector<float> samples_padded(n_samples_in + 2 * pad, 0.0f); std::copy(samples_preprocessed.begin(), samples_preprocessed.end(), samples_padded.begin() + pad); - out_full.n_mel = params.n_mel;- out_full.n_len = (samples_padded.size() - frame_size) / frame_step + 1;- out_full.n_len_org = out_full.n_len;- out_full.data.resize(out_full.n_mel * out_full.n_len);-- const int n_threads = 4;- std::vector<std::thread> workers(n_threads - 1);- for (int iw = 0; iw < n_threads - 1; ++iw) {- workers[iw] = std::thread(- worker_thread, iw + 1,- window_func,- window_size,- std::cref(samples_padded),- samples_padded.size(),- frame_size,- frame_step,- n_threads,- params.n_fft_bins,- std::cref(cache),- std::ref(out_full)- );- }-- worker_thread(0,- window_func,- window_size,- samples_padded,- samples_padded.size(),- frame_size,- frame_step,- n_threads,- params.n_fft_bins,- cache,- out_full);-- for (int iw = 0; iw < n_threads - 1; ++iw) {- workers[iw].join();+ if (!log_mel_spectrogram(samples_padded.data(), (int) samples_padded.size(), mel_config, shape_only, out_full)) {+ return false; }+ out_full.n_len_org = out_full.n_len;+ return true; } -bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples,- size_t n_samples_in,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_parakeet::process(const float * samples,+ size_t n_samples_in,+ std::vector<mtmd_audio_mel> & output,+ bool shape_only) const { if (n_samples_in == 0) { return false; } mtmd_audio_mel out_full;- log_mel(samples, n_samples_in, out_full);+ if (!log_mel(samples, n_samples_in, out_full, shape_only)) {+ return false;+ } // Per-feature normalization (only on valid frames) {@@ -1980,15 +1816,18 @@ bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples, // mtmd_audio_preprocessor_nemotron_transcribe implementation // -bool mtmd_audio_preprocessor_nemotron_transcribe::preprocess(const float * samples,- size_t n_samples,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_nemotron_transcribe::process(const float * samples,+ size_t n_samples,+ std::vector<mtmd_audio_mel> & output,+ bool shape_only) const { if (n_samples < (size_t) hparams.audio_n_fft) { return false; } mtmd_audio_mel mel;- log_mel(samples, n_samples, mel);+ if (!log_mel(samples, n_samples, mel, shape_only)) {+ return false;+ } // frames past n_samples / hop are padding (NeMo get_seq_len); kept raw, encode_windows normalizes them for ASR const int64_t n_frames = n_samples / hparams.audio_hop_len;@@ -2010,9 +1849,10 @@ void mtmd_audio_preprocessor_gemma4ua::initialize() { // no-op: no FFT or filterbank needed } -bool mtmd_audio_preprocessor_gemma4ua::preprocess(const float * samples,- size_t n_samples,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_gemma4ua::process(const float * samples,+ size_t n_samples,+ std::vector<mtmd_audio_mel> & output,+ bool /*shape_only*/) const { if (n_samples == 0) { return false; }@@ -2151,9 +1991,10 @@ std::vector<float> mtmd_audio_streaming_istft::flush() { // the samples are handed over as a single-row "mel", to reuse the normal chunk path // -bool mtmd_audio_preprocessor_pockettts::preprocess(const float * samples,- size_t n_samples,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_pockettts::process(const float * samples,+ size_t n_samples,+ std::vector<mtmd_audio_mel> & output,+ bool /*shape_only*/) const { // the encoder needs whole frames, see pad_for_conv1d() in the reference const int64_t frame_size = (int64_t) hparams.mimi_downsample * 120; if (n_samples == 0 || frame_size <= 0) {@@ -2183,10 +2024,11 @@ bool mtmd_audio_preprocessor_pockettts::preprocess(const float * } // Muse Spark reads the raw waveform: pass the samples through, zero padded to a whole number of embeddings-bool mtmd_audio_preprocessor_muse_spark_audio::preprocess(- const float * samples,- size_t n_samples,- std::vector<mtmd_audio_mel> & output) const {+bool mtmd_audio_preprocessor_muse_spark_audio::process(+ const float * samples,+ size_t n_samples,+ std::vector<mtmd_audio_mel> & output,+ bool /*shape_only*/) const { const int64_t fold = hparams.audio_samples_per_token(); const int64_t n_padded = ((int64_t) n_samples + fold - 1) / fold * fold; tools/mtmd/mtmd-audio.h71 + / 27 −
@@ -1,7 +1,9 @@ #pragma once #include "ggml.h"+#include "ggml-backend.h" #include "clip-model.h"+#include "clip-impl.h" #include <cstdint> #include <vector>@@ -53,21 +55,69 @@ struct mtmd_audio_cache { // compute one log10-mel frame from n_fft windowed samples, cache must be initialized for n_fft void mtmd_audio_log_mel_frame(const mtmd_audio_cache & cache, const float * samples, int n_fft, int n_mel, float * out); +struct mtmd_audio_mel_config {+ int64_t n_mel = 0;+ int64_t n_fft_bins = 0;+ int32_t hann_window_size = 0;+ int32_t hop_length = 0;+ int32_t sample_rate = 0;+ bool no_padding = false;+ bool center_padding = false;+ float preemph = 0.f;+ bool use_natural_log = false;+ bool norm_per_feature = false;+ bool use_magnitude = false; // |X| instead of |X|^2+ float mel_floor = 5.960464477539063e-08f;+ bool mel_floor_add = false; // log(x + floor) instead of log(max(x, floor))+ bool std_eps_after_sqrt = false; // std + eps instead of sqrt(var + eps)+ int n_threads = 1;++ const mtmd_audio_cache * cache = nullptr;+ ggml_backend_sched_t sched = nullptr;+};++struct mtmd_audio_preproc_out {+ std::vector<clip_image_f32> entries; // one per chunk, empty on failure+ std::vector<float> samples; // input of compute(), empty if entries already hold the data++ bool needs_compute() const {+ return !samples.empty() && (entries.empty() || entries[0].is_placeholder());+ }+};+ struct mtmd_audio_preprocessor { const clip_hparams & hparams;+ mtmd_audio_mel_config mel_config; // filled by initialize() of the preprocessors that use the log-mel graph++ mtmd_audio_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {+ mel_config.sched = clip_get_sched(ctx);+ }+ mtmd_audio_preprocessor(const clip_hparams & hparams, ggml_backend_sched_t sched): hparams(hparams) {+ mel_config.sched = sched;+ } - mtmd_audio_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {}- mtmd_audio_preprocessor(const clip_hparams & hparams): hparams(hparams) {}+ // mel_config points to the cache of this instance+ mtmd_audio_preprocessor(const mtmd_audio_preprocessor &) = delete;+ mtmd_audio_preprocessor & operator=(const mtmd_audio_preprocessor &) = delete; virtual ~mtmd_audio_preprocessor() = default; virtual void initialize() = 0; // NOT thread-safe- virtual bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const = 0;++ // entries have the final shapes, but no data if needs_compute()+ mtmd_audio_preproc_out preprocess(const float * samples, size_t n_samples) const;++ // run it on the thread that encodes, it uses mel_config.sched+ bool compute(mtmd_audio_preproc_out & out) const;++ protected:+ // shape_only skips the log-mel graph: the shapes are right, the data is zero-filled+ virtual bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const = 0; }; struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor { mtmd_audio_preprocessor_whisper(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override;- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; private: mtmd_audio_cache cache;@@ -76,14 +126,14 @@ struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor { mtmd_audio_preprocessor_muse_spark_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override {}- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; }; struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor { mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}- mtmd_audio_preprocessor_conformer(const clip_hparams & hparams) : mtmd_audio_preprocessor(hparams) {}+ mtmd_audio_preprocessor_conformer(const clip_hparams & hparams, ggml_backend_sched_t sched) : mtmd_audio_preprocessor(hparams, sched) {} void initialize() override;- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; private: mtmd_audio_cache cache;@@ -92,13 +142,13 @@ struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor { // same as conformer, the audio is cut to 30 s and padded to 0.5 s (d1-omni audio.py) struct mtmd_audio_preprocessor_d1omni : mtmd_audio_preprocessor_conformer { using mtmd_audio_preprocessor_conformer::mtmd_audio_preprocessor_conformer;- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; }; struct mtmd_audio_preprocessor_lfm3a : mtmd_audio_preprocessor { mtmd_audio_preprocessor_lfm3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override;- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; private: mtmd_audio_cache cache;@@ -107,7 +157,7 @@ struct mtmd_audio_preprocessor_lfm3a : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor { mtmd_audio_preprocessor_granite_speech(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override;- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; private: mtmd_audio_cache cache;@@ -116,7 +166,7 @@ struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor { mtmd_audio_preprocessor_gemma4a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override;- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; private: mtmd_audio_cache cache;@@ -125,13 +175,13 @@ struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_gemma4ua : mtmd_audio_preprocessor { mtmd_audio_preprocessor_gemma4ua(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override;- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; }; struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor { mtmd_audio_preprocessor_qwen3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override;- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; private: mtmd_audio_cache cache;@@ -140,7 +190,7 @@ struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor { mtmd_audio_preprocessor_dots3note(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override;- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; private: mtmd_audio_cache cache;@@ -149,7 +199,7 @@ struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor { mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override;- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; private: mtmd_audio_cache cache;@@ -158,7 +208,7 @@ struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor { mtmd_audio_preprocessor_qwen3tts_spk(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override;- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; private: mtmd_audio_cache cache;@@ -168,7 +218,7 @@ struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor { mtmd_audio_preprocessor_pockettts(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} void initialize() override {}- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; }; // chatterbox reference front-end: one clip gives five entries, used by position in mtmd.cpp@@ -181,7 +231,7 @@ struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_chatterbox_ref : mtmd_audio_preprocessor { mtmd_audio_preprocessor_chatterbox_ref(const clip_ctx * ctx, bool is_mtl) : mtmd_audio_preprocessor(ctx), is_mtl(is_mtl) {} void initialize() override;- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; private: bool is_mtl;@@ -194,25 +244,19 @@ struct mtmd_audio_preprocessor_chatterbox_ref : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor { mtmd_audio_preprocessor_parakeet(clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) { } void initialize() override;- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; protected:- void log_mel(const float * samples, size_t n_samples, mtmd_audio_mel & out) const;+ bool log_mel(const float * samples, size_t n_samples, mtmd_audio_mel & out, bool shape_only) const; private: mtmd_audio_cache cache;-- static void worker_thread(int ith, const float * window_func, int window_size,- const std::vector<float> & samples, int n_samples,- int frame_size, int frame_step, int n_threads,- int n_fft_bins,- const mtmd_audio_cache & cache, mtmd_audio_mel & mel); }; // one chunk per recording: the raw log-mel of its valid frames struct mtmd_audio_preprocessor_nemotron_transcribe : mtmd_audio_preprocessor_parakeet { mtmd_audio_preprocessor_nemotron_transcribe(clip_ctx * ctx) : mtmd_audio_preprocessor_parakeet(ctx) { }- bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;+ bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override; }; //tools/mtmd/mtmd.cpp68 + / 51 −
@@ -348,10 +348,14 @@ struct mtmd_image_tokens { }; using mtmd_image_tokens_ptr = std::unique_ptr<mtmd_image_tokens>; +// the mel of a clip is computed by its first encode, its chunks share the result+using mtmd_audio_source_ptr = std::shared_ptr<mtmd_audio_preproc_out>;+ // an audio track split across chunks (e.g. the soundtrack of a video): it is encoded once, as one clip, and every // chunk takes its rows of it struct mtmd_audio_track { clip_image_f32_batch batch_f32; // the mel of the whole track+ mtmd_audio_source_ptr source; uint32_t n_tokens = 0; std::vector<float> embd; // filled by the first encode std::shared_ptr<const clip_gen_ref> gen_ref; // gen-side reference of a predefined voice, if any@@ -362,12 +366,18 @@ struct mtmd_audio_tokens { clip_image_f32_batch batch_f32; // preprocessed image patches std::string id; // optional user-defined ID, useful for KV cache tracking + mtmd_audio_source_ptr source; // if set, batch_f32 has the shape of the mel of chunk source_idx+ uint32_t source_idx = 0;+ // a slice of a track: the rows [row_begin, row_begin + n_tokens) of it, batch_f32 is unused std::shared_ptr<mtmd_audio_track> track; uint32_t row_begin = 0; // true if one of entries in batch_f32 is a placeholder bool is_placeholder() const {+ if (source || (track && track->source)) {+ return false;+ } if (track && track->batch_f32.entries.empty()) { return track->embd.empty(); // a voice: its rows are not serialized }@@ -384,6 +394,8 @@ struct mtmd_audio_tokens { n_tokens, batch_f32.clone(), id,+ source,+ source_idx, track, row_begin, };@@ -413,6 +425,7 @@ struct mtmd_audio_tokens { track->n_tokens = ser.read<uint32_t>(); track->batch_f32.deserialize(ser); }+ source.reset(); } }; using mtmd_audio_tokens_ptr = std::unique_ptr<mtmd_audio_tokens>;@@ -682,6 +695,7 @@ struct mtmd_context { /* no_alloc */ no_alloc, /* progress_callback */ ctx_params.progress_callback, /* progress_callback_user_data */ ctx_params.progress_callback_user_data,+ /* n_threads */ ctx_params.n_threads, }; auto res = clip_init(mmproj_fname, ctx_clip_params);@@ -1215,6 +1229,7 @@ struct mtmd_context { // initialize audio preprocessor if (audio_preproc) {+ audio_preproc->mel_config.n_threads = n_threads; audio_preproc->initialize(); } }@@ -1839,7 +1854,7 @@ struct mtmd_tokenizer { GGML_ASSERT(ctx->audio_preproc != nullptr); // preprocess audio- std::vector<mtmd_audio_mel> mel_spec_chunks;+ auto source = std::make_shared<mtmd_audio_preproc_out>(); { std::vector<float> dummy; const float * samples = nullptr;@@ -1863,31 +1878,23 @@ struct mtmd_tokenizer { LOG_ERR("%s: this model takes audio input only via mtmd_audio_stream\n", __func__); return 2; }- bool ok = ctx->audio_preproc->preprocess(samples, n_samples, mel_spec_chunks);- if (!ok) {+ *source = ctx->audio_preproc->preprocess(samples, n_samples);+ if (source->entries.empty()) { LOG_ERR("Unable to preprocess audio\n"); return 2; } } // one chunk for the five entries of a chatterbox clip, one token per talker conditioning row if (clip_get_projector_type(ctx->ctx_a) == PROJECTOR_TYPE_CHATTERBOX_SPKENC) {- GGML_ASSERT(mel_spec_chunks.size() == 5);+ GGML_ASSERT(source->entries.size() == 5); // mtl: spkr, perceiver query rows, emotion; turbo: spkr, t3 speech tokens const size_t n_query = clip_cbx_read_tensor(ctx->ctx_a, "a.cenc.perceiver.pre_attention_query", nullptr, 0) / ctx->n_embd_out();- const size_t n_tokens = cbx_is_mtl(ctx->ctx_a) ? 2 + n_query : 1 + (size_t) mel_spec_chunks[2].n_len / 4;+ const size_t n_tokens = cbx_is_mtl(ctx->ctx_a) ? 2 + n_query : 1 + (size_t) source->entries[2].nx() / 4; clip_image_f32_batch batch_f32; batch_f32.is_audio = true;- for (auto & mel_spec : mel_spec_chunks) {- GGML_ASSERT(mel_spec.n_len <= INT32_MAX && mel_spec.n_len >= 0);- GGML_ASSERT(mel_spec.n_mel <= INT32_MAX && mel_spec.n_mel >= 0);- clip_image_f32 mel_f32;- mel_f32.set_size({(int) mel_spec.n_len, (int) mel_spec.n_mel},- mel_spec.data.empty(), /* is_audio */ true);- mel_f32.cpy_buf(mel_spec.data);- batch_f32.entries.push_back(std::move(mel_f32));- }+ batch_f32.entries = std::move(source->entries); mtmd_audio_tokens_ptr audio_tokens(new mtmd_audio_tokens); audio_tokens->n_tokens = (uint32_t) n_tokens;@@ -1904,17 +1911,8 @@ struct mtmd_tokenizer { } else // consider each mel_spec as a separate audio chunk // TODO: maybe support batching, but this may come with memory cost- for (auto & mel_spec : mel_spec_chunks) {- const bool is_placeholder = mel_spec.data.empty();-- // Validate dimensions fit in clip_image_size (int)- GGML_ASSERT(mel_spec.n_len <= INT32_MAX && mel_spec.n_len >= 0);- GGML_ASSERT(mel_spec.n_mel <= INT32_MAX && mel_spec.n_mel >= 0);- clip_image_f32 mel_f32;- mel_f32.set_size(- {(int)mel_spec.n_len, (int)mel_spec.n_mel},- is_placeholder, /* is_audio */ true);- mel_f32.cpy_buf(mel_spec.data);+ for (size_t i = 0; i < source->entries.size(); i++) {+ clip_image_f32 mel_f32 = source->entries[i]; size_t n_tokens = clip_n_output_tokens(ctx->ctx_a, &mel_f32); @@ -1926,6 +1924,10 @@ struct mtmd_tokenizer { audio_tokens->n_tokens = n_tokens; audio_tokens->batch_f32 = std::move(batch_f32); audio_tokens->id = bitmap->id; // optional+ if (source->needs_compute()) {+ audio_tokens->source = source;+ audio_tokens->source_idx = (uint32_t) i;+ } LOG_DBG("audio_tokens->n_tokens = %d\n", audio_tokens->n_tokens); @@ -1983,16 +1985,21 @@ struct mtmd_tokenizer { samples.insert(samples.end(), data, data + piece->nx); } }- std::vector<mtmd_audio_mel> mels;- if (!ctx->audio_preproc || !ctx->audio_preproc->preprocess(samples.data(), samples.size(), mels) || mels.size() != 1) {+ if (!ctx->audio_preproc) {+ LOG_ERR("%s: unable to preprocess the soundtrack\n", __func__);+ return 2;+ }+ auto source = std::make_shared<mtmd_audio_preproc_out>(ctx->audio_preproc->preprocess(samples.data(), samples.size()));+ if (source->entries.size() != 1) { LOG_ERR("%s: unable to preprocess the soundtrack\n", __func__); return 2; }- clip_image_f32 mel_f32;- mel_f32.set_size({(int) mels[0].n_len, (int) mels[0].n_mel}, mels[0].data.empty(), /* is_audio */ true);- mel_f32.cpy_buf(mels[0].data);+ clip_image_f32 mel_f32 = source->entries[0]; st.track = std::make_shared<mtmd_audio_track>(); st.track->n_tokens = clip_n_output_tokens(ctx->ctx_a, &mel_f32);+ if (source->needs_compute()) {+ st.track->source = source;+ } st.track->batch_f32.is_audio = true; st.track->batch_f32.entries.push_back(std::move(mel_f32)); }@@ -2249,6 +2256,19 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im static int32_t cbx_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd); +static bool mtmd_audio_batch_encode(mtmd_context * ctx, const clip_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector<float> & out_embd) {+ clip_image_f32_batch batch = batch_f32.clone();+ if (source) {+ if (!ctx->audio_preproc->compute(*source)) {+ LOG_ERR("%s: unable to compute the mel\n", __func__);+ return false;+ }+ GGML_ASSERT(idx < source->entries.size());+ batch.entries[0] = source->entries[idx];+ }+ return clip_image_batch_encode(ctx->ctx_a, ctx->n_threads, &batch, out_embd);+}+ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk * chunk, std::vector<float> & out_embd) { ctx->gen_ref = {}; // see mtmd_gen_state_set_ref() if (chunk->type == MTMD_INPUT_CHUNK_TYPE_TEXT) {@@ -2286,7 +2306,7 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk if (track) { if (track->embd.empty()) { track->embd.resize((size_t) track->n_tokens * ctx->n_embd_out());- if (!clip_image_batch_encode(ctx->ctx_a, ctx->n_threads, &track->batch_f32, track->embd)) {+ if (!mtmd_audio_batch_encode(ctx, track->batch_f32, track->source.get(), 0, track->embd)) { track->embd.clear(); return 1; }@@ -2304,11 +2324,7 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk return cbx_ref_encode(ctx, chunk->tokens_audio.get(), out_embd); } out_embd.resize((size_t)chunk->tokens_audio->n_tokens * ctx->n_embd_out());- bool ok = clip_image_batch_encode(- ctx->ctx_a,- ctx->n_threads,- &chunk->tokens_audio->batch_f32,- out_embd);+ bool ok = mtmd_audio_batch_encode(ctx, chunk->tokens_audio->batch_f32, chunk->tokens_audio->source.get(), chunk->tokens_audio->source_idx, out_embd); return ok ? 0 : 1; } @@ -3884,21 +3900,21 @@ void mtmd_debug_preprocess_audio(mtmd_context * ctx, const std::vector<float> & LOG_ERR("%s: model does not support audio input\n", __func__); return; }- std::vector<mtmd_audio_mel> mel_spec_chunks;- bool ok = ctx->audio_preproc->preprocess(samples.data(), samples.size(), mel_spec_chunks);- if (!ok) {+ mtmd_audio_preproc_out out = ctx->audio_preproc->preprocess(samples.data(), samples.size());+ if (out.entries.empty() || !ctx->audio_preproc->compute(out)) { LOG_ERR("%s: failed to preprocess audio\n", __func__); return; }- LOG_INF("%s: preprocessed audio to %zu mel spec chunks\n", __func__, mel_spec_chunks.size());- for (size_t i = 0; i < mel_spec_chunks.size(); i++) {- LOG_INF("%s: mel spec chunk %zu has n_len=%d, n_mel=%d\n", __func__, i, mel_spec_chunks[i].n_len, mel_spec_chunks[i].n_mel);+ LOG_INF("%s: preprocessed audio to %zu mel spec chunks\n", __func__, out.entries.size());+ for (size_t i = 0; i < out.entries.size(); i++) {+ const auto & mel = out.entries[i];+ const auto & data = mel.get_ro_buf();+ LOG_INF("%s: mel spec chunk %zu has n_len=%d, n_mel=%d\n", __func__, i, mel.nx(), mel.ny()); // dump mel entries: data is stored as [n_mel][n_len] (mel-major)- const auto & mel = mel_spec_chunks[i];- for (int m = 0; m < mel.n_mel; m++) {- for (int t = 0; t < mel.n_len; t++) {- LOG_INF("mel[%zu][m=%d][t=%d] = %f\n", i, m, t, mel.data[m * mel.n_len + t]);+ for (int m = 0; m < mel.ny(); m++) {+ for (int t = 0; t < mel.nx(); t++) {+ LOG_INF("mel[%zu][m=%d][t=%d] = %f\n", i, m, t, data[(size_t) m * mel.nx() + t]); } } }@@ -4449,16 +4465,17 @@ static bool mtmd_audio_stream_compute_mel(mtmd_audio_stream_lfm3a * s) { } const int64_t start = s->n_frames * s->hop - s->buf_pos; const size_t len = (size_t) (s->win + (n_new - 1) * s->hop);- std::vector<mtmd_audio_mel> mel_chunks;- if (!s->ctx->audio_preproc->preprocess(s->buf.data() + start, len, mel_chunks) || mel_chunks.size() != 1) {+ mtmd_audio_preproc_out out = s->ctx->audio_preproc->preprocess(s->buf.data() + start, len);+ if (out.entries.size() != 1 || !s->ctx->audio_preproc->compute(out)) { LOG_ERR("%s: unable to preprocess audio\n", __func__); return false; }- const auto & mel = mel_chunks[0];- GGML_ASSERT(mel.n_len == n_new && mel.n_mel == s->n_mel);+ const auto & mel = out.entries[0];+ const auto & data = mel.get_ro_buf();+ GGML_ASSERT(mel.nx() == n_new && mel.ny() == s->n_mel); for (int64_t t = 0; t < n_new; t++) { for (int m = 0; m < s->n_mel; m++) {- s->mel.push_back(mel.data[(size_t) m * mel.n_len + t]);+ s->mel.push_back(data[(size_t) m * mel.nx() + t]); } } s->n_frames = n_total;