From 573f2cf58e4bde1a8b3238b0612cc5a7b26613a0 Mon Sep 17 00:00:00 2001 From: andrewmd5 <1297077+andrewmd5@users.noreply.github.com> Date: Fri, 6 Mar 2026 21:37:07 +0900 Subject: [PATCH 1/6] feat: add video support for Qwen3.5 --- tools/mtmd/clip.cpp | 40 ++++++---- tools/mtmd/models/qwen3vl.cpp | 28 +++++-- tools/mtmd/mtmd-helper.cpp | 29 ++++++- tools/mtmd/mtmd.cpp | 145 ++++++++++++++++++++++++++++++++-- tools/mtmd/mtmd.h | 29 +++++-- 5 files changed, 235 insertions(+), 36 deletions(-) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index b70bad33b686..02d8bcb7c20f 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -3596,9 +3596,17 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima // set input pixel values if (!imgs.is_audio) { + // detect number of channels from the buffer size + const int nx = imgs.entries[0]->nx; + const int ny = imgs.entries[0]->ny; + const int n = nx * ny; + const size_t buf_size = imgs.entries[0]->buf.size(); + const int n_channels = (int)(buf_size / n); + GGML_ASSERT(n_channels == 3 || n_channels == 6); + size_t nelem = 0; for (const auto & img : imgs.entries) { - nelem += img->nx * img->ny * 3; + nelem += img->nx * img->ny * n_channels; } std::vector inp_raw(nelem); @@ -3612,21 +3620,21 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima // │ H │ channel = B // └─────┘ │ // ──────┘ x B - - for (size_t i = 0; i < imgs.entries.size(); i++) { - const int nx = imgs.entries[i]->nx; - const int ny = imgs.entries[i]->ny; - const int n = nx * ny; - - for (int b = 0; b < batch_size; b++) { - float * batch_entry = inp_raw.data() + b * (3*n); - for (int y = 0; y < ny; y++) { - for (int x = 0; x < nx; x++) { - size_t base_src = 3*(y * nx + x); // idx of the first channel - size_t base_dst = y * nx + x; // idx of the first channel - batch_entry[ base_dst] = imgs.entries[b]->buf[base_src ]; - batch_entry[1*n + base_dst] = imgs.entries[b]->buf[base_src + 1]; - batch_entry[2*n + base_dst] = imgs.entries[b]->buf[base_src + 2]; + // + // for 6-channel video input, same layout but with 6 planar channels + + for (int b = 0; b < batch_size; b++) { + const int cur_nx = imgs.entries[b]->nx; + const int cur_ny = imgs.entries[b]->ny; + const int cur_n = cur_nx * cur_ny; + + float * batch_entry = inp_raw.data() + b * (n_channels * cur_n); + for (int y = 0; y < cur_ny; y++) { + for (int x = 0; x < cur_nx; x++) { + size_t base_src = n_channels * (y * cur_nx + x); + size_t base_dst = y * cur_nx + x; + for (int c = 0; c < n_channels; c++) { + batch_entry[c * cur_n + base_dst] = imgs.entries[b]->buf[base_src + c]; } } } diff --git a/tools/mtmd/models/qwen3vl.cpp b/tools/mtmd/models/qwen3vl.cpp index 5ecb10fe4382..0669e5af1428 100644 --- a/tools/mtmd/models/qwen3vl.cpp +++ b/tools/mtmd/models/qwen3vl.cpp @@ -13,16 +13,34 @@ ggml_cgraph * clip_graph_qwen3vl::build() { int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4}; - ggml_tensor * inp_raw = build_inp_raw(); - ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1); + // detect video: 6-channel input means interleaved frame pairs (even_rgb + odd_rgb) + // for images (3ch), both Conv2Ds receive the same input (original behavior) + // for video (6ch), Conv2D_0 gets even frames (ch 0-2), Conv2D_1 gets odd frames (ch 3-5) + const bool is_video = (img.buf.size() == (size_t)img.nx * img.ny * 6); + const int n_channels = is_video ? 6 : 3; + + ggml_tensor * inp_raw = build_inp_raw(n_channels); + + ggml_tensor * inp; + if (is_video) { + const size_t nb1 = ggml_row_size(inp_raw->type, img.nx); + const size_t nb2 = nb1 * img.ny; + ggml_tensor * inp_even = ggml_view_3d(ctx0, inp_raw, img.nx, img.ny, 3, nb1, nb2, 0); + ggml_tensor * inp_odd = ggml_view_3d(ctx0, inp_raw, img.nx, img.ny, 3, nb1, nb2, nb2 * 3); + inp = ggml_add(ctx0, + ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_even, patch_size, patch_size, 0, 0, 1, 1), + ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_odd, patch_size, patch_size, 0, 0, 1, 1)); + } else { + inp = ggml_add(ctx0, + ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1), + ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1)); + } GGML_ASSERT(img.nx % (patch_size * 2) == 0); GGML_ASSERT(img.ny % (patch_size * 2) == 0); - // second conv dimension + // spatial merge { - auto inp_1 = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1); - inp = ggml_add(ctx0, inp, inp_1); inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); // [w, h, c, b] -> [c, w, h, b] inp = ggml_cont_4d( diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index c75f90730f1f..44e616aa5528 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -174,6 +174,28 @@ struct decode_embd_batch { } } + // M-RoPE for video: 3D positions [temporal, height, width] + void set_position_mrope_3d(llama_pos pos_0, int nx, int ny, int nt, llama_seq_id seq_id) { + GGML_ASSERT(n_pos_per_embd == 4); + seq_id_0[0] = seq_id; + for (int t = 0; t < nt; t++) { + for (int y = 0; y < ny; y++) { + for (int x = 0; x < nx; x++) { + int i = t * ny * nx + y * nx + x; + pos[i ] = pos_0 + t; + pos[i + batch.n_tokens ] = pos_0 + y; + pos[i + batch.n_tokens * 2] = pos_0 + x; + pos[i + batch.n_tokens * 3] = 0; + } + } + } + for (int i = 0; i < batch.n_tokens; i++) { + batch.n_seq_id[i] = 1; + batch.seq_id [i] = seq_id_0.data(); + batch.logits [i] = false; + } + } + // M-RoPE for audio void set_position_mrope_1d(llama_pos pos_0, llama_seq_id seq_id) { GGML_ASSERT(n_pos_per_embd == 4); @@ -260,7 +282,12 @@ int32_t mtmd_helper_decode_image_chunk( } const int nx = mtmd_image_tokens_get_nx(image_tokens); const int ny = mtmd_image_tokens_get_ny(image_tokens); - batch_embd.set_position_mrope_2d(n_past, nx, ny, seq_id); + const int nt = mtmd_image_tokens_get_nt(image_tokens); + if (nt > 1) { + batch_embd.set_position_mrope_3d(n_past, nx, ny, nt, seq_id); + } else { + batch_embd.set_position_mrope_2d(n_past, nx, ny, seq_id); + } } else if (chunk_type == MTMD_INPUT_CHUNK_TYPE_AUDIO) { batch_embd.set_position_mrope_1d(n_past, seq_id); } else { diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 8ca979c86cf7..c380b73ea2a1 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -23,9 +23,11 @@ // represents raw image data, layout is RGBRGBRGB... // length of data must be nx * ny * 3 +// for video: data is n_frames sequential RGB frames, each nx * ny * 3 bytes struct mtmd_bitmap { uint32_t nx; uint32_t ny; + uint32_t n_frames = 0; // 0 for single images, >= 2 (even) for video std::vector data; std::string id; // optional user-defined id, for ex: can be set to image hash, useful for KV cache tracking bool is_audio = false; // true if the bitmap is audio @@ -34,8 +36,9 @@ struct mtmd_bitmap { struct mtmd_image_tokens { uint32_t nx; // number of tokens in x direction uint32_t ny; // number of tokens in y direction + uint32_t nt = 1; // number of temporal positions (1 for images, > 1 for video) bool use_mrope_pos = false; // use M-RoPE position counting (the whole image is 1 temporal position) - uint32_t n_tokens() const { return nx * ny; } + uint32_t n_tokens() const { return nt * nx * ny; } clip_image_f32_batch batch_f32; // preprocessed image patches std::string id; // optional user-defined ID, useful for KV cache tracking @@ -43,6 +46,7 @@ struct mtmd_image_tokens { return mtmd_image_tokens{ nx, ny, + nt, use_mrope_pos, batch_f32.clone(), id @@ -549,6 +553,10 @@ struct mtmd_tokenizer { } int32_t add_media(const mtmd_bitmap * bitmap) { + if (bitmap->n_frames >= 2) { + return add_video(bitmap); + } + if (!bitmap->is_audio) { // handle image @@ -739,6 +747,102 @@ struct mtmd_tokenizer { return 0; } + // preprocess video frames and create an image chunk with temporal dimension + // frames are paired (even+odd), each pair becomes one 6-channel image + // each pair is encoded independently through the ViT (per-frame attention) + int32_t add_video(const mtmd_bitmap * bitmap) { + if (!ctx->ctx_v) { + LOG_ERR("%s: error: model does not support vision input\n", __func__); + return 2; + } + + const uint32_t n_frames = bitmap->n_frames; + const uint32_t n_pairs = n_frames / 2; + const size_t frame_bytes = (size_t)bitmap->nx * bitmap->ny * 3; + + if (!ctx->img_beg.empty()) { + add_text(ctx->img_beg, true); + } + + // preprocess each frame individually + clip_image_f32_batch all_frames; + for (uint32_t f = 0; f < n_frames; f++) { + clip_image_u8_ptr img_u8(clip_image_u8_init()); + img_u8->nx = bitmap->nx; + img_u8->ny = bitmap->ny; + img_u8->buf.resize(frame_bytes); + std::memcpy(img_u8->buf.data(), bitmap->data.data() + f * frame_bytes, frame_bytes); + + clip_image_f32_batch frame_batch; + bool ok = clip_image_preprocess(ctx->ctx_v, img_u8.get(), &frame_batch); + if (!ok) { + LOG_ERR("Unable to preprocess video frame %u\n", f); + return 2; + } + GGML_ASSERT(frame_batch.entries.size() == 1); + all_frames.entries.push_back(std::move(frame_batch.entries[0])); + } + + const int frame_nx = all_frames.entries[0]->nx; + const int frame_ny = all_frames.entries[0]->ny; + const int n_pixels = frame_nx * frame_ny; + + // interleave frame pairs into 6-channel images (even_rgb + odd_rgb) + // each pair is a separate batch entry, encoded independently + clip_image_f32_batch pair_batch; + for (uint32_t p = 0; p < n_pairs; p++) { + const auto & even = all_frames.entries[p * 2]; + const auto & odd = all_frames.entries[p * 2 + 1]; + GGML_ASSERT(even->nx == frame_nx && even->ny == frame_ny); + GGML_ASSERT(odd->nx == frame_nx && odd->ny == frame_ny); + + clip_image_f32_ptr pair(clip_image_f32_init()); + pair->nx = frame_nx; + pair->ny = frame_ny; + pair->buf.resize((size_t)n_pixels * 6); + + for (int i = 0; i < n_pixels; i++) { + const int dst = i * 6; + const int src = i * 3; + pair->buf[dst + 0] = even->buf[src + 0]; + pair->buf[dst + 1] = even->buf[src + 1]; + pair->buf[dst + 2] = even->buf[src + 2]; + pair->buf[dst + 3] = odd->buf[src + 0]; + pair->buf[dst + 4] = odd->buf[src + 1]; + pair->buf[dst + 5] = odd->buf[src + 2]; + } + pair_batch.entries.push_back(std::move(pair)); + } + + const uint32_t tokens_x = clip_n_output_tokens_x(ctx->ctx_v, pair_batch.entries[0].get()); + const uint32_t tokens_y = clip_n_output_tokens_y(ctx->ctx_v, pair_batch.entries[0].get()); + + mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens); + image_tokens->nx = tokens_x; + image_tokens->ny = tokens_y; + image_tokens->nt = n_pairs; + image_tokens->use_mrope_pos = true; + image_tokens->batch_f32 = std::move(pair_batch); + image_tokens->id = bitmap->id; + + LOG_DBG("video: nt=%u, nx=%u, ny=%u, n_tokens=%u\n", + image_tokens->nt, image_tokens->nx, image_tokens->ny, image_tokens->n_tokens()); + + mtmd_input_chunk chunk{ + MTMD_INPUT_CHUNK_TYPE_IMAGE, + {}, // text tokens + std::move(image_tokens), + nullptr, // audio tokens + }; + cur.entries.emplace_back(std::move(chunk)); + + if (!ctx->img_end.empty()) { + add_text(ctx->img_end, true); + } + + return 0; + } + std::vector split_batch_to_chunk(clip_image_f32_batch && batch_f32, const std::string & id) { std::vector chunks; @@ -851,10 +955,13 @@ int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens) ctx->image_embd_v.resize(image_tokens->n_tokens() * n_mmproj_embd); bool ok = false; - if (clip_is_llava(ctx_clip) + if (image_tokens->nt > 1 + || clip_is_llava(ctx_clip) || clip_is_minicpmv(ctx_clip) || clip_is_glm(ctx_clip)) { - // TODO @ngxson : llava does not support batched encoding ; this should be fixed inside clip_image_batch_encode() + // encode each batch entry independently + // video: each entry is one frame pair, encoded with per-frame attention + // llava/minicpmv/glm: does not support batched encoding const auto & entries = image_tokens->batch_f32.entries; for (size_t i = 0; i < entries.size(); i++) { int n_tokens_per_image = clip_n_output_tokens(ctx_clip, entries[i].get()); @@ -934,6 +1041,21 @@ mtmd_bitmap * mtmd_bitmap_init(uint32_t nx, return bitmap; } +mtmd_bitmap * mtmd_bitmap_init_from_video(uint32_t nx, + uint32_t ny, + uint32_t n_frames, + const unsigned char * data) { + GGML_ASSERT(n_frames >= 2 && n_frames % 2 == 0); + mtmd_bitmap * bitmap = new mtmd_bitmap; + bitmap->nx = nx; + bitmap->ny = ny; + bitmap->n_frames = n_frames; + size_t data_size = (size_t)nx * ny * 3 * n_frames; + bitmap->data.resize(data_size); + std::memcpy(bitmap->data.data(), data, data_size); + return bitmap; +} + mtmd_bitmap * mtmd_bitmap_init_from_audio(size_t n_samples, const float * data) { mtmd_bitmap * bitmap = new mtmd_bitmap; @@ -966,6 +1088,14 @@ bool mtmd_bitmap_is_audio(const mtmd_bitmap * bitmap) { return bitmap->is_audio; } +bool mtmd_bitmap_is_video(const mtmd_bitmap * bitmap) { + return bitmap->n_frames >= 2; +} + +uint32_t mtmd_bitmap_get_n_frames(const mtmd_bitmap * bitmap) { + return bitmap->n_frames; +} + const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap) { return bitmap->id.c_str(); } @@ -1102,15 +1232,18 @@ size_t mtmd_image_tokens_get_ny(const mtmd_image_tokens * image_tokens) { return image_tokens->ny; } +size_t mtmd_image_tokens_get_nt(const mtmd_image_tokens * image_tokens) { + return image_tokens->nt; +} + const char * mtmd_image_tokens_get_id(const mtmd_image_tokens * image_tokens) { return image_tokens->id.c_str(); } llama_pos mtmd_image_tokens_get_n_pos(const mtmd_image_tokens * image_tokens) { if (image_tokens->use_mrope_pos) { - // for M-RoPE, temporal dimension = max(t,h,w) - // t is omitted as we don't support video input - return std::max(image_tokens->nx, image_tokens->ny); + // for M-RoPE, n_pos = max(t, h, w) + return (llama_pos)std::max({image_tokens->nt, image_tokens->nx, image_tokens->ny}); } return image_tokens->n_tokens(); } diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index ef25d32bbef8..61d271c5609e 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -134,17 +134,24 @@ MTMD_API int mtmd_get_audio_bitrate(mtmd_context * ctx); // if bitmap is image: // length of data must be nx * ny * 3 // the data is in RGBRGBRGB... format +// if bitmap is video: +// length of data must be nx * ny * 3 * n_frames +// n_frames must be >= 2 and even +// frames are sequential RGB, each nx * ny * 3 bytes // if bitmap is audio: // length of data must be n_samples * sizeof(float) // the data is in float format (PCM F32) MTMD_API mtmd_bitmap * mtmd_bitmap_init (uint32_t nx, uint32_t ny, const unsigned char * data); +MTMD_API mtmd_bitmap * mtmd_bitmap_init_from_video(uint32_t nx, uint32_t ny, uint32_t n_frames, const unsigned char * data); MTMD_API mtmd_bitmap * mtmd_bitmap_init_from_audio(size_t n_samples, const float * data); -MTMD_API uint32_t mtmd_bitmap_get_nx (const mtmd_bitmap * bitmap); -MTMD_API uint32_t mtmd_bitmap_get_ny (const mtmd_bitmap * bitmap); -MTMD_API const unsigned char * mtmd_bitmap_get_data (const mtmd_bitmap * bitmap); -MTMD_API size_t mtmd_bitmap_get_n_bytes(const mtmd_bitmap * bitmap); -MTMD_API bool mtmd_bitmap_is_audio (const mtmd_bitmap * bitmap); -MTMD_API void mtmd_bitmap_free (mtmd_bitmap * bitmap); +MTMD_API uint32_t mtmd_bitmap_get_nx (const mtmd_bitmap * bitmap); +MTMD_API uint32_t mtmd_bitmap_get_ny (const mtmd_bitmap * bitmap); +MTMD_API const unsigned char * mtmd_bitmap_get_data (const mtmd_bitmap * bitmap); +MTMD_API size_t mtmd_bitmap_get_n_bytes (const mtmd_bitmap * bitmap); +MTMD_API bool mtmd_bitmap_is_audio (const mtmd_bitmap * bitmap); +MTMD_API bool mtmd_bitmap_is_video (const mtmd_bitmap * bitmap); +MTMD_API uint32_t mtmd_bitmap_get_n_frames(const mtmd_bitmap * bitmap); +MTMD_API void mtmd_bitmap_free (mtmd_bitmap * bitmap); // bitmap ID is optional, but useful for KV cache tracking // these getters/setters are dedicated functions, so you can for example calculate the hash of the image based on mtmd_bitmap_get_data() MTMD_API const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap); @@ -187,6 +194,7 @@ MTMD_API void mtmd_input_chunk_free(mtmd_input_chunk * chunk); MTMD_API size_t mtmd_image_tokens_get_n_tokens(const mtmd_image_tokens * image_tokens); // TODO: deprecate MTMD_API size_t mtmd_image_tokens_get_nx (const mtmd_image_tokens * image_tokens); MTMD_API size_t mtmd_image_tokens_get_ny (const mtmd_image_tokens * image_tokens); +MTMD_API size_t mtmd_image_tokens_get_nt (const mtmd_image_tokens * image_tokens); MTMD_API const char * mtmd_image_tokens_get_id (const mtmd_image_tokens * image_tokens); // TODO: deprecate // number of temporal positions (equals to max(t,h,w) for M-RoPE; equals to n_tokens otherwise) MTMD_API llama_pos mtmd_image_tokens_get_n_pos (const mtmd_image_tokens * image_tokens); // TODO: deprecate @@ -276,9 +284,14 @@ struct bitmap { bitmap(uint32_t nx, uint32_t ny, const unsigned char * data) { ptr.reset(mtmd_bitmap_init(nx, ny, data)); } + bitmap(uint32_t nx, uint32_t ny, uint32_t n_frames, const unsigned char * data) { + ptr.reset(mtmd_bitmap_init_from_video(nx, ny, n_frames, data)); + } ~bitmap() = default; - uint32_t nx() const { return mtmd_bitmap_get_nx(ptr.get()); } - uint32_t ny() const { return mtmd_bitmap_get_ny(ptr.get()); } + uint32_t nx() const { return mtmd_bitmap_get_nx(ptr.get()); } + uint32_t ny() const { return mtmd_bitmap_get_ny(ptr.get()); } + uint32_t n_frames() const { return mtmd_bitmap_get_n_frames(ptr.get()); } + bool is_video() const { return mtmd_bitmap_is_video(ptr.get()); } const unsigned char * data() const { return mtmd_bitmap_get_data(ptr.get()); } size_t n_bytes() const { return mtmd_bitmap_get_n_bytes(ptr.get()); } std::string id() const { return mtmd_bitmap_get_id(ptr.get()); } From c5b682b25c81a52938e25f884e7eb326738329e3 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Mon, 13 Apr 2026 17:39:14 +0200 Subject: [PATCH 2/6] various clean up --- tools/mtmd/clip-graph.h | 3 + tools/mtmd/clip-impl.h | 2 + tools/mtmd/clip.cpp | 61 ++++++++------ tools/mtmd/clip.h | 3 + tools/mtmd/models/models.h | 5 +- tools/mtmd/models/qwen2vl.cpp | 35 ++++++-- tools/mtmd/models/qwen3vl.cpp | 27 +----- tools/mtmd/mtmd-helper.cpp | 29 +------ tools/mtmd/mtmd.cpp | 153 +++++++++++++++------------------- tools/mtmd/mtmd.h | 37 ++++---- 10 files changed, 161 insertions(+), 194 deletions(-) diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/clip-graph.h index d3e7b1ed044b..d280bfdd82f0 100644 --- a/tools/mtmd/clip-graph.h +++ b/tools/mtmd/clip-graph.h @@ -32,6 +32,9 @@ struct clip_graph { float kq_scale; // TODO: maybe move this to hparams const clip_flash_attn_type flash_attn_type; + // TODO [QWEN_VIDEO]: improve this in the future + int nt = 1; // number of temporal dim, to be used by Qwen-VL models + ggml_context_ptr ctx0_ptr; ggml_context * ctx0; ggml_cgraph * gf; diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 17cb703f7fbb..7dc839ed9367 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -448,6 +448,7 @@ struct clip_image_u8_batch { struct clip_image_f32_batch { std::vector entries; bool is_audio = false; + bool is_seq = true; // for llava-uhd style models, we need to know the grid size // note: entries.size() == grid_x * grid_y + 1 (one overview image) @@ -458,6 +459,7 @@ struct clip_image_f32_batch { clip_image_f32_batch new_batch{ /* entries */ {}, /* is_audio */ is_audio, + /* is_seq */ is_seq, /* grid_x */ grid_x, /* grid_y */ grid_y, }; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index ca68c840f203..ac8a94f6a92c 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -515,7 +515,7 @@ ggml_tensor * clip_graph::build_inp() { } ggml_tensor * clip_graph::build_inp_raw(int channels) { - ggml_tensor * inp_raw = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, img.nx, img.ny, channels); + ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, img.nx, img.ny, channels, nt); ggml_set_name(inp_raw, "inp_raw"); ggml_set_input(inp_raw); return inp_raw; @@ -951,6 +951,9 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 GGML_ABORT("missing cgraph builder"); } + // TODO [QWEN_VIDEO]: improve this in the future + builder->nt = imgs.entries.size(); + return builder->build(); } @@ -3042,10 +3045,11 @@ bool clip_image_encode(struct clip_ctx * ctx, const int n_threads, clip_image_f3 bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_image_f32_batch * imgs_c_ptr, float * vec) { const clip_image_f32_batch & imgs = *imgs_c_ptr; int batch_size = imgs.entries.size(); + bool support_seq = clip_model_supports_seq_input(ctx); // TODO @ngxson : implement batch size > 1 as a loop // we don't need true batching support because the cgraph will gonna be big anyway - if (batch_size != 1) { + if (batch_size != 1 && !support_seq) { return false; // only support batch size of 1 } @@ -3100,17 +3104,9 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima // set input pixel values if (!imgs.is_audio) { - // detect number of channels from the buffer size - const int nx = imgs.entries[0]->nx; - const int ny = imgs.entries[0]->ny; - const int n = nx * ny; - const size_t buf_size = imgs.entries[0]->buf.size(); - const int n_channels = (int)(buf_size / n); - GGML_ASSERT(n_channels == 3 || n_channels == 6); - size_t nelem = 0; for (const auto & img : imgs.entries) { - nelem += img->nx * img->ny * n_channels; + nelem += img->nx * img->ny * 3; } std::vector inp_raw(nelem); @@ -3124,21 +3120,23 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima // │ H │ channel = B // └─────┘ │ // ──────┘ x B - // - // for 6-channel video input, same layout but with 6 planar channels - - for (int b = 0; b < batch_size; b++) { - const int cur_nx = imgs.entries[b]->nx; - const int cur_ny = imgs.entries[b]->ny; - const int cur_n = cur_nx * cur_ny; - - float * batch_entry = inp_raw.data() + b * (n_channels * cur_n); - for (int y = 0; y < cur_ny; y++) { - for (int x = 0; x < cur_nx; x++) { - size_t base_src = n_channels * (y * cur_nx + x); - size_t base_dst = y * cur_nx + x; - for (int c = 0; c < n_channels; c++) { - batch_entry[c * cur_n + base_dst] = imgs.entries[b]->buf[base_src + c]; + + // IMPORTANT: [QWEN_VIDEO] the batch dim is currently used for temporal dim in Qwen-VL models + + for (size_t i = 0; i < imgs.entries.size(); i++) { + const int nx = imgs.entries[i]->nx; + const int ny = imgs.entries[i]->ny; + const int n = nx * ny; + + for (int b = 0; b < batch_size; b++) { + float * batch_entry = inp_raw.data() + b * (3*n); + for (int y = 0; y < ny; y++) { + for (int x = 0; x < nx; x++) { + size_t base_src = 3*(y * nx + x); // idx of the first channel + size_t base_dst = y * nx + x; // idx of the first channel + batch_entry[ base_dst] = imgs.entries[b]->buf[base_src ]; + batch_entry[1*n + base_dst] = imgs.entries[b]->buf[base_src + 1]; + batch_entry[2*n + base_dst] = imgs.entries[b]->buf[base_src + 2]; } } } @@ -3755,6 +3753,17 @@ bool clip_has_whisper_encoder(const struct clip_ctx * ctx) { } } +bool clip_model_supports_seq_input(const struct clip_ctx * ctx) { + switch (ctx->proj_type()) { + case PROJECTOR_TYPE_QWEN2VL: + case PROJECTOR_TYPE_QWEN25VL: + case PROJECTOR_TYPE_QWEN3VL: + return true; + default: + return false; + } +} + bool clip_encode_float_image (struct clip_ctx * ctx, int n_threads, float * img, int h, int w, float * vec) { clip_image_f32 clip_img; clip_img.buf.resize(h * w * 3); diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index a859b38658d3..6053f1b9e8fe 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -116,3 +116,6 @@ void clip_image_f32_batch_add_mel(struct clip_image_f32_batch * batch, int n_mel bool clip_has_vision_encoder(const struct clip_ctx * ctx); bool clip_has_audio_encoder(const struct clip_ctx * ctx); bool clip_has_whisper_encoder(const struct clip_ctx * ctx); + +// true if model graph support image->nt (temporal dimension) as input +bool clip_model_supports_seq_input(const struct clip_ctx * ctx); diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 03d99e15b054..8e91bf23df08 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -26,10 +26,11 @@ struct clip_graph_pixtral : clip_graph { struct clip_graph_qwen2vl : clip_graph { clip_graph_qwen2vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; + ggml_tensor * build_inp_with_temporal_merge(); }; -struct clip_graph_qwen3vl : clip_graph { - clip_graph_qwen3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} +struct clip_graph_qwen3vl : clip_graph_qwen2vl { + clip_graph_qwen3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen2vl(ctx, img) {} ggml_cgraph * build() override; }; diff --git a/tools/mtmd/models/qwen2vl.cpp b/tools/mtmd/models/qwen2vl.cpp index ebf10757376b..aa35fa492eaf 100644 --- a/tools/mtmd/models/qwen2vl.cpp +++ b/tools/mtmd/models/qwen2vl.cpp @@ -1,5 +1,31 @@ #include "models.h" +ggml_tensor * clip_graph_qwen2vl::build_inp_with_temporal_merge() { + ggml_tensor * inp_raw = build_inp_raw(); + + GGML_ASSERT(img.nx % (patch_size * 2) == 0); + GGML_ASSERT(img.ny % (patch_size * 2) == 0); + + const size_t nb1 = ggml_row_size(inp_raw->type, img.nx); + const size_t nb2 = nb1 * img.ny; + + if (nt == 1) { + // still image input + return ggml_add(ctx0, + ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1), + ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1)); + } else if (nt == 2) { + // 2 frames input (video input) + ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw, img.nx, img.ny, 3, nb1, nb2, 0); + ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw, img.nx, img.ny, 3, nb1, nb2, nb2 * 3); + return ggml_add(ctx0, + ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1), + ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1)); + } else { + GGML_ASSERT(false && "nt > 2 is not supported"); + } +} + ggml_cgraph * clip_graph_qwen2vl::build() { GGML_ASSERT(model.patch_bias == nullptr); GGML_ASSERT(model.class_embedding == nullptr); @@ -16,17 +42,10 @@ ggml_cgraph * clip_graph_qwen2vl::build() { int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4}; - ggml_tensor * inp_raw = build_inp_raw(); - ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1); - - GGML_ASSERT(img.nx % (patch_size * 2) == 0); - GGML_ASSERT(img.ny % (patch_size * 2) == 0); + ggml_tensor * inp = build_inp_with_temporal_merge(); // second conv dimension { - auto inp_1 = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1); - inp = ggml_add(ctx0, inp, inp_1); - inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); // [w, h, c, b] -> [c, w, h, b] inp = ggml_cont_4d( ctx0, inp, diff --git a/tools/mtmd/models/qwen3vl.cpp b/tools/mtmd/models/qwen3vl.cpp index 8afb811a653c..261e77a198af 100644 --- a/tools/mtmd/models/qwen3vl.cpp +++ b/tools/mtmd/models/qwen3vl.cpp @@ -13,35 +13,10 @@ ggml_cgraph * clip_graph_qwen3vl::build() { int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4}; - // detect video: 6-channel input means interleaved frame pairs (even_rgb + odd_rgb) - // for images (3ch), both Conv2Ds receive the same input (original behavior) - // for video (6ch), Conv2D_0 gets even frames (ch 0-2), Conv2D_1 gets odd frames (ch 3-5) - const bool is_video = (img.buf.size() == (size_t)img.nx * img.ny * 6); - const int n_channels = is_video ? 6 : 3; - - ggml_tensor * inp_raw = build_inp_raw(n_channels); - - ggml_tensor * inp; - if (is_video) { - const size_t nb1 = ggml_row_size(inp_raw->type, img.nx); - const size_t nb2 = nb1 * img.ny; - ggml_tensor * inp_even = ggml_view_3d(ctx0, inp_raw, img.nx, img.ny, 3, nb1, nb2, 0); - ggml_tensor * inp_odd = ggml_view_3d(ctx0, inp_raw, img.nx, img.ny, 3, nb1, nb2, nb2 * 3); - inp = ggml_add(ctx0, - ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_even, patch_size, patch_size, 0, 0, 1, 1), - ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_odd, patch_size, patch_size, 0, 0, 1, 1)); - } else { - inp = ggml_add(ctx0, - ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1), - ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1)); - } - - GGML_ASSERT(img.nx % (patch_size * 2) == 0); - GGML_ASSERT(img.ny % (patch_size * 2) == 0); + ggml_tensor * inp = build_inp_with_temporal_merge(); // spatial merge { - inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); // [w, h, c, b] -> [c, w, h, b] inp = ggml_cont_4d( ctx0, inp, diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index f825c5f97d10..2f45dab44778 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -176,28 +176,6 @@ struct decode_embd_batch { } } - // M-RoPE for video: 3D positions [temporal, height, width] - void set_position_mrope_3d(llama_pos pos_0, int nx, int ny, int nt, llama_seq_id seq_id) { - GGML_ASSERT(n_pos_per_embd == 4); - seq_id_0[0] = seq_id; - for (int t = 0; t < nt; t++) { - for (int y = 0; y < ny; y++) { - for (int x = 0; x < nx; x++) { - int i = t * ny * nx + y * nx + x; - pos[i ] = pos_0 + t; - pos[i + batch.n_tokens ] = pos_0 + y; - pos[i + batch.n_tokens * 2] = pos_0 + x; - pos[i + batch.n_tokens * 3] = 0; - } - } - } - for (int i = 0; i < batch.n_tokens; i++) { - batch.n_seq_id[i] = 1; - batch.seq_id [i] = seq_id_0.data(); - batch.logits [i] = false; - } - } - // M-RoPE for audio void set_position_mrope_1d(llama_pos pos_0, llama_seq_id seq_id) { GGML_ASSERT(n_pos_per_embd == 4); @@ -286,12 +264,7 @@ int32_t mtmd_helper_decode_image_chunk( } const int nx = mtmd_image_tokens_get_nx(image_tokens); const int ny = mtmd_image_tokens_get_ny(image_tokens); - const int nt = mtmd_image_tokens_get_nt(image_tokens); - if (nt > 1) { - batch_embd.set_position_mrope_3d(n_past, nx, ny, nt, seq_id); - } else { - batch_embd.set_position_mrope_2d(n_past, nx, ny, seq_id); - } + batch_embd.set_position_mrope_2d(n_past, nx, ny, seq_id); } else if (chunk_type == MTMD_INPUT_CHUNK_TYPE_AUDIO) { batch_embd.set_position_mrope_1d(n_past, seq_id); } else { diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 41d46e991aa0..1fb3bfc298e2 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -25,11 +25,11 @@ // represents raw image data, layout is RGBRGBRGB... // length of data must be nx * ny * 3 -// for video: data is n_frames sequential RGB frames, each nx * ny * 3 bytes +// for sequence of images (i.e. video): data is nt sequential RGB frames, each nx * ny * 3 bytes struct mtmd_bitmap { uint32_t nx; uint32_t ny; - uint32_t n_frames = 0; // 0 for single images, >= 2 (even) for video + uint32_t nt = 1; // 1 for single images, >= 2 (even) for sequence std::vector data; std::string id; // optional user-defined id, for ex: can be set to image hash, useful for KV cache tracking bool is_audio = false; // true if the bitmap is audio @@ -38,17 +38,15 @@ struct mtmd_bitmap { struct mtmd_image_tokens { uint32_t nx; // number of tokens in x direction uint32_t ny; // number of tokens in y direction - uint32_t nt = 1; // number of temporal positions (1 for images, > 1 for video) bool use_mrope_pos = false; // use M-RoPE position counting (the whole image is 1 temporal position) - uint32_t n_tokens() const { return nt * nx * ny; } clip_image_f32_batch batch_f32; // preprocessed image patches + uint32_t n_tokens() const { return nx * ny; } // TODO [QWEN_VIDEO]: we don't count nt here to be compatible with Qwen-VL, but other models in the future might have different logic std::string id; // optional user-defined ID, useful for KV cache tracking mtmd_image_tokens clone() { return mtmd_image_tokens{ nx, ny, - nt, use_mrope_pos, batch_f32.clone(), id @@ -678,10 +676,6 @@ struct mtmd_tokenizer { } int32_t add_media(const mtmd_bitmap * bitmap) { - if (bitmap->n_frames >= 2) { - return add_video(bitmap); - } - if (!bitmap->is_audio) { // handle image @@ -883,25 +877,26 @@ struct mtmd_tokenizer { return 0; } - // preprocess video frames and create an image chunk with temporal dimension - // frames are paired (even+odd), each pair becomes one 6-channel image - // each pair is encoded independently through the ViT (per-frame attention) - int32_t add_video(const mtmd_bitmap * bitmap) { - if (!ctx->ctx_v) { - LOG_ERR("%s: error: model does not support vision input\n", __func__); + int32_t add_seq_image(const mtmd_bitmap * bitmap) { + GGML_ASSERT(ctx->ctx_v); + GGML_ASSERT(bitmap->nt > 1); + // TODO [QWEN_VIDEO]: we only support even frames (Qwen-VL style) for now + GGML_ASSERT(bitmap->nt % 2 == 0); + bool support_seq = clip_model_supports_seq_input(ctx->ctx_v); + if (!support_seq) { + LOG_ERR("%s: error: model does not support sequential image input (usually requires Qwen-VL style models)\n", __func__); return 2; } - const uint32_t n_frames = bitmap->n_frames; - const uint32_t n_pairs = n_frames / 2; + const uint32_t n_frames = bitmap->nt; const size_t frame_bytes = (size_t)bitmap->nx * bitmap->ny * 3; - if (!ctx->img_beg.empty()) { - add_text(ctx->img_beg, true); - } - // preprocess each frame individually clip_image_f32_batch all_frames; + all_frames.is_seq = true; + all_frames.grid_x = 0; // currently, we don't support tiling for video input + all_frames.grid_y = 0; // currently, we don't support tiling for video input + for (uint32_t f = 0; f < n_frames; f++) { clip_image_u8_ptr img_u8(clip_image_u8_init()); img_u8->nx = bitmap->nx; @@ -910,59 +905,29 @@ struct mtmd_tokenizer { std::memcpy(img_u8->buf.data(), bitmap->data.data() + f * frame_bytes, frame_bytes); clip_image_f32_batch frame_batch; - bool ok = clip_image_preprocess(ctx->ctx_v, img_u8.get(), &frame_batch); + bool ok = ctx->image_preproc->preprocess(*img_u8, frame_batch); if (!ok) { - LOG_ERR("Unable to preprocess video frame %u\n", f); + LOG_ERR("Unable to preprocess image\n"); return 2; } GGML_ASSERT(frame_batch.entries.size() == 1); all_frames.entries.push_back(std::move(frame_batch.entries[0])); } - const int frame_nx = all_frames.entries[0]->nx; - const int frame_ny = all_frames.entries[0]->ny; - const int n_pixels = frame_nx * frame_ny; - - // interleave frame pairs into 6-channel images (even_rgb + odd_rgb) - // each pair is a separate batch entry, encoded independently - clip_image_f32_batch pair_batch; - for (uint32_t p = 0; p < n_pairs; p++) { - const auto & even = all_frames.entries[p * 2]; - const auto & odd = all_frames.entries[p * 2 + 1]; - GGML_ASSERT(even->nx == frame_nx && even->ny == frame_ny); - GGML_ASSERT(odd->nx == frame_nx && odd->ny == frame_ny); - - clip_image_f32_ptr pair(clip_image_f32_init()); - pair->nx = frame_nx; - pair->ny = frame_ny; - pair->buf.resize((size_t)n_pixels * 6); - - for (int i = 0; i < n_pixels; i++) { - const int dst = i * 6; - const int src = i * 3; - pair->buf[dst + 0] = even->buf[src + 0]; - pair->buf[dst + 1] = even->buf[src + 1]; - pair->buf[dst + 2] = even->buf[src + 2]; - pair->buf[dst + 3] = odd->buf[src + 0]; - pair->buf[dst + 4] = odd->buf[src + 1]; - pair->buf[dst + 5] = odd->buf[src + 2]; - } - pair_batch.entries.push_back(std::move(pair)); - } - - const uint32_t tokens_x = clip_n_output_tokens_x(ctx->ctx_v, pair_batch.entries[0].get()); - const uint32_t tokens_y = clip_n_output_tokens_y(ctx->ctx_v, pair_batch.entries[0].get()); - mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens); - image_tokens->nx = tokens_x; - image_tokens->ny = tokens_y; - image_tokens->nt = n_pairs; - image_tokens->use_mrope_pos = true; - image_tokens->batch_f32 = std::move(pair_batch); - image_tokens->id = bitmap->id; + if (mtmd_decode_use_mrope(ctx)) { + // for Qwen2VL, we need this information for M-RoPE decoding positions + image_tokens->nx = clip_n_output_tokens_x(ctx->ctx_v, all_frames.entries[0].get()); + image_tokens->ny = clip_n_output_tokens_y(ctx->ctx_v, all_frames.entries[0].get()); + image_tokens->use_mrope_pos = true; + } else { + GGML_ASSERT(false && "not supported"); + } + image_tokens->batch_f32 = std::move(all_frames); + image_tokens->id = bitmap->id; // optional - LOG_DBG("video: nt=%u, nx=%u, ny=%u, n_tokens=%u\n", - image_tokens->nt, image_tokens->nx, image_tokens->ny, image_tokens->n_tokens()); + LOG_DBG("seq_image: nt=%u, nx=%u, ny=%u, n_tokens=%u\n", + bitmap->nt, image_tokens->nx, image_tokens->ny, image_tokens->n_tokens()); mtmd_input_chunk chunk{ MTMD_INPUT_CHUNK_TYPE_IMAGE, @@ -1092,8 +1057,7 @@ int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens) ctx->image_embd_v.resize(image_tokens->n_tokens() * n_mmproj_embd); bool ok = false; - if (image_tokens->nt > 1 - || clip_is_llava(ctx_clip) + if (clip_is_llava(ctx_clip) || clip_is_minicpmv(ctx_clip) || clip_is_glm(ctx_clip) || proj_type == PROJECTOR_TYPE_INTERNVL) { @@ -1181,24 +1145,45 @@ mtmd_bitmap * mtmd_bitmap_init(uint32_t nx, mtmd_bitmap * bitmap = new mtmd_bitmap; bitmap->nx = nx; bitmap->ny = ny; + bitmap->nt = 1; size_t data_size = (size_t)nx * ny * 3; bitmap->data.resize(data_size); std::memcpy(bitmap->data.data(), data, data_size); return bitmap; } -mtmd_bitmap * mtmd_bitmap_init_from_video(uint32_t nx, - uint32_t ny, - uint32_t n_frames, - const unsigned char * data) { - GGML_ASSERT(n_frames >= 2 && n_frames % 2 == 0); +mtmd_bitmap * mtmd_bitmap_init_from_seq(uint32_t nx, + uint32_t ny, + uint32_t nt, + const unsigned char * data) { + if (nt == 0) { + LOG_ERR("%s: error: nt must be greater than 0 for sequence input\n", __func__); + return nullptr; + } + if (nt == 1) { + // if nt == 1, it's not really a sequence, we can treat it as a single image + return mtmd_bitmap_init(nx, ny, data); + } + // TODO [QWEN_VIDEO]: we only support Qwen-VL style for now, which requires even number of frames + // therefore, we duplicate the last frame if nt is odd, to avoid issues in video preprocessing + bool is_odd = (nt % 2 == 1); + if (is_odd) { + nt += 1; + } + size_t frame_size = (size_t)nx * ny * 3; mtmd_bitmap * bitmap = new mtmd_bitmap; bitmap->nx = nx; bitmap->ny = ny; - bitmap->n_frames = n_frames; - size_t data_size = (size_t)nx * ny * 3 * n_frames; + bitmap->nt = nt; + size_t data_size = frame_size * nt; bitmap->data.resize(data_size); std::memcpy(bitmap->data.data(), data, data_size); + if (is_odd) { + // duplicate the last frame + std::memcpy(bitmap->data.data() + (nt - 1) * frame_size, + data + (nt - 2) * frame_size, + frame_size); + } return bitmap; } @@ -1207,6 +1192,7 @@ mtmd_bitmap * mtmd_bitmap_init_from_audio(size_t n_samples, mtmd_bitmap * bitmap = new mtmd_bitmap; bitmap->nx = n_samples; bitmap->ny = 1; + bitmap->nt = 1; bitmap->is_audio = true; size_t data_size = n_samples * sizeof(float); bitmap->data.resize(data_size); @@ -1222,6 +1208,10 @@ uint32_t mtmd_bitmap_get_ny(const mtmd_bitmap * bitmap) { return bitmap->ny; } +uint32_t mtmd_bitmap_get_nt(const mtmd_bitmap * bitmap) { + return bitmap->nt; +} + const unsigned char * mtmd_bitmap_get_data(const mtmd_bitmap * bitmap) { return bitmap->data.data(); } @@ -1234,12 +1224,8 @@ bool mtmd_bitmap_is_audio(const mtmd_bitmap * bitmap) { return bitmap->is_audio; } -bool mtmd_bitmap_is_video(const mtmd_bitmap * bitmap) { - return bitmap->n_frames >= 2; -} - -uint32_t mtmd_bitmap_get_n_frames(const mtmd_bitmap * bitmap) { - return bitmap->n_frames; +bool mtmd_bitmap_is_seq(const mtmd_bitmap * bitmap) { + return bitmap->nt >= 2; } const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap) { @@ -1378,10 +1364,6 @@ size_t mtmd_image_tokens_get_ny(const mtmd_image_tokens * image_tokens) { return image_tokens->ny; } -size_t mtmd_image_tokens_get_nt(const mtmd_image_tokens * image_tokens) { - return image_tokens->nt; -} - const char * mtmd_image_tokens_get_id(const mtmd_image_tokens * image_tokens) { return image_tokens->id.c_str(); } @@ -1389,7 +1371,8 @@ const char * mtmd_image_tokens_get_id(const mtmd_image_tokens * image_tokens) { llama_pos mtmd_image_tokens_get_n_pos(const mtmd_image_tokens * image_tokens) { if (image_tokens->use_mrope_pos) { // for M-RoPE, n_pos = max(t, h, w) - return (llama_pos)std::max({image_tokens->nt, image_tokens->nx, image_tokens->ny}); + // t is omitted as we don't support batching + return std::max(image_tokens->nx, image_tokens->ny); } return image_tokens->n_tokens(); } diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index ec2a61954115..82fb533c4049 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -135,24 +135,24 @@ MTMD_API int mtmd_get_audio_sample_rate(mtmd_context * ctx); // if bitmap is image: // length of data must be nx * ny * 3 // the data is in RGBRGBRGB... format -// if bitmap is video: -// length of data must be nx * ny * 3 * n_frames -// n_frames must be >= 2 and even +// if bitmap is sequence of images (i.e. video): +// nt is the number of frames +// length of data must be nx * ny * 3 * nt // frames are sequential RGB, each nx * ny * 3 bytes // if bitmap is audio: // length of data must be n_samples * sizeof(float) // the data is in float format (PCM F32) MTMD_API mtmd_bitmap * mtmd_bitmap_init (uint32_t nx, uint32_t ny, const unsigned char * data); -MTMD_API mtmd_bitmap * mtmd_bitmap_init_from_video(uint32_t nx, uint32_t ny, uint32_t n_frames, const unsigned char * data); +MTMD_API mtmd_bitmap * mtmd_bitmap_init_from_seq (uint32_t nx, uint32_t ny, uint32_t nt, const unsigned char * data); MTMD_API mtmd_bitmap * mtmd_bitmap_init_from_audio(size_t n_samples, const float * data); -MTMD_API uint32_t mtmd_bitmap_get_nx (const mtmd_bitmap * bitmap); -MTMD_API uint32_t mtmd_bitmap_get_ny (const mtmd_bitmap * bitmap); -MTMD_API const unsigned char * mtmd_bitmap_get_data (const mtmd_bitmap * bitmap); -MTMD_API size_t mtmd_bitmap_get_n_bytes (const mtmd_bitmap * bitmap); -MTMD_API bool mtmd_bitmap_is_audio (const mtmd_bitmap * bitmap); -MTMD_API bool mtmd_bitmap_is_video (const mtmd_bitmap * bitmap); -MTMD_API uint32_t mtmd_bitmap_get_n_frames(const mtmd_bitmap * bitmap); -MTMD_API void mtmd_bitmap_free (mtmd_bitmap * bitmap); +MTMD_API uint32_t mtmd_bitmap_get_nx (const mtmd_bitmap * bitmap); +MTMD_API uint32_t mtmd_bitmap_get_ny (const mtmd_bitmap * bitmap); +MTMD_API uint32_t mtmd_bitmap_get_nt (const mtmd_bitmap * bitmap); +MTMD_API const unsigned char * mtmd_bitmap_get_data (const mtmd_bitmap * bitmap); +MTMD_API size_t mtmd_bitmap_get_n_bytes(const mtmd_bitmap * bitmap); +MTMD_API bool mtmd_bitmap_is_audio (const mtmd_bitmap * bitmap); +MTMD_API bool mtmd_bitmap_is_seq (const mtmd_bitmap * bitmap); +MTMD_API void mtmd_bitmap_free (mtmd_bitmap * bitmap); // bitmap ID is optional, but useful for KV cache tracking // these getters/setters are dedicated functions, so you can for example calculate the hash of the image based on mtmd_bitmap_get_data() MTMD_API const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap); @@ -195,7 +195,6 @@ MTMD_API void mtmd_input_chunk_free(mtmd_input_chunk * chunk); MTMD_API size_t mtmd_image_tokens_get_n_tokens(const mtmd_image_tokens * image_tokens); // TODO: deprecate MTMD_API size_t mtmd_image_tokens_get_nx (const mtmd_image_tokens * image_tokens); MTMD_API size_t mtmd_image_tokens_get_ny (const mtmd_image_tokens * image_tokens); -MTMD_API size_t mtmd_image_tokens_get_nt (const mtmd_image_tokens * image_tokens); MTMD_API const char * mtmd_image_tokens_get_id (const mtmd_image_tokens * image_tokens); // TODO: deprecate // number of temporal positions (equals to max(t,h,w) for M-RoPE; equals to n_tokens otherwise) MTMD_API llama_pos mtmd_image_tokens_get_n_pos (const mtmd_image_tokens * image_tokens); // TODO: deprecate @@ -285,14 +284,14 @@ struct bitmap { bitmap(uint32_t nx, uint32_t ny, const unsigned char * data) { ptr.reset(mtmd_bitmap_init(nx, ny, data)); } - bitmap(uint32_t nx, uint32_t ny, uint32_t n_frames, const unsigned char * data) { - ptr.reset(mtmd_bitmap_init_from_video(nx, ny, n_frames, data)); + bitmap(uint32_t nx, uint32_t ny, uint32_t nt, const unsigned char * data) { + ptr.reset(mtmd_bitmap_init_from_seq(nx, ny, nt, data)); } ~bitmap() = default; - uint32_t nx() const { return mtmd_bitmap_get_nx(ptr.get()); } - uint32_t ny() const { return mtmd_bitmap_get_ny(ptr.get()); } - uint32_t n_frames() const { return mtmd_bitmap_get_n_frames(ptr.get()); } - bool is_video() const { return mtmd_bitmap_is_video(ptr.get()); } + uint32_t nx() const { return mtmd_bitmap_get_nx(ptr.get()); } + uint32_t ny() const { return mtmd_bitmap_get_ny(ptr.get()); } + uint32_t nt() const { return mtmd_bitmap_get_nt(ptr.get()); } + bool is_seq() const { return mtmd_bitmap_is_seq(ptr.get()); } const unsigned char * data() const { return mtmd_bitmap_get_data(ptr.get()); } size_t n_bytes() const { return mtmd_bitmap_get_n_bytes(ptr.get()); } std::string id() const { return mtmd_bitmap_get_id(ptr.get()); } From a404c4ea0b549afb0f9045ba35ff8b5fcda07dc5 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Sat, 6 Jun 2026 17:44:23 +0200 Subject: [PATCH 3/6] revise the design --- tools/mtmd/clip-graph.h | 2 +- tools/mtmd/clip-impl.h | 20 +-- tools/mtmd/clip.cpp | 24 ++-- tools/mtmd/clip.h | 10 +- tools/mtmd/models/qwen2vl.cpp | 21 +-- tools/mtmd/mtmd-image.cpp | 2 +- tools/mtmd/mtmd.cpp | 240 ++++++++++++++-------------------- tools/mtmd/mtmd.h | 18 +-- 8 files changed, 148 insertions(+), 189 deletions(-) diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/clip-graph.h index c1eae6bf1935..7d10586217b8 100644 --- a/tools/mtmd/clip-graph.h +++ b/tools/mtmd/clip-graph.h @@ -38,7 +38,7 @@ struct clip_graph { const clip_flash_attn_type flash_attn_type; // TODO [QWEN_VIDEO]: improve this in the future - int nt = 1; // number of temporal dim, to be used by Qwen-VL models + int n_batch = 1; ggml_context_ptr ctx0_ptr; ggml_context * ctx0; diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 3f7518beeafc..aaa8f0812784 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -480,10 +480,6 @@ struct clip_image_u8 { buf[idx + 2] = rgb[2]; } - size_t n_pixels() const { - return (size_t) nx * (size_t) ny; - } - size_t n_elements() const { return n_pixels() * 3; } @@ -492,10 +488,17 @@ struct clip_image_u8 { std::vector buf; int nx = 0; int ny = 0; + // note: we don't need nt here because preprocessor always expect one single image + + size_t n_pixels() const { + return (size_t) nx * (size_t) ny; + } }; // For images, buf.size() == nx*ny*3 // Memory layout: RGBRGBRGB... +// For seq, buf.size() == nx*ny*3*nt +// Memory layout: RGBRGB...RGBRGB... (nt times) // For audio, only one channel is used, buf.size() == nx*ny // nx will be n_frames and ny will be n_mel struct clip_image_f32 { @@ -544,10 +547,6 @@ struct clip_image_f32 { } } - size_t n_pixels() const { - return (size_t) nx_ * (size_t) ny_; - } - size_t n_elements() const { return n_pixels() * 3; } @@ -580,6 +579,10 @@ struct clip_image_f32 { std::vector buf; int nx_ = 0; int ny_ = 0; + + size_t n_pixels() const { + return (size_t) nx_ * (size_t) ny_; + } }; // @@ -627,6 +630,7 @@ static void clip_log_internal(enum ggml_log_level level, const char * format, .. va_end(args); } +#define LOG_TRC(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) #define LOG_DBG(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) #define LOG_INF(...) clip_log_internal(GGML_LOG_LEVEL_INFO, __VA_ARGS__) #define LOG_WRN(...) clip_log_internal(GGML_LOG_LEVEL_WARN, __VA_ARGS__) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 0c70ad6e8fc8..4cfd16b0d4f9 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -527,7 +527,7 @@ ggml_tensor * clip_graph::build_inp() { } ggml_tensor * clip_graph::build_inp_raw(int channels) { - ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, img.nx(), img.ny(), channels, nt); + ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, img.nx(), img.ny(), channels, n_batch); ggml_set_name(inp_raw, "inp_raw"); ggml_set_input(inp_raw); return inp_raw; @@ -848,8 +848,6 @@ ggml_tensor * clip_graph::build_patch_merge_permute(ggml_tensor * cur, int scale } static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32_batch & imgs) { - GGML_ASSERT(imgs.entries.size() == 1 && "n_batch > 1 is not supported"); - const clip_image_f32 & img = *imgs.entries[0]; std::unique_ptr builder; @@ -1010,7 +1008,7 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 } // TODO [QWEN_VIDEO]: improve this in the future - builder->nt = imgs.entries.size(); + builder->n_batch = imgs.entries.size(); return builder->build(); } @@ -3482,13 +3480,15 @@ bool clip_image_encode(struct clip_ctx * ctx, const int n_threads, clip_image_f3 bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_image_f32_batch * imgs_c_ptr, float * vec) { const clip_image_f32_batch & imgs = *imgs_c_ptr; - int batch_size = imgs.entries.size(); - bool support_seq = clip_model_supports_seq_input(ctx); + int n_batch_cur = imgs.entries.size(); + + // maximum supported batch size, usually == 2 for qwen-vl-based models + int n_batch_max = clip_model_n_batch_max(ctx); // TODO @ngxson : implement batch size > 1 as a loop // we don't need true batching support because the cgraph will gonna be big anyway - if (batch_size != 1 && !support_seq) { - return false; // only support batch size of 1 + if (n_batch_cur > n_batch_max) { + return false; } // if buffers are not allocated, we need to do a warmup run to allocate them @@ -3566,7 +3566,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima const int ny = imgs.entries[i]->ny(); const int n = nx * ny; - for (int b = 0; b < batch_size; b++) { + for (int b = 0; b < n_batch_cur; b++) { const auto & buf = imgs.entries[b]->get_ro_buf(); float * batch_entry = inp_raw.data() + b * (3*n); for (int y = 0; y < ny; y++) { @@ -4555,14 +4555,14 @@ bool clip_has_audio_encoder(const struct clip_ctx * ctx) { return ctx->model.modality == CLIP_MODALITY_AUDIO; } -bool clip_model_supports_seq_input(const struct clip_ctx * ctx) { +int clip_model_n_batch_max(const struct clip_ctx * ctx) { switch (ctx->proj_type()) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: - return true; + return 2; default: - return false; + return 1; } } diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index ed6aa7e18cb5..7387b7b87cfc 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -20,6 +20,12 @@ struct clip_image_size { bool operator==(const clip_image_size & other) const { return width == other.width && height == other.height; } + bool operator!=(const clip_image_size & other) const { + return !(*this == other); + } + int area() const { + return width * height; + } }; struct clip_image_f32; @@ -101,8 +107,8 @@ bool clip_is_llava(const struct clip_ctx * ctx); bool clip_has_vision_encoder(const struct clip_ctx * ctx); bool clip_has_audio_encoder(const struct clip_ctx * ctx); -// true if model graph support image->nt (temporal dimension) as input -bool clip_model_supports_seq_input(const struct clip_ctx * ctx); +// return the max number of model's cgraph image->nt (temporal dimension) as input +int clip_model_n_batch_max(const struct clip_ctx * ctx); std::map clip_get_mem_usage(const struct clip_ctx * ctx); diff --git a/tools/mtmd/models/qwen2vl.cpp b/tools/mtmd/models/qwen2vl.cpp index aa35fa492eaf..2220c2692a19 100644 --- a/tools/mtmd/models/qwen2vl.cpp +++ b/tools/mtmd/models/qwen2vl.cpp @@ -3,26 +3,29 @@ ggml_tensor * clip_graph_qwen2vl::build_inp_with_temporal_merge() { ggml_tensor * inp_raw = build_inp_raw(); - GGML_ASSERT(img.nx % (patch_size * 2) == 0); - GGML_ASSERT(img.ny % (patch_size * 2) == 0); + GGML_ASSERT(img.nx() % (patch_size * 2) == 0); + GGML_ASSERT(img.ny() % (patch_size * 2) == 0); - const size_t nb1 = ggml_row_size(inp_raw->type, img.nx); - const size_t nb2 = nb1 * img.ny; + const size_t nb1 = ggml_row_size(inp_raw->type, img.nx()); + const size_t nb2 = ggml_row_size(inp_raw->type, img.nx() * img.ny()); - if (nt == 1) { + if (n_batch == 1) { // still image input return ggml_add(ctx0, ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1), ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1)); - } else if (nt == 2) { + } else if (n_batch == 2) { // 2 frames input (video input) - ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw, img.nx, img.ny, 3, nb1, nb2, 0); - ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw, img.nx, img.ny, 3, nb1, nb2, nb2 * 3); + ggml_tensor * inp_0 = ggml_view_3d(ctx0, inp_raw, + img.nx(), img.ny(), 3, nb1, nb2, 0); + ggml_tensor * inp_1 = ggml_view_3d(ctx0, inp_raw, + img.nx(), img.ny(), 3, nb1, nb2, + nb2 * 3); // move to the second frame return ggml_add(ctx0, ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_0, patch_size, patch_size, 0, 0, 1, 1), ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_1, patch_size, patch_size, 0, 0, 1, 1)); } else { - GGML_ASSERT(false && "nt > 2 is not supported"); + GGML_ASSERT(false && "n_batch > 2 is not supported"); } } diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index c86a065c814e..bedf44e07cf3 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -1116,7 +1116,7 @@ bool mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img, static constexpr int native_resolutions[] = { 1024 /* base */, 1280 /* large */ }; // TODO: support 512 (tiny) and 640 (small) once we have eval data for them - const int64_t orig_area = static_cast(img.n_pixels()); + const int64_t orig_area = static_cast(img.get_size().area()); size_t mode_i = 0; int64_t min_diff = std::numeric_limits::max(); diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 3388a9bfedc8..cf8169245cd7 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -35,12 +35,11 @@ struct mtmd_bitmap { uint32_t nx = 0; uint32_t ny = 0; - uint32_t nt = 1; // 1 for single images, >= 2 (even) for sequence std::string id; // optional user-defined id, for ex: can be set to image hash, useful for KV cache tracking bool is_audio = false; // true if the bitmap is audio mtmd_bitmap(const unsigned char * data, uint32_t nx, uint32_t ny) - : nx(nx), ny(ny) { + : nx(nx), ny(ny), is_audio(false) { if (data) { size_t data_size = (size_t)nx * ny * 3; this->data.resize(data_size); @@ -69,6 +68,11 @@ struct mtmd_bitmap { return data.size(); } + bool can_batch_with(const mtmd_bitmap & other) const { + // [QWEN_VIDEO] can batch if both are images with same size + return !is_audio && !other.is_audio && nx == other.nx && ny == other.ny; + } + private: std::vector data; }; @@ -93,7 +97,6 @@ struct mtmd_image_tokens { return nx * ny; } clip_image_f32_batch batch_f32; // preprocessed image patches - uint32_t n_tokens() const { return nx * ny; } // TODO [QWEN_VIDEO]: we don't count nt here to be compatible with Qwen-VL, but other models in the future might have different logic std::string id; // optional user-defined ID, useful for KV cache tracking // true if one of entries in batch_f32 is a placeholder @@ -756,16 +759,55 @@ struct mtmd_tokenizer { cur.entries.clear(); std::vector parts = split_text(input_text, ctx->media_marker); size_t i_bm = 0; // index of the current bitmap + + // [QWEN_VIDEO] handle frame merging for models that support it (i.e. qwen-vl) + int n_merge_frames = 1; + if (ctx->ctx_v) { + n_merge_frames = clip_model_n_batch_max(ctx->ctx_v); + GGML_ASSERT(n_merge_frames <= 2 && "we only support merging maximum 2 images for now; open an issue if this model supports merging more"); + } + + std::vector> merged_bitmaps; + if (n_merge_frames > 1) { + size_t i_bm_scan = 0; + for (size_t i = 0; i < parts.size(); ++i) { + if (parts[i] != ctx->media_marker) { + continue; + } + if (i + 1 < parts.size() + && parts[i + 1] == ctx->media_marker + && i_bm_scan + 1 < bitmaps.size()) { + const mtmd_bitmap * bm_a = bitmaps[i_bm_scan]; + const mtmd_bitmap * bm_b = bitmaps[i_bm_scan + 1]; + if (bm_a->can_batch_with(*bm_b)) { + LOG_DBG("%s: merging 2 frames at bitmap index %zu and %zu\n", __func__, i_bm_scan, i_bm_scan + 1); + merged_bitmaps.push_back({bm_a, bm_b}); + parts.erase(parts.begin() + i + 1); // remove the second marker + i_bm_scan += 2; + continue; + } + } + LOG_DBG("%s: no merging for bitmap index %zu\n", __func__, i_bm_scan); + merged_bitmaps.push_back({bitmaps[i_bm_scan]}); + ++i_bm_scan; + } + } else { + for (size_t i = 0; i < bitmaps.size(); ++i) { + merged_bitmaps.push_back({bitmaps[i]}); + } + } + + i_bm = 0; for (auto & part : parts) { if (part == ctx->media_marker) { // this is a marker, we should add the next bitmap - if (i_bm >= bitmaps.size()) { + if (i_bm >= merged_bitmaps.size()) { LOG_ERR("%s: error: number of bitmaps (%zu) does not match number of markers (%zu)\n", - __func__, bitmaps.size(), parts.size() - 1); + __func__, merged_bitmaps.size(), parts.size() - 1); return 1; } - const mtmd_bitmap * bitmap = bitmaps[i_bm++]; - int32_t res = add_media(bitmap); + auto & bmps = merged_bitmaps[i_bm++]; + int32_t res = add_media(bmps); if (res != 0) { return res; } @@ -800,9 +842,9 @@ struct mtmd_tokenizer { } } - if (i_bm != bitmaps.size()) { + if (i_bm != merged_bitmaps.size()) { LOG_ERR("%s: error: number of bitmaps (%zu) does not match number of markers (%zu)\n", - __func__, bitmaps.size(), parts.size() - 1); + __func__, merged_bitmaps.size(), parts.size() - 1); return 1; } @@ -841,8 +883,10 @@ struct mtmd_tokenizer { } } - int32_t add_media(const mtmd_bitmap * bitmap) { - if (!bitmap->is_audio) { + int32_t add_media(std::vector & bitmaps) { + GGML_ASSERT(!bitmaps.empty()); + + if (!bitmaps[0]->is_audio) { // handle image if (!ctx->ctx_v) { @@ -854,27 +898,39 @@ struct mtmd_tokenizer { add_text(ctx->img_beg, true); // add image begin token } - // sanity check - if (bitmap->nx <= 0 || bitmap->ny <= 0) { - LOG_ERR("%s: error: invalid bitmap dimensions: nx = %d, ny = %d\n", - __func__, bitmap->nx, bitmap->ny); - return 2; - } - GGML_ASSERT(ctx->image_preproc != nullptr); - - // convert mtmd_bitmap to clip_image_u8 - clip_image_u8_ptr img_u8(clip_image_u8_init()); - img_u8->set_size( - {(int)bitmap->nx, (int)bitmap->ny}, - bitmap->is_placeholder()); - img_u8->cpy_buf(bitmap->get_ro_buf()); + // TODO @ngxson : this is quite hacky because preprocessor only support batch with one single element, that need to be fixed in the future (e.g. by changing the preprocessor interface always take single input) - // preprocess image clip_image_f32_batch batch_f32; - bool ok = ctx->image_preproc->preprocess(*img_u8, batch_f32); - if (!ok) { - LOG_ERR("Unable to preprocess image\n"); - return 2; + + for (const auto * bmp : bitmaps) { + // sanity check + GGML_ASSERT(!bmp->is_audio); + GGML_ASSERT(ctx->image_preproc != nullptr); + if (bmp->nx <= 0 || bmp->ny <= 0) { + LOG_ERR("%s: error: invalid bitmap dimensions: nx = %d, ny = %d\n", + __func__, bmp->nx, bmp->ny); + return 2; + } + + // convert mtmd_bitmap to clip_image_u8 + clip_image_u8_ptr img_u8(clip_image_u8_init()); + img_u8->set_size( + {(int)bmp->nx, (int)bmp->ny}, + bmp->is_placeholder()); + img_u8->cpy_buf(bmp->get_ro_buf()); + + // preprocess image + clip_image_f32_batch tmp_batch; + bool ok = ctx->image_preproc->preprocess(*img_u8, tmp_batch); + if (!ok) { + LOG_ERR("Unable to preprocess image\n"); + return 2; + } + + // move them back to the "global" batch_f32 + for (auto & entry : tmp_batch.entries) { + batch_f32.entries.emplace_back(std::move(entry)); + } } // Annotate llava-next style tiles so clip_n_output_tokens accounts @@ -902,11 +958,14 @@ struct mtmd_tokenizer { || ctx->slice_tmpl == MTMD_SLICE_TMPL_STEP3VL || (ctx->slice_tmpl == MTMD_SLICE_TMPL_LFM2 && has_tiling_grid) ) { + // [QWEN_VIDEO] we do not support "frame merging" for llama-uhd style, so no batching for now + GGML_ASSERT(bitmaps.size() == 1); + const int n_col = batch_f32.grid_x; const int n_row = batch_f32.grid_y; // split batch into chunks of single images // NOTE: batch_f32 will be invalidated after this call - auto chunks = split_batch_to_chunk(std::move(batch_f32), bitmap->id); + auto chunks = split_batch_to_chunk(std::move(batch_f32), bitmaps[0]->id); GGML_ASSERT(chunks.size() > 0); auto ov_chunk = std::move(chunks.front()); @@ -960,6 +1019,10 @@ struct mtmd_tokenizer { size_t n_tokens = 0; for (const auto & e : batch_f32.entries) { n_tokens += clip_n_output_tokens(ctx->ctx_v, e.get()); + if (clip_model_n_batch_max(ctx->ctx_v) == 2) { + // [QWEN_VIDEO] pair input is merged to the same embd, so only count as one image + break; + } } mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens); @@ -982,7 +1045,7 @@ struct mtmd_tokenizer { GGML_ASSERT(n_tokens == (size_t)image_tokens->n_tokens()); } image_tokens->batch_f32 = std::move(batch_f32); - image_tokens->id = bitmap->id; // optional + image_tokens->id = bitmaps[0]->id; // optional LOG_DBG("image_tokens->nx = %d\n", image_tokens->nx); LOG_DBG("image_tokens->ny = %d\n", image_tokens->ny); @@ -1007,6 +1070,9 @@ struct mtmd_tokenizer { } else { // handle audio + GGML_ASSERT(bitmaps.size() == 1); // no batching support for now + auto & bitmap = bitmaps[0]; + if (!ctx->ctx_a) { LOG_ERR("%s: error: model does not support audio input\n", __func__); return 2; @@ -1093,73 +1159,6 @@ struct mtmd_tokenizer { return 0; } - int32_t add_seq_image(const mtmd_bitmap * bitmap) { - GGML_ASSERT(ctx->ctx_v); - GGML_ASSERT(bitmap->nt > 1); - // TODO [QWEN_VIDEO]: we only support even frames (Qwen-VL style) for now - GGML_ASSERT(bitmap->nt % 2 == 0); - bool support_seq = clip_model_supports_seq_input(ctx->ctx_v); - if (!support_seq) { - LOG_ERR("%s: error: model does not support sequential image input (usually requires Qwen-VL style models)\n", __func__); - return 2; - } - - const uint32_t n_frames = bitmap->nt; - const size_t frame_bytes = (size_t)bitmap->nx * bitmap->ny * 3; - - // preprocess each frame individually - clip_image_f32_batch all_frames; - all_frames.is_seq = true; - all_frames.grid_x = 0; // currently, we don't support tiling for video input - all_frames.grid_y = 0; // currently, we don't support tiling for video input - - for (uint32_t f = 0; f < n_frames; f++) { - clip_image_u8_ptr img_u8(clip_image_u8_init()); - img_u8->nx = bitmap->nx; - img_u8->ny = bitmap->ny; - img_u8->buf.resize(frame_bytes); - std::memcpy(img_u8->buf.data(), bitmap->data.data() + f * frame_bytes, frame_bytes); - - clip_image_f32_batch frame_batch; - bool ok = ctx->image_preproc->preprocess(*img_u8, frame_batch); - if (!ok) { - LOG_ERR("Unable to preprocess image\n"); - return 2; - } - GGML_ASSERT(frame_batch.entries.size() == 1); - all_frames.entries.push_back(std::move(frame_batch.entries[0])); - } - - mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens); - if (mtmd_decode_use_mrope(ctx)) { - // for Qwen2VL, we need this information for M-RoPE decoding positions - image_tokens->nx = clip_n_output_tokens_x(ctx->ctx_v, all_frames.entries[0].get()); - image_tokens->ny = clip_n_output_tokens_y(ctx->ctx_v, all_frames.entries[0].get()); - image_tokens->pos = MTMD_POS_TYPE_MROPE; - } else { - GGML_ASSERT(false && "not supported"); - } - image_tokens->batch_f32 = std::move(all_frames); - image_tokens->id = bitmap->id; // optional - - LOG_DBG("seq_image: nt=%u, nx=%u, ny=%u, n_tokens=%u\n", - bitmap->nt, image_tokens->nx, image_tokens->ny, image_tokens->n_tokens()); - - mtmd_input_chunk chunk{ - MTMD_INPUT_CHUNK_TYPE_IMAGE, - {}, // text tokens - std::move(image_tokens), - nullptr, // audio tokens - }; - cur.entries.emplace_back(std::move(chunk)); - - if (!ctx->img_end.empty()) { - add_text(ctx->img_end, true); - } - - return 0; - } - std::vector split_batch_to_chunk(clip_image_f32_batch && batch_f32, const std::string & id) { std::vector chunks; @@ -1386,41 +1385,6 @@ mtmd_bitmap * mtmd_bitmap_init(uint32_t nx, return bitmap; } -mtmd_bitmap * mtmd_bitmap_init_from_seq(uint32_t nx, - uint32_t ny, - uint32_t nt, - const unsigned char * data) { - if (nt == 0) { - LOG_ERR("%s: error: nt must be greater than 0 for sequence input\n", __func__); - return nullptr; - } - if (nt == 1) { - // if nt == 1, it's not really a sequence, we can treat it as a single image - return mtmd_bitmap_init(nx, ny, data); - } - // TODO [QWEN_VIDEO]: we only support Qwen-VL style for now, which requires even number of frames - // therefore, we duplicate the last frame if nt is odd, to avoid issues in video preprocessing - bool is_odd = (nt % 2 == 1); - if (is_odd) { - nt += 1; - } - size_t frame_size = (size_t)nx * ny * 3; - mtmd_bitmap * bitmap = new mtmd_bitmap; - bitmap->nx = nx; - bitmap->ny = ny; - bitmap->nt = nt; - size_t data_size = frame_size * nt; - bitmap->data.resize(data_size); - std::memcpy(bitmap->data.data(), data, data_size); - if (is_odd) { - // duplicate the last frame - std::memcpy(bitmap->data.data() + (nt - 1) * frame_size, - data + (nt - 2) * frame_size, - frame_size); - } - return bitmap; -} - mtmd_bitmap * mtmd_bitmap_init_from_audio(size_t n_samples, const float * data) { mtmd_bitmap * bitmap = new mtmd_bitmap((const unsigned char *)data, n_samples); @@ -1439,10 +1403,6 @@ uint32_t mtmd_bitmap_get_ny(const mtmd_bitmap * bitmap) { return bitmap->ny; } -uint32_t mtmd_bitmap_get_nt(const mtmd_bitmap * bitmap) { - return bitmap->nt; -} - const unsigned char * mtmd_bitmap_get_data(const mtmd_bitmap * bitmap) { return bitmap->get_ro_buf().data(); } @@ -1455,10 +1415,6 @@ bool mtmd_bitmap_is_audio(const mtmd_bitmap * bitmap) { return bitmap->is_audio; } -bool mtmd_bitmap_is_seq(const mtmd_bitmap * bitmap) { - return bitmap->nt >= 2; -} - const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap) { return bitmap->id.c_str(); } diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index 987f92e1ab79..128fb18261b9 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -133,10 +133,8 @@ MTMD_API int mtmd_get_audio_sample_rate(const mtmd_context * ctx); // if bitmap is image: // length of data must be nx * ny * 3 // the data is in RGBRGBRGB... format -// if bitmap is sequence of images (i.e. video): -// nt is the number of frames -// length of data must be nx * ny * 3 * nt -// frames are sequential RGB, each nx * ny * 3 bytes +// note: some video-capable models (i.e. qwen-vl) can merge consecutive bitmaps +// into one chunk, mtmd_tokenize() will automatically handle this // if bitmap is audio: // length of data must be n_samples * sizeof(float) // the data is in float format (PCM F32) @@ -146,15 +144,12 @@ MTMD_API int mtmd_get_audio_sample_rate(const mtmd_context * ctx); // you can pass the bitmap via mtmd_tokenize(), then call mtmd_*_get_n_tokens() to count the tokens // note: passing a placeholder bitmap to mtmd_encode() will return an error MTMD_API mtmd_bitmap * mtmd_bitmap_init (uint32_t nx, uint32_t ny, const unsigned char * data); -MTMD_API mtmd_bitmap * mtmd_bitmap_init_from_seq (uint32_t nx, uint32_t ny, uint32_t nt, const unsigned char * data); MTMD_API mtmd_bitmap * mtmd_bitmap_init_from_audio(size_t n_samples, const float * data); MTMD_API uint32_t mtmd_bitmap_get_nx (const mtmd_bitmap * bitmap); MTMD_API uint32_t mtmd_bitmap_get_ny (const mtmd_bitmap * bitmap); -MTMD_API uint32_t mtmd_bitmap_get_nt (const mtmd_bitmap * bitmap); MTMD_API const unsigned char * mtmd_bitmap_get_data (const mtmd_bitmap * bitmap); MTMD_API size_t mtmd_bitmap_get_n_bytes(const mtmd_bitmap * bitmap); MTMD_API bool mtmd_bitmap_is_audio (const mtmd_bitmap * bitmap); -MTMD_API bool mtmd_bitmap_is_seq (const mtmd_bitmap * bitmap); MTMD_API void mtmd_bitmap_free (mtmd_bitmap * bitmap); // bitmap ID is optional, but useful for KV cache tracking // these getters/setters are dedicated functions, so you can for example calculate the hash of the image based on mtmd_bitmap_get_data() @@ -318,14 +313,9 @@ struct bitmap { bitmap(uint32_t nx, uint32_t ny, const unsigned char * data) { ptr.reset(mtmd_bitmap_init(nx, ny, data)); } - bitmap(uint32_t nx, uint32_t ny, uint32_t nt, const unsigned char * data) { - ptr.reset(mtmd_bitmap_init_from_seq(nx, ny, nt, data)); - } ~bitmap() = default; - uint32_t nx() const { return mtmd_bitmap_get_nx(ptr.get()); } - uint32_t ny() const { return mtmd_bitmap_get_ny(ptr.get()); } - uint32_t nt() const { return mtmd_bitmap_get_nt(ptr.get()); } - bool is_seq() const { return mtmd_bitmap_is_seq(ptr.get()); } + uint32_t nx() const { return mtmd_bitmap_get_nx(ptr.get()); } + uint32_t ny() const { return mtmd_bitmap_get_ny(ptr.get()); } const unsigned char * data() const { return mtmd_bitmap_get_data(ptr.get()); } size_t n_bytes() const { return mtmd_bitmap_get_n_bytes(ptr.get()); } std::string id() const { return mtmd_bitmap_get_id(ptr.get()); } From 82b48212fd2267dddd7f59f1a7b866d0650b795a Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Sat, 6 Jun 2026 18:20:01 +0200 Subject: [PATCH 4/6] fix llava-uhd case --- tools/mtmd/mtmd.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index cf8169245cd7..a697dcabfe57 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -927,10 +927,15 @@ struct mtmd_tokenizer { return 2; } - // move them back to the "global" batch_f32 + // move entries and grid dimensions to the "global" batch_f32 for (auto & entry : tmp_batch.entries) { batch_f32.entries.emplace_back(std::move(entry)); } + + // for llava-uhd style, we need to handle grid too + // we don't care about overwriting these values for now because llama-uhd doesn't support batching anyway + batch_f32.grid_x = tmp_batch.grid_x; + batch_f32.grid_y = tmp_batch.grid_y; } // Annotate llava-next style tiles so clip_n_output_tokens accounts From 9819ad4317ec1f22e8bc75671ee2614aec0453e8 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Sat, 6 Jun 2026 18:23:06 +0200 Subject: [PATCH 5/6] nits --- tools/mtmd/clip-impl.h | 2 -- tools/mtmd/clip.cpp | 14 +++++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index aaa8f0812784..4b75b23887a3 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -666,7 +666,6 @@ struct clip_image_u8_batch { struct clip_image_f32_batch { std::vector entries; bool is_audio = false; - bool is_seq = true; // for llava-uhd style models, we need to know the grid size // note: entries.size() == grid_x * grid_y + 1 (one overview image) @@ -677,7 +676,6 @@ struct clip_image_f32_batch { clip_image_f32_batch new_batch{ /* entries */ {}, /* is_audio */ is_audio, - /* is_seq */ is_seq, /* grid_x */ grid_x, /* grid_y */ grid_y, }; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 4cfd16b0d4f9..bd33f430625a 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -3560,19 +3560,19 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima // ──────┘ x B // IMPORTANT: [QWEN_VIDEO] the batch dim is currently used for temporal dim in Qwen-VL models - - for (size_t i = 0; i < imgs.entries.size(); i++) { - const int nx = imgs.entries[i]->nx(); - const int ny = imgs.entries[i]->ny(); - const int n = nx * ny; + // All entries must have the same spatial size (enforced by can_batch_with() during merging) + { + const int nx = imgs.entries[0]->nx(); + const int ny = imgs.entries[0]->ny(); + const int n = nx * ny; for (int b = 0; b < n_batch_cur; b++) { const auto & buf = imgs.entries[b]->get_ro_buf(); float * batch_entry = inp_raw.data() + b * (3*n); for (int y = 0; y < ny; y++) { for (int x = 0; x < nx; x++) { - size_t base_src = 3*(y * nx + x); // idx of the first channel - size_t base_dst = y * nx + x; // idx of the first channel + size_t base_src = 3*(y * nx + x); + size_t base_dst = y * nx + x; batch_entry[ base_dst] = buf[base_src ]; batch_entry[1*n + base_dst] = buf[base_src + 1]; batch_entry[2*n + base_dst] = buf[base_src + 2]; From b031b6062da7244ef16095522703edd2e7fa2b4e Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Sat, 6 Jun 2026 18:25:05 +0200 Subject: [PATCH 6/6] nits 2 --- tools/mtmd/clip-impl.h | 1 - tools/mtmd/clip.h | 1 - tools/mtmd/mtmd.cpp | 6 ------ 3 files changed, 8 deletions(-) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 4b75b23887a3..b104f373618b 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -488,7 +488,6 @@ struct clip_image_u8 { std::vector buf; int nx = 0; int ny = 0; - // note: we don't need nt here because preprocessor always expect one single image size_t n_pixels() const { return (size_t) nx * (size_t) ny; diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index 7387b7b87cfc..18c7a1d1a7c4 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -107,7 +107,6 @@ bool clip_is_llava(const struct clip_ctx * ctx); bool clip_has_vision_encoder(const struct clip_ctx * ctx); bool clip_has_audio_encoder(const struct clip_ctx * ctx); -// return the max number of model's cgraph image->nt (temporal dimension) as input int clip_model_n_batch_max(const struct clip_ctx * ctx); std::map clip_get_mem_usage(const struct clip_ctx * ctx); diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index a697dcabfe57..c93fb1e0a4a4 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -27,9 +27,6 @@ // for still image data, layout is RGBRGBRGB... // length of data must be nx * ny * 3 bytes // -// for sequence of images (i.e. video): data is nt sequential RGB frames, each nx * ny * 3 bytes -// length of data must be nt * nx * ny * 3 bytes -// // for audio bitmap: nx = sample count, ny = 1, layout is F32 F32 F32 ... // length of data must be nx * sizeof(float) bytes struct mtmd_bitmap { @@ -1306,7 +1303,6 @@ int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens) || proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2 || proj_type == PROJECTOR_TYPE_GRANITE4_VISION) { // TODO @ngxson : llava does not support batched encoding ; this should be fixed inside clip_image_batch_encode() - // video: each entry is one frame pair, encoded with per-frame attention const auto & entries = image_tokens->batch_f32.entries; // entries may have different token counts // e.g., DeepSeek-OCR-2: 144 per tile views, 257 for the global view @@ -1618,8 +1614,6 @@ const char * mtmd_image_tokens_get_id(const mtmd_image_tokens * image_tokens) { llama_pos mtmd_image_tokens_get_n_pos(const mtmd_image_tokens * image_tokens) { switch (image_tokens->pos) { case MTMD_POS_TYPE_MROPE: - // for M-RoPE, n_pos = max(t, h, w) - // t is omitted as we don't support batching return std::max(image_tokens->nx, image_tokens->ny); case MTMD_POS_TYPE_NORMAL: return image_tokens->n_tokens();