Repository: ggerganov/llama.cpp
File: examples/llava/clip.cpp (C++)

Code before:
    {
        embeddings = ggml_mul_mat(ctx0, model.mm_1_w, embeddings);
        embeddings = ggml_add(ctx0, embeddings, model.mm_1_b);

        embeddings = ggml_gelu(ctx0, embeddings);
        embeddings = ggml_mul_mat(ctx0, model.mm_2_w, embeddings);
        embeddings = ggml_add(ctx0, embeddings, model.mm_2_b);
    }

    // arrangement of the [IMG_BREAK] token
    {
        // not efficient, but works
        // the trick is to view the embeddings as a 3D tensor with shape [hidden_size, n_patches_per_row, n_rows]
        // and then concatenate the [IMG_BREAK] token to the end of each row, aka n_patches_per_row dimension
        // after the concatenation, we have a tensor with shape [hidden_size, n_patches_per_row + 1, n_rows]

        const int n_embd_text     = embeddings->ne[0];
        const int n_tokens_output = num_patches + n_patches_y - 1; // one [IMG_BREAK] per row, except the last row

        ggml_tensor * cur = ggml_reshape_3d(ctx0, embeddings, n_embd_text, n_patches_x, n_patches_y);
        ggml_tensor * tok = ggml_new_tensor_3d(ctx0, embeddings->type, n_embd_text, 1, n_patches_y);
        tok = ggml_scale(ctx0, tok, 0.0); // clear the tensor
        tok = ggml_add(ctx0, tok, model.token_embd_img_break);
        cur = ggml_concat(ctx0, cur, tok, 1);
        embeddings = ggml_view_2d(ctx0, cur,
            n_embd_text, n_tokens_output,
            ggml_row_size(cur->type, n_embd_text), 0);
    }

    // build the graph
    ggml_build_forward_expand(gf, embeddings);

    return gf;
}

static ggml_cgraph * clip_image_build_graph_qwen25vl(clip_ctx * ctx, const clip_image_f32_batch & imgs) {
    const auto & model = ctx->vision_model;
    const auto & hparams = model.hparams;

    const int image_size_width  = imgs.entries[0]->nx;
    const int image_size_height = imgs.entries[0]->ny;

    const bool use_window_attn = hparams.n_wa_pattern > 0;

    const int n_wa_pattern         = hparams.n_wa_pattern;
    const int patch_size           = hparams.patch_size;
    const int num_patches          = ((image_size_width / patch_size) * (image_size_height / patch_size));
    const int patches_w            = image_size_width / patch_size;
    const int patches_h            = image_size_height / patch_size;
    const int num_positions        = num_patches + (model.class_embedding ? 1 : 0);
    const int num_position_ids     = num_positions * 4; // m-rope requires 4 dim per position

Review comment (suggestion):
@ggerganov @slaren just want to double check with you here, I suppose that `ggml_im2col` only care about the shape of `kernel`, not the actual data inside. Is this looks OK for you? (Or maybe there is another way?) Thanks!

Code after:
            cur = ggml_mul_mat(ctx0, model.layers[il].ff_down_w, cur);
        }

        // residual 2
        cur = ggml_add(ctx0, embeddings, cur);

        embeddings = cur;
    }

    // mistral small 3.1 patch merger
    // ref: https://github.com/huggingface/transformers/blob/7a3e208892c06a5e278144eaf38c8599a42f53e7/src/transformers/models/mistral3/modeling_mistral3.py#L67
    if (model.mm_patch_merger_w) {
        GGML_ASSERT(hparams.spatial_merge_size > 0);

        ggml_tensor * cur = embeddings;
        cur = ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, eps), model.mm_input_norm_w);

        // reshape image tokens to 2D grid
        cur = ggml_reshape_3d(ctx0, cur, hidden_size, n_patches_x, n_patches_y);
        cur = ggml_permute(ctx0, cur, 2, 0, 1, 3); // [x, y, hidden_size]
        cur = ggml_cont(ctx0, cur);

        // torch.nn.functional.unfold is just an im2col under the hood
        // we just need a dummy kernel to make it work
        ggml_tensor * kernel = ggml_view_3d(ctx0, cur, n_merge, n_merge, cur->ne[2], 0, 0, 0);
        cur = ggml_im2col(ctx0, kernel, cur, n_merge, n_merge, 0, 0, 1, 1, true, inp->type);

        // project to hidden_size
        cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]);
        cur = ggml_mul_mat(ctx0, model.mm_patch_merger_w, cur);
        embeddings = cur;
    }

    // LlavaMultiModalProjector (always using GELU activation)
    {
        embeddings = ggml_mul_mat(ctx0, model.mm_1_w, embeddings);
        if (model.mm_1_b) {
            embeddings = ggml_add(ctx0, embeddings, model.mm_1_b);
        }

        embeddings = ggml_gelu(ctx0, embeddings);
        embeddings = ggml_mul_mat(ctx0, model.mm_2_w, embeddings);
        if (model.mm_2_b) {
            embeddings = ggml_add(ctx0, embeddings, model.mm_2_b);
        }
    }

    // arrangement of the [IMG_BREAK] token
    {
        // not efficient, but works
        // the trick is to view the embeddings as a 3D tensor with shape [hidden_size, n_patches_per_row, n_rows]