Repository: ggerganov/llama.cpp
File: tools/mtmd/clip.cpp (C++)

Code before:

            // doing a pool2d to reduce the number of output tokens
            cur = ggml_pool_2d(ctx0, cur, GGML_OP_POOL_AVG, kernel_size, kernel_size, kernel_size, kernel_size, 0, 0);
            cur = ggml_reshape_3d(ctx0, cur, cur->ne[0] * cur->ne[0], n_embd, batch_size);
            cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));

            // apply norm before projection
            cur = ggml_rms_norm(ctx0, cur, eps);
            cur = ggml_mul(ctx0, cur, model.mm_soft_emb_norm_w);

            // apply projection
            cur = ggml_mul_mat(ctx0,
                ggml_cont(ctx0, ggml_transpose(ctx0, model.mm_input_proj_w)),
                cur);

        } else if (ctx->proj_type() == PROJECTOR_TYPE_IDEFICS3) {
            // pixel_shuffle
            // https://github.com/huggingface/transformers/blob/0a950e0bbe1ed58d5401a6b547af19f15f0c195e/src/transformers/models/idefics3/modeling_idefics3.py#L578
            const int scale_factor = model.hparams.proj_scale_factor;
            cur = build_pixel_shuffle(cur, scale_factor);
            cur = ggml_mul_mat(ctx0, model.projection, cur);

        } else if (ctx->proj_type() == PROJECTOR_TYPE_LFM2) {
            // pixel unshuffle block
            const int scale_factor = model.hparams.proj_scale_factor;
            cur = build_pixel_shuffle(cur, scale_factor);

            // projection
            cur = ggml_norm(ctx0, cur, 1e-5); // default nn.LayerNorm
            cur = ggml_mul(ctx0, cur, model.mm_input_norm_w);
            cur = ggml_add(ctx0, cur, model.mm_input_norm_b);

            cur = ggml_mul_mat(ctx0, model.mm_1_w, cur);
            cur = ggml_add(ctx0, cur, model.mm_1_b);
            cur = ggml_gelu(ctx0, cur);
            cur = ggml_mul_mat(ctx0, model.mm_2_w, cur);
            cur = ggml_add(ctx0, cur, model.mm_2_b);
        } else {
            GGML_ABORT("SigLIP: Unsupported projector type");
        }

        // build the graph
        ggml_build_forward_expand(gf, cur);

        return gf;
    }

    ggml_cgraph * build_pixtral() {
        const int n_merge = hparams.spatial_merge_size;

        // 2D input positions

Review comment (suggestion):
In the HF checkpoint, the block is called `PixelUnshuffleBlock`. I followed that when I added a comment above `// pixel unshuffle block`. The function name is `build_pixel_shuffle` though.

Should it be `shuffle` or `unshuffle`? :) I'm personally fine with any as soon as we are consistent.

Code after:

            // doing a pool2d to reduce the number of output tokens
            cur = ggml_pool_2d(ctx0, cur, GGML_OP_POOL_AVG, kernel_size, kernel_size, kernel_size, kernel_size, 0, 0);
            cur = ggml_reshape_3d(ctx0, cur, cur->ne[0] * cur->ne[0], n_embd, batch_size);
            cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));

            // apply norm before projection
            cur = ggml_rms_norm(ctx0, cur, eps);
            cur = ggml_mul(ctx0, cur, model.mm_soft_emb_norm_w);

            // apply projection
            cur = ggml_mul_mat(ctx0,
                ggml_cont(ctx0, ggml_transpose(ctx0, model.mm_input_proj_w)),
                cur);

        } else if (ctx->proj_type() == PROJECTOR_TYPE_IDEFICS3) {
            // pixel_shuffle
            // https://github.com/huggingface/transformers/blob/0a950e0bbe1ed58d5401a6b547af19f15f0c195e/src/transformers/models/idefics3/modeling_idefics3.py#L578
            const int scale_factor = model.hparams.proj_scale_factor;
            cur = build_patch_merge_permute(cur, scale_factor);
            cur = ggml_mul_mat(ctx0, model.projection, cur);

        } else if (ctx->proj_type() == PROJECTOR_TYPE_LFM2) {
            // pixel unshuffle block
            const int scale_factor = model.hparams.proj_scale_factor;
            cur = build_patch_merge_permute(cur, scale_factor);

            // projection
            cur = ggml_norm(ctx0, cur, 1e-5); // default nn.LayerNorm
            cur = ggml_mul(ctx0, cur, model.mm_input_norm_w);
            cur = ggml_add(ctx0, cur, model.mm_input_norm_b);

            cur = ggml_mul_mat(ctx0, model.mm_1_w, cur);
            cur = ggml_add(ctx0, cur, model.mm_1_b);
            cur = ggml_gelu(ctx0, cur);
            cur = ggml_mul_mat(ctx0, model.mm_2_w, cur);
            cur = ggml_add(ctx0, cur, model.mm_2_b);
        } else {
            GGML_ABORT("SigLIP: Unsupported projector type");
        }

        // build the graph
        ggml_build_forward_expand(gf, cur);

        return gf;
    }

    ggml_cgraph * build_pixtral() {
        const int n_merge = hparams.spatial_merge_size;

        // 2D input positions