Repository: ggerganov/llama.cpp
File: tools/mtmd/clip.cpp (C++)

Code before:
            const int scale_factor = model.hparams.proj_scale_factor;
            GGML_ASSERT(scale_factor > 1);

            const int n_embd = cur->ne[0];
            int width  = img.nx / patch_size;
            int height = img.ny / patch_size;

            // pad width and height to factor
            const int64_t pad_width = CLIP_ALIGN(width, scale_factor) - width;
            const int64_t pad_height = CLIP_ALIGN(height, scale_factor) - height;
            cur = ggml_reshape_3d(ctx0, cur, n_embd, width, height);
            if (pad_width || pad_height) {
                cur     = ggml_pad(ctx0, cur, 0, pad_width, pad_height, 0);
                width  += pad_width;
                height += pad_height;
            }

            // unshuffle h
            cur = ggml_reshape_3d(ctx0, ggml_cont(ctx0, cur), n_embd * scale_factor, width / scale_factor, height);
            cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);

            // unshuffle w
            cur = ggml_reshape_3d(ctx0, ggml_cont(ctx0, cur), n_embd * scale_factor * scale_factor, height / scale_factor, width / scale_factor);
            cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);

            cur = ggml_reshape_2d(ctx0, ggml_cont(ctx0, cur), cur->ne[0], cur->ne[1] * cur->ne[2]);

            // projection
            cur = ggml_norm(ctx0, cur, 1e-5); // default nn.LayerNorm
            cur = ggml_mul(ctx0, cur, model.mm_input_norm_w);
            cur = ggml_add(ctx0, cur, model.mm_input_norm_b);

            cur = ggml_mul_mat(ctx0, model.mm_1_w, cur);
            cur = ggml_add(ctx0, cur, model.mm_1_b);
            cur = ggml_gelu(ctx0, cur);
            cur = ggml_mul_mat(ctx0, model.mm_2_w, cur);
            cur = ggml_add(ctx0, cur, model.mm_2_b);
        } else {
            GGML_ABORT("SigLIP: Unsupported projector type");
        }

        // build the graph
        ggml_build_forward_expand(gf, cur);

        return gf;
    }

    ggml_cgraph * build_pixtral() {
        const int n_merge = hparams.spatial_merge_size;

        // 2D input positions

Review comment (suggestion):
```suggestion
            cur = ggml_reshape_3d(ctx0, cur, n_embd, width, height);
            if (pad_width || pad_height) {
                cur     = ggml_pad(ctx0, cur, 0, pad_width, pad_height, 0);
                width  += pad_width;
                height += pad_height;
            }

            // unshuffle h
            cur = ggml_reshape_3d(ctx0, cur, n_embd * scale_factor, width / scale_factor, height);
            cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3));

            // unshuffle w
            cur = ggml_reshape_3d(ctx0, cur, n_embd * scale_factor * scale_factor, height / scale_factor, width / scale_factor);
            cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3));

            cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]);
```
`ggml_pad` returns a contiguous tensor, so no `ggml_cont` needed, I think maybe the confusion stems from the placement of `ggml_cont`, so moved them to where they really apply.

Code after:
            const int scale_factor = model.hparams.proj_scale_factor;
            GGML_ASSERT(scale_factor > 1);

            const int n_embd = cur->ne[0];
            int width  = img.nx / patch_size;
            int height = img.ny / patch_size;

            // pad width and height to factor
            const int64_t pad_width = CLIP_ALIGN(width, scale_factor) - width;
            const int64_t pad_height = CLIP_ALIGN(height, scale_factor) - height;
            cur = ggml_reshape_3d(ctx0, cur, n_embd, width, height);
            if (pad_width || pad_height) {
                cur     = ggml_pad(ctx0, cur, 0, pad_width, pad_height, 0);
                width  += pad_width;
                height += pad_height;
            }

            // unshuffle h
            cur = ggml_reshape_3d(ctx0, cur, n_embd * scale_factor, width / scale_factor, height);
            cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3));

            // unshuffle w
            cur = ggml_reshape_3d(ctx0, cur, n_embd * scale_factor * scale_factor, height / scale_factor, width / scale_factor);
            cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3));

            cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]);

            // projection
            cur = ggml_norm(ctx0, cur, 1e-5); // default nn.LayerNorm
            cur = ggml_mul(ctx0, cur, model.mm_input_norm_w);
            cur = ggml_add(ctx0, cur, model.mm_input_norm_b);

            cur = ggml_mul_mat(ctx0, model.mm_1_w, cur);
            cur = ggml_add(ctx0, cur, model.mm_1_b);
            cur = ggml_gelu(ctx0, cur);
            cur = ggml_mul_mat(ctx0, model.mm_2_w, cur);
            cur = ggml_add(ctx0, cur, model.mm_2_b);
        } else {
            GGML_ABORT("SigLIP: Unsupported projector type");
        }

        // build the graph
        ggml_build_forward_expand(gf, cur);

        return gf;
    }

    ggml_cgraph * build_pixtral() {
        const int n_merge = hparams.spatial_merge_size;

        // 2D input positions