Repository: ggerganov/llama.cpp
File: tools/mtmd/clip.cpp (C++)

Code before:
        GGML_ASSERT(model.class_embedding == nullptr); // No CLS token

        ggml_tensor * inp = build_inp();

        ggml_tensor * learned_pos_embd = model.position_embeddings;

        ggml_tensor * cur = build_vit(
                                inp, n_patches,
                                NORM_TYPE_NORMAL,
                                hparams.ffn_op,
                                learned_pos_embd,
                                nullptr);

        cur = build_ffn(cur,
            model.mm_0_w, model.mm_0_b,
            nullptr, nullptr,
            model.mm_1_w, model.mm_1_b,
            hparams.ffn_op,
            -1);
        cb(cur, "aligner_1", -1);

        // build the graph
        ggml_build_forward_expand(gf, cur);

        return gf;
    }

    // whisper encoder with custom projector
    ggml_cgraph * build_whisper_enc() {
        const int n_frames = img.nx;
        const int n_pos    = n_frames / 2;
        GGML_ASSERT(model.position_embeddings->ne[1] >= n_pos);

        ggml_tensor * inp = build_inp_raw(1);

        // conv1d block
        {
            // convolution + gelu
            ggml_tensor * cur = ggml_conv_1d_ph(ctx0, model.conv1d_1_w, inp, 1, 1);
            cur = ggml_add(ctx0, cur, model.conv1d_1_b);

            cur = ggml_gelu_erf(ctx0, cur);

            cur = ggml_conv_1d_ph(ctx0, model.conv1d_2_w, cur, 2, 1);
            cur = ggml_add(ctx0, cur, model.conv1d_2_b);

            cur = ggml_gelu_erf(ctx0, cur);
            // transpose
            inp = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
            cb(inp, "after_conv1d", -1);
        }

Review comment (suggestion):
I think we can now merge this function with `build_siglip()`, you only need to add another `else if (ctx->proj_type() == ...)`

Code after:
                    embeddings = ggml_concat(ctx0, embeddings, embedding_stack[i], 0);
                }
            }
        }

        // llava projector (also used by granite)
        if (ctx->model.hparams.has_llava_projector) {
            embeddings = ggml_reshape_2d(ctx0, embeddings, embeddings->ne[0], embeddings->ne[1]);

            ggml_tensor * patches = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
            ggml_set_name(patches, "patches");
            ggml_set_input(patches);

            // shape [1, 576, 1024]
            // ne is whcn, ne = [1024, 576, 1, 1]
            embeddings = ggml_get_rows(ctx0, embeddings, patches);

            // print_tensor_info(embeddings, "embeddings");

            // llava projector
            if (ctx->proj_type() == PROJECTOR_TYPE_MLP) {
                embeddings = ggml_mul_mat(ctx0, model.mm_0_w, embeddings);
                embeddings = ggml_add(ctx0, embeddings, model.mm_0_b);

                embeddings = ggml_gelu(ctx0, embeddings);
                if (model.mm_2_w) {
                    embeddings = ggml_mul_mat(ctx0, model.mm_2_w, embeddings);
                    embeddings = ggml_add(ctx0, embeddings, model.mm_2_b);
                }
            }
            else if (ctx->proj_type() == PROJECTOR_TYPE_MLP_NORM) {
                embeddings = ggml_mul_mat(ctx0, model.mm_0_w, embeddings);
                embeddings = ggml_add(ctx0, embeddings, model.mm_0_b);
                // ggml_tensor_printf(embeddings, "mm_0_w",0,true,false);
                // First LayerNorm
                embeddings = ggml_norm(ctx0, embeddings, eps);
                embeddings = ggml_add(ctx0, ggml_mul(ctx0, embeddings, model.mm_1_w),
                                    model.mm_1_b);

                // GELU activation
                embeddings = ggml_gelu(ctx0, embeddings);

                // Second linear layer
                embeddings = ggml_mul_mat(ctx0, model.mm_3_w, embeddings);
                embeddings = ggml_add(ctx0, embeddings, model.mm_3_b);

                // Second LayerNorm
                embeddings = ggml_norm(ctx0, embeddings, eps);
                embeddings = ggml_add(ctx0, ggml_mul(ctx0, embeddings, model.mm_4_w),
                                    model.mm_4_b);
            }