Repository: ggerganov/llama.cpp
File: tools/mtmd/clip.cpp (C++)

Code before:
            // see SwiGLU in ultravox_model.py, the second half passed through is silu, not the first half
            cur = ggml_swiglu_swapped(ctx0, cur);

            // mid-norm
            cur = ggml_rms_norm(ctx0, cur, 1e-6);
            cur = ggml_mul(ctx0, cur, model.mm_norm_mid_w);

            // ffn out
            cur = ggml_mul_mat(ctx0, model.mm_2_w, cur);

        } else if (ctx->proj_type() == PROJECTOR_TYPE_QWEN2A) {
            // projector
            cur = ggml_mul_mat(ctx0, model.mm_fc_w, cur);
            cur = ggml_add(ctx0, cur, model.mm_fc_b);

        } else if (ctx->proj_type() == PROJECTOR_TYPE_VOXTRAL) {
            // projector
            cur = ggml_mul_mat(ctx0, model.mm_1_w, cur);
            cur = ggml_gelu_erf(ctx0, cur);
            cur = ggml_mul_mat(ctx0, model.mm_2_w, cur);

        } else if (ctx->proj_type() == PROJECTOR_TYPE_GLMA) {
            cur = ggml_norm(ctx0, cur, hparams.eps);
            cur = ggml_mul(ctx0, cur, model.mm_norm_pre_w);
            cur = ggml_add(ctx0, cur, model.mm_norm_pre_b);
            cur = ggml_reshape_2d(ctx0, cur, cur->ne[0] * 4, cur->ne[1] / 4);
            cur = ggml_mul_mat(ctx0, model.mm_1_w, cur);
            cur = ggml_add(ctx0, cur, model.mm_1_b);
            cur = ggml_gelu_erf(ctx0, cur);
            cur = ggml_mul_mat(ctx0, model.mm_2_w, cur);
            cur = ggml_add(ctx0, cur, model.mm_2_b);
            cur = ggml_concat(ctx0, model.mm_boi, cur, 1);
            cur = ggml_concat(ctx0, cur, model.mm_eoi, 1);
        } else {
            GGML_ABORT("%s: unknown projector type", __func__);
        }

        cb(cur, "projected", -1);

        ggml_build_forward_expand(gf, cur);

        return gf;
    }

    // cogvlm vision encoder
    ggml_cgraph * build_cogvlm() {
        GGML_ASSERT(model.class_embedding != nullptr);
        GGML_ASSERT(model.position_embeddings != nullptr);

        const int n_pos = n_patches + 1; // +1 for [CLS]


Review comment (bug):
this will fail if number of elements is not divisible by 4

instead, you should abstract out the `StackAudioFrames` used by ultravox into a new function, `build_stack()`, and reuse it here

Code after:
            if (ctx_params.warmup) {
                loader.warmup(*ctx_audio);
            }
        }

    } catch (const std::exception & e) {
        LOG_ERR("%s: failed to load model '%s': %s\n", __func__, fname, e.what());

        delete ctx_vision;
        delete ctx_audio;

        return {nullptr, nullptr};
    }

    return {ctx_vision, ctx_audio};
}

struct clip_image_size * clip_image_size_init() {
    struct clip_image_size * load_image_size = new struct clip_image_size();
    load_image_size->width = 448;
    load_image_size->height = 448;
    return load_image_size;
}

struct clip_image_u8 * clip_image_u8_init() {
    return new clip_image_u8();
}

struct clip_image_f32 * clip_image_f32_init() {
    return new clip_image_f32();
}

struct clip_image_f32_batch * clip_image_f32_batch_init() {
    return new clip_image_f32_batch();
}

unsigned char * clip_image_u8_get_data(struct clip_image_u8 * img, uint32_t * nx, uint32_t * ny) {
    if (nx) *nx = img->nx;
    if (ny) *ny = img->ny;
    return img->buf.data();
}

void clip_image_size_free(struct clip_image_size * load_image_size) {
    if (load_image_size == nullptr) {
        return;
    }
    delete load_image_size;
}
void clip_image_u8_free(struct clip_image_u8  * img) { delete img; }
void clip_image_f32_free(struct clip_image_f32 * img) { delete img; }
void clip_image_u8_batch_free(struct clip_image_u8_batch * batch) { delete batch; }