Repository: ggerganov/llama.cpp
File: tools/mtmd/clip.cpp (C++)

Code before:

        res_imgs->grid_x = inst.grid_size.width;
        res_imgs->grid_y = inst.grid_size.height;
        return true;

    } else if (ctx->proj_type() == PROJECTOR_TYPE_QWEN2VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN25VL) {
        clip_image_u8 resized;
        auto patch_size = params.patch_size * 2;
        auto new_size = image_manipulation::calc_size_preserved_ratio(original_size, patch_size, params.image_size);
        image_manipulation::bicubic_resize(*img, resized, new_size.width, new_size.height);

        clip_image_f32_ptr img_f32(clip_image_f32_init());
        // clip_image_f32_ptr res(clip_image_f32_init());
        normalize_image_u8_to_f32(resized, *img_f32, params.image_mean, params.image_std);
        // res_imgs->data[0] = *res;
        res_imgs->entries.push_back(std::move(img_f32));
        return true;
    } else if (ctx->proj_type() == PROJECTOR_TYPE_IDEFICS3) {
        // The refined size has two steps:
        // 1. Resize w/ aspect-ratio preserving such that the longer side is
        //      the preprocessor longest size
        // 2. Resize w/out preserving aspect ratio such that both sides are
        //      multiples of image_size (always rounding up)
        //
        // CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737
        const float scale = std::min(
            static_cast<float>(params.preproc_image_size) / original_size.width,
            static_cast<float>(params.preproc_image_size) / original_size.height);
        int refined_w = static_cast<int>(original_size.width * scale);
        int refined_h = static_cast<int>(original_size.height * scale);
        refined_w = static_cast<int>(params.image_size * std::ceil(static_cast<float>(refined_w) / params.image_size));
        refined_h = static_cast<int>(params.image_size * std::ceil(static_cast<float>(refined_h) / params.image_size));
        const clip_image_size refined_size{refined_w, refined_h};

        llava_uhd::slice_instructions instructions;
        instructions.overview_size = clip_image_size{params.image_size, params.image_size};
        instructions.refined_size = refined_size;
        instructions.grid_size = clip_image_size{
            static_cast<int>(std::ceil(static_cast<float>(refined_size.width) / params.image_size)),
            static_cast<int>(std::ceil(static_cast<float>(refined_size.height) / params.image_size)),
        };
        for (int y = 0; y < refined_size.height; y += params.image_size) {
            for (int x = 0; x < refined_size.width; x += params.image_size) {
                instructions.slices.push_back(llava_uhd::slice_coordinates{
                    /* x    */x,
                    /* y    */y,
                    /* size */clip_image_size{
                        std::min(params.image_size, refined_size.width - x),
                        std::min(params.image_size, refined_size.height - y)
                    }
                });

Review comment (suggestion):
This logic is _similar_ to `llava_uhd::get_slice_instructions`, but it's different enough that I opted not to put this there for simplicity. I could certainly move it if that would be better though.

Code after:

        res_imgs->grid_x = inst.grid_size.width;
        res_imgs->grid_y = inst.grid_size.height;
        return true;

    } else if (ctx->proj_type() == PROJECTOR_TYPE_QWEN2VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN25VL) {
        clip_image_u8 resized;
        auto patch_size = params.patch_size * 2;
        auto new_size = image_manipulation::calc_size_preserved_ratio(original_size, patch_size, params.image_size);
        image_manipulation::bicubic_resize(*img, resized, new_size.width, new_size.height);

        clip_image_f32_ptr img_f32(clip_image_f32_init());
        // clip_image_f32_ptr res(clip_image_f32_init());
        normalize_image_u8_to_f32(resized, *img_f32, params.image_mean, params.image_std);
        // res_imgs->data[0] = *res;
        res_imgs->entries.push_back(std::move(img_f32));
        return true;
    } else if (ctx->proj_type() == PROJECTOR_TYPE_IDEFICS3) {
        // The refined size has two steps:
        // 1. Resize w/ aspect-ratio preserving such that the longer side is
        //      the preprocessor longest size
        // 2. Resize w/out preserving aspect ratio such that both sides are
        //      multiples of image_size (always rounding up)
        //
        // CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737
        const clip_image_size refined_size = image_manipulation::calc_size_preserved_ratio(
            original_size, params.image_size, params.preproc_image_size);

        llava_uhd::slice_instructions instructions;
        instructions.overview_size = clip_image_size{params.image_size, params.image_size};
        instructions.refined_size = refined_size;
        instructions.grid_size = clip_image_size{
            static_cast<int>(std::ceil(static_cast<float>(refined_size.width) / params.image_size)),
            static_cast<int>(std::ceil(static_cast<float>(refined_size.height) / params.image_size)),
        };
        for (int y = 0; y < refined_size.height; y += params.image_size) {
            for (int x = 0; x < refined_size.width; x += params.image_size) {
                instructions.slices.push_back(llava_uhd::slice_coordinates{
                    /* x    */x,
                    /* y    */y,
                    /* size */clip_image_size{
                        std::min(params.image_size, refined_size.width - x),
                        std::min(params.image_size, refined_size.height - y)
                    }
                });
            }
        }
        auto imgs = llava_uhd::slice_image(img, instructions);

        // cast and normalize to f32
        for (size_t i = 0; i < imgs.size(); ++i) {