Repository: ggerganov/llama.cpp
File: tools/mtmd/legacy-models/minicpmv-convert-image-encoder-to-gguf.py (Python)

Code before:
    fout.add_int32("clip.minicpmv_version", minicpmv_version)
else:
    fout.add_description("two-tower CLIP model")

if has_vision_encoder:
    # vision_model hparams - use actual config values
    vision_image_size = model_config.get("image_size", 448) if model_config else 448
    vision_patch_size = default_vision_config.get("patch_size", 14)
    vision_hidden_size = default_vision_config.get("hidden_size", 1152)
    vision_intermediate_size = default_vision_config.get("intermediate_size", 4304)
    vision_attention_heads = default_vision_config.get("num_attention_heads", 16)

    fout.add_uint32("clip.vision.image_size", vision_image_size)
    fout.add_uint32("clip.vision.patch_size", vision_patch_size)
    fout.add_uint32(add_key_str(KEY_EMBEDDING_LENGTH, VISION), vision_hidden_size)
    fout.add_uint32(add_key_str(KEY_FEED_FORWARD_LENGTH, VISION), vision_intermediate_size)
    fout.add_uint32("clip.vision.projection_dim", 0)
    fout.add_uint32(add_key_str(KEY_ATTENTION_HEAD_COUNT, VISION), vision_attention_heads)
    fout.add_float32(add_key_str(KEY_ATTENTION_LAYERNORM_EPS, VISION), 1e-6)
    fout.add_uint32(add_key_str(KEY_BLOCK_COUNT, VISION), block_count)

    # Add MiniCPM-V specific parameters
    query_num = model_config.get("query_num", 0) if model_config else 0
    resampler_emb_dim = model_config.get("hidden_size", 0) if model_config else 0
    fout.add_uint32("clip.minicpmv_query_num", query_num)
    fout.add_uint32("clip.minicpmv_projection_dim", resampler_emb_dim)

    if processor is not None:
        image_mean = processor.image_processor.image_mean if args.image_mean is None or args.image_mean == default_image_mean else args.image_mean
        image_std = processor.image_processor.image_std if args.image_std is None or args.image_std == default_image_std else args.image_std
    else:
        image_mean = args.image_mean if args.image_mean is not None else default_image_mean
        image_std = args.image_std if args.image_std is not None else default_image_std
    fout.add_array("clip.vision.image_mean", image_mean)
    fout.add_array("clip.vision.image_std", image_std)

use_gelu = True
fout.add_bool("clip.use_gelu", use_gelu)

def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
    """
    embed_dim: output dimension for each position
    pos: a list of positions to be encoded: size (M,)
    out: (M, D)
    """
    assert embed_dim % 2 == 0
    omega = np.arange(embed_dim // 2, dtype=np.float32)
    omega /= embed_dim / 2.
    omega = 1. / 10000 ** omega  # (D/2,)

    pos = pos.reshape(-1)  # (M,)

Review comment (suggestion):
```suggestion
    fout.add_uint32("clip.minicpmv_query_num", query_num)
```

Code after:
output_dir = args.output_dir
os.makedirs(output_dir, exist_ok=True)
output_prefix = os.path.basename(output_dir).replace("ggml_", "")
fname_out = os.path.join(output_dir, f"{fname_middle}model-{ftype_str[ftype]}.gguf")
fout = GGUFWriter(path=fname_out, arch="clip")

fout.add_bool("clip.has_text_encoder", has_text_encoder)
fout.add_bool("clip.has_vision_encoder", has_vision_encoder)
fout.add_bool("clip.has_minicpmv_projector", has_minicpmv_projector)
fout.add_file_type(ftype)
if args.text_only:
    fout.add_description("text-only CLIP model")
elif args.vision_only and not has_minicpmv_projector:
    fout.add_description("vision-only CLIP model")
elif has_minicpmv_projector:
    fout.add_description("image encoder for MiniCPM-V")
    # add projector type
    fout.add_string("clip.projector_type", "resampler")
    fout.add_int32("clip.minicpmv_version", minicpmv_version)
else:
    fout.add_description("two-tower CLIP model")

if has_vision_encoder:
    # vision_model hparams - use actual config values
    vision_image_size = model_config.get("image_size", 448) if model_config else 448
    vision_patch_size = default_vision_config.get("patch_size", 14)
    vision_hidden_size = default_vision_config.get("hidden_size", 1152)
    vision_intermediate_size = default_vision_config.get("intermediate_size", 4304)
    vision_attention_heads = default_vision_config.get("num_attention_heads", 16)

    fout.add_uint32("clip.vision.image_size", vision_image_size)
    fout.add_uint32("clip.vision.patch_size", vision_patch_size)
    fout.add_uint32(add_key_str(KEY_EMBEDDING_LENGTH, VISION), vision_hidden_size)
    fout.add_uint32(add_key_str(KEY_FEED_FORWARD_LENGTH, VISION), vision_intermediate_size)
    fout.add_uint32("clip.vision.projection_dim", 0)
    fout.add_uint32(add_key_str(KEY_ATTENTION_HEAD_COUNT, VISION), vision_attention_heads)
    fout.add_float32(add_key_str(KEY_ATTENTION_LAYERNORM_EPS, VISION), 1e-6)
    fout.add_uint32(add_key_str(KEY_BLOCK_COUNT, VISION), block_count)

    # Add MiniCPM-V specific parameters
    query_num = model_config.get("query_num", 0) if model_config else 0
    resampler_emb_dim = model_config.get("hidden_size", 0) if model_config else 0
    fout.add_uint32("clip.minicpmv_query_num", query_num)

    if processor is not None:
        image_mean = processor.image_processor.image_mean if args.image_mean is None or args.image_mean == default_image_mean else args.image_mean
        image_std = processor.image_processor.image_std if args.image_std is None or args.image_std == default_image_std else args.image_std
    else:
        image_mean = args.image_mean if args.image_mean is not None else default_image_mean
        image_std = args.image_std if args.image_std is not None else default_image_std
    fout.add_array("clip.vision.image_mean", image_mean)