Repository: ggml-org/llama.cpp
Pull request: mtmd : support home-cooked Mistral Small Omni

Support a home-cooked version of Mistral Small which can take **both audio and image** as input

Link to GGUF: https://huggingface.co/ngxson/Home-Cook-Mistral-Small-Omni-24B-2507-GGUF

<img width="300" alt="image" src="https://github.com/user-attachments/assets/e28a86ba-8ca9-421d-abdf-fe2ef1917d50" />

### Try it

```sh
# audio
llama-mtmd-cli \
  -m ../models/mistral_omni_gguf/Home-Cook-Mistral-Small-Omni-2507-Q4_K_M.gguf \
  --mmproj ../models/mistral_omni_gguf/mmproj-model-f16.gguf \
  --audio ../models/i-have-a-dream-30s.mp3  \
  -p "Transcribe this"

# Here is the transcription of the provided text:
#
# "I have a dream that one day every valley shall be exalted, every hill and mountain shall be made low, the rough places will
# be made plain, and the crooked places will be made straight, and the glory of the Lord shall be revealed, and all flesh shall
# see it together. This is our hope. With this faith, we will be able to hew out of the mountain of despair a stone of hope."


# vision
llama-mtmd-cli \
  -m ../models/mistral_omni_gguf/Home-Cook-Mistral-Small-Omni-2507-Q4_K_M.gguf \
  --mmproj ../models/mistral_omni_gguf/mmproj-model-f16.gguf \
  --image ../models/bliss.png  \
  -p "What is this"

# This image depicts a serene and expansive landscape featuring a vast, rolling green field under a clear blue sky dotted with 
# fluffy white clouds. The field appears to be lush and well-maintained, possibly a meadow or pasture. The horizon is marked by 
# a gentle, grassy hill that adds depth and dimension to the scene. The overall atmosphere is tranquil and inviting, 
# suggesting a peaceful countryside setting.
```




Changed files:
filename: tools/mtmd/clip-impl.h
patch: @@ -30,6 +30,7 @@
 #define KEY_LAYER_NORM_EPS      "clip.%s.attention.layer_norm_epsilon"
 
 // vision-specific
+#define KEY_VISION_PROJ_TYPE    "clip.vision.projector_type" // for models with mixed modalities
 #define KEY_IMAGE_SIZE          "clip.vision.image_size"
 #define KEY_PREPROC_IMAGE_SIZE  "clip.vision.preproc_image_size"
 #define KEY_PATCH_SIZE          "clip.vision.patch_size"
@@ -48,6 +49,7 @@
 #define KEY_MINICPMV_QUERY_NUM    "clip.minicpmv_query_num"
 
 // audio-specific
+#define KEY_AUDIO_PROJ_TYPE     "clip.audio.projector_type" // for models with mixed modalities
 #define KEY_A_NUM_MEL_BINS      "clip.audio.num_mel_bins"
 #define KEY_A_PROJ_STACK_FACTOR "clip.audio.projector.stack_factor"
 

filename: tools/mtmd/clip.cpp
patch: @@ -2221,15 +2221,27 @@ struct clip_model_loader {
         // projector type
         std::string proj_type;
         {
+            // default key
             get_string(KEY_PROJ_TYPE, proj_type, false);
-            if (!proj_type.empty()) {
-                model.proj_type = clip_projector_type_from_string(proj_type);
+
+            // for models with mixed modalities
+            if (proj_type.empty()) {
+                if (modality == CLIP_MODALITY_VISION) {
+                    get_string(KEY_VISION_PROJ_TYPE, proj_type, false);
+                } else if (modality == CLIP_MODALITY_AUDIO) {
+                    get_string(KEY_AUDIO_PROJ_TYPE, proj_type, false);
+                } else {
+                    GGML_ABORT("unknown modality");
+                }
             }
+
+            model.proj_type = clip_projector_type_from_string(proj_type);
+
             if (model.proj_type == PROJECTOR_TYPE_UNKNOWN) {
                 throw std::runtime_error(string_format("%s: unknown projector type: %s\n", __func__, proj_type.c_str()));
             }
 
-            // correct arch for multimodal models
+            // correct arch for multimodal models (legacy method)
             if (model.proj_type == PROJECTOR_TYPE_QWEN25O) {
                 model.proj_type = modality == CLIP_MODALITY_VISION
                                     ? PROJECTOR_TYPE_QWEN25VL