Repository: ollama/ollama
File: llama/llama.cpp/examples/llava/clip.cpp (C++)

Code before:
            }
        }
    }

    clip_ctx * new_clip = new clip_ctx{};

    // update projector type
    {
        int idx = gguf_find_key(ctx, KEY_PROJ_TYPE);
        if (idx != -1) {
            const std::string proj_type = gguf_get_val_str(ctx, idx);
            new_clip->proj_type = clip_projector_type_from_string(proj_type);
        } else {
            new_clip->proj_type = PROJECTOR_TYPE_MLP;
        }

        if (new_clip->proj_type == PROJECTOR_TYPE_MLP) {
            if (gguf_find_tensor(ctx, format(TN_LLAVA_PROJ, 3, "weight").c_str()) != -1) {
                new_clip->proj_type = PROJECTOR_TYPE_MLP_NORM;
            }
        }
    }

    ggml_backend_t backend = ggml_backend_init_best();
    if (backend == nullptr) {
        LOG_ERR("%s: failed to initialize %s backend\n", __func__, ggml_backend_dev_name(ggml_backend_dev_get(0)));
        clip_free(new_clip);
        gguf_free(ctx);
        return nullptr;
    }
    LOG_INF("%s: using %s backend\n", __func__, ggml_backend_name(backend));
    new_clip->backend = backend;

    // model size and capabilities
    {
        int idx = get_key_idx(ctx, KEY_HAS_TEXT_ENC);
        new_clip->has_text_encoder = gguf_get_val_bool(ctx, idx);

        idx = get_key_idx(ctx, KEY_HAS_VIS_ENC);
        new_clip->has_vision_encoder = gguf_get_val_bool(ctx, idx);

        idx = gguf_find_key(ctx, KEY_HAS_LLAVA_PROJ);
        if (idx != -1) {
            new_clip->has_llava_projector = gguf_get_val_bool(ctx, idx);
        }

        idx = gguf_find_key(ctx, KEY_HAS_MINICPMV_PROJ);
        if (idx != -1) {
            new_clip->has_minicpmv_projector = gguf_get_val_bool(ctx, idx);
        }


Review comment (bug):
might not be safe to call `ggml_backend_dev_get` since one possible failure scenario is lack of devices. `ggml_backend_dev_get` may assert. if you really want to get the backend name, it should check `ggml_backend_dev_count` first.

Code after:
            }
        }
    }

    clip_ctx * new_clip = new clip_ctx{};

    // update projector type
    {
        int idx = gguf_find_key(ctx, KEY_PROJ_TYPE);
        if (idx != -1) {
            const std::string proj_type = gguf_get_val_str(ctx, idx);
            new_clip->proj_type = clip_projector_type_from_string(proj_type);
        } else {
            new_clip->proj_type = PROJECTOR_TYPE_MLP;
        }

        if (new_clip->proj_type == PROJECTOR_TYPE_MLP) {
            if (gguf_find_tensor(ctx, format(TN_LLAVA_PROJ, 3, "weight").c_str()) != -1) {
                new_clip->proj_type = PROJECTOR_TYPE_MLP_NORM;
            }
        }
    }

    ggml_backend_t backend = ggml_backend_init_best();
    if (backend == nullptr) {
        LOG_ERR("%s: failed to initialize backend\n", __func__);
        clip_free(new_clip);
        gguf_free(ctx);
        return nullptr;
    }
    LOG_INF("%s: using %s backend\n", __func__, ggml_backend_name(backend));
    new_clip->backend = backend;

    // model size and capabilities
    {
        int idx = get_key_idx(ctx, KEY_HAS_TEXT_ENC);
        new_clip->has_text_encoder = gguf_get_val_bool(ctx, idx);

        idx = get_key_idx(ctx, KEY_HAS_VIS_ENC);
        new_clip->has_vision_encoder = gguf_get_val_bool(ctx, idx);

        idx = gguf_find_key(ctx, KEY_HAS_LLAVA_PROJ);
        if (idx != -1) {
            new_clip->has_llava_projector = gguf_get_val_bool(ctx, idx);
        }

        idx = gguf_find_key(ctx, KEY_HAS_MINICPMV_PROJ);
        if (idx != -1) {
            new_clip->has_minicpmv_projector = gguf_get_val_bool(ctx, idx);
        }