Repository: ggerganov/llama.cpp
File: tools/llama-bench/llama-bench.cpp (C++)

Code before:
    llama_split_mode   split_mode;
    int                main_gpu;
    bool               no_kv_offload;
    bool               flash_attn;
    std::vector<ggml_backend_dev_t> devices;
    std::vector<float> tensor_split;
    std::vector<llama_model_tensor_buft_override> tensor_buft_overrides;
    bool               use_mmap;
    bool               embeddings;
    bool               no_op_offload;

    llama_model_params to_llama_mparams() const {
        llama_model_params mparams = llama_model_default_params();

        mparams.n_gpu_layers = n_gpu_layers;
        if (!devices.empty()) {
            mparams.devices = const_cast<ggml_backend_dev_t *>(devices.data());
        } else if (!rpc_devices.empty()) {
            static std::vector<ggml_backend_dev_t> merged_devices;
            merged_devices.clear();
            merged_devices.insert(merged_devices.end(), rpc_devices.begin(), rpc_devices.end());

            for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
                ggml_backend_dev_t dev = ggml_backend_dev_get(i);
                auto dev_type = ggml_backend_dev_type(dev);
                if (dev_type == GGML_BACKEND_DEVICE_TYPE_CPU || dev_type == GGML_BACKEND_DEVICE_TYPE_ACCEL) {
                    continue;
                }
                if (std::find(merged_devices.begin(), merged_devices.end(), dev) == merged_devices.end()) {
                    merged_devices.push_back(dev);
                }
            }

            merged_devices.push_back(nullptr);
            mparams.devices = merged_devices.data();
        }
        mparams.split_mode   = split_mode;
        mparams.main_gpu     = main_gpu;
        mparams.tensor_split = tensor_split.data();
        mparams.use_mmap     = use_mmap;

        if (n_cpu_moe <= 0) {
            if (tensor_buft_overrides.empty()) {
                mparams.tensor_buft_overrides = nullptr;
            } else {
                GGML_ASSERT(tensor_buft_overrides.back().pattern == nullptr &&
                            "Tensor buffer overrides not terminated with empty pattern");
                mparams.tensor_buft_overrides = tensor_buft_overrides.data();
            }
        } else {
            static std::vector<llama_model_tensor_buft_override> merged;

Review comment (suggestion):
To match the llama.cpp default behavior, IGPU devices should be ignored here (they are ignored when there is a GPU present, and the RPC device counts as a GPU).

Code after:
            }
        } else {
            static std::vector<llama_model_tensor_buft_override> merged;
            static std::vector<std::string> patterns;

            merged.clear();
            patterns.clear();

            auto first = tensor_buft_overrides.begin();
            auto last  = tensor_buft_overrides.end();
            if (first != last && (last - 1)->pattern == nullptr) {
                --last;
            }
            merged.insert(merged.end(), first, last);

            patterns.reserve((size_t) n_cpu_moe);
            merged.reserve(merged.size() + (size_t) n_cpu_moe + 1);

            for (int i = 0; i < n_cpu_moe; ++i) {
                patterns.push_back(llm_ffn_exps_block_regex(i));
                merged.push_back({ patterns.back().c_str(),
                                ggml_backend_cpu_buffer_type() });
            }

            merged.push_back({ nullptr, nullptr });

            mparams.tensor_buft_overrides = merged.data();
        }

        return mparams;
    }

    bool equal_mparams(const cmd_params_instance & other) const {
        return model == other.model && n_gpu_layers == other.n_gpu_layers && n_cpu_moe == other.n_cpu_moe &&
               split_mode == other.split_mode &&
               main_gpu == other.main_gpu && use_mmap == other.use_mmap && tensor_split == other.tensor_split &&
               devices == other.devices &&
               vec_tensor_buft_override_equal(tensor_buft_overrides, other.tensor_buft_overrides);
    }

    llama_context_params to_llama_cparams() const {
        llama_context_params cparams = llama_context_default_params();

        cparams.n_ctx           = n_prompt + n_gen + n_depth;
        cparams.n_batch         = n_batch;
        cparams.n_ubatch        = n_ubatch;
        cparams.type_k          = type_k;
        cparams.type_v          = type_v;
        cparams.offload_kqv     = !no_kv_offload;
        cparams.flash_attn_type = flash_attn ? LLAMA_FLASH_ATTN_TYPE_ENABLED : LLAMA_FLASH_ATTN_TYPE_DISABLED;
        cparams.embeddings      = embeddings;