Repository: ggerganov/llama.cpp
File: tools/llama-bench/llama-bench.cpp (C++)

Code before:
        throw std::invalid_argument("failed to find RPC backend");
    }

    using add_rpc_device_fn = ggml_backend_dev_t (*)(const char * endpoint);
    auto * ggml_backend_rpc_add_device_fn = (add_rpc_device_fn) ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_device");
    if (!ggml_backend_rpc_add_device_fn) {
        throw std::invalid_argument("failed to find RPC device add function");
    }

    static std::unordered_set<std::string> registered;
    std::vector<ggml_backend_dev_t> devices;
    for (const auto & server : rpc_servers) {
        ggml_backend_dev_t dev = nullptr;

        std::string name = string_format("RPC[%s]", server.c_str());

        if (registered.find(server) != registered.end()) {
            dev = ggml_backend_dev_by_name(name.c_str());
        }

        if (!dev) {
            dev = ggml_backend_rpc_add_device_fn(server.c_str());
            if (!dev) {
                throw std::invalid_argument(string_format("failed to add RPC device for server '%s'", server.c_str()));
            }
            ggml_backend_device_register(dev);
            registered.insert(server);
        }

        devices.push_back(dev);
    }

    return devices;
}

[[noreturn]] static void print_available_devices_and_exit() {
    std::vector<ggml_backend_dev_t> devices;
    for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
        auto * dev = ggml_backend_dev_get(i);
        auto   ty  = ggml_backend_dev_type(dev);
        if (ty == GGML_BACKEND_DEVICE_TYPE_CPU) {
            continue;
        }
        devices.push_back(dev);
    }

    printf("Available devices:\n");
    if (devices.empty()) {
        printf("  (none)\n");
    }
    for (auto * dev : devices) {

Review comment (suggestion):
I don't think the RPC device needs to be registered here, it will cause llama.cpp to use it automatically when `mparams.devices` is not set.

Ideally, instead of trying to recreate the llama.cpp default device list, which is going to be hard to keep in sync, the RPC devices would be registered before loading the model, and then unregistered before the next model is loaded. This would require adding a `ggml_backend_device_unregister` function.

Code after:
        throw std::invalid_argument("failed to find RPC backend");
    }

    using add_rpc_device_fn = ggml_backend_dev_t (*)(const char * endpoint);
    auto * ggml_backend_rpc_add_device_fn = (add_rpc_device_fn) ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_device");
    if (!ggml_backend_rpc_add_device_fn) {
        throw std::invalid_argument("failed to find RPC device add function");
    }

    static std::unordered_set<std::string> registered;
    std::vector<ggml_backend_dev_t> devices;
    for (const auto & server : rpc_servers) {
        ggml_backend_dev_t dev = nullptr;

        std::string name = string_format("RPC[%s]", server.c_str());

        if (registered.find(server) != registered.end()) {
            dev = ggml_backend_dev_by_name(name.c_str());
        }

        if (!dev) {
            dev = ggml_backend_rpc_add_device_fn(server.c_str());
            if (!dev) {
                throw std::invalid_argument(string_format("failed to add RPC device for server '%s'", server.c_str()));
            }
            ggml_backend_device_register(dev);
            registered.insert(server);
        }

        devices.push_back(dev);
    }

    return devices;
}

static std::string devices_to_string(const std::vector<ggml_backend_dev_t> & devices) {
    if (devices.empty()) {
        return "auto";
    }

    if (devices.size() == 1 && devices[0] == nullptr) {
        return "none";
    }

    std::vector<std::string> names;
    for (auto * dev : devices) {
        if (dev == nullptr) {
            break;
        }
        names.push_back(ggml_backend_dev_name(dev));
    }