Repository: ggerganov/llama.cpp
File: tools/server/server.cpp (C++)

Code before:
        res->status = 500;
        try {
            json error_data = format_error_response(message, error);
            res->status = json_value(error_data, "code", 500);
            res->data = safe_json_to_str({{ "error", error_data }});
            SRV_WRN("got exception: %s\n", res->data.c_str());
        } catch (const std::exception & e) {
            SRV_ERR("got another exception: %s | while handling exception: %s\n", e.what(), message.c_str());
            res->data = "Internal Server Error";
        }
        return res;
    };
}

int main(int argc, char ** argv, char ** envp) {
    // own arguments required by this example
    common_params params;

    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) {
        return 1;
    }

    json webui_settings = json::object();

    // Env var takes full precedence
    const char * env = std::getenv("LLAMA_WEBUI_CONFIG");
    if (env != nullptr) {
        try {
            webui_settings = json::parse(env);
        } catch (const std::exception & e) {
            LOG_ERR("%s: failed to parse LLAMA_WEBUI_CONFIG: %s\n", __func__, e.what());
            return 1;
        }
    } else if (!params.webui_config_file.empty()) {
        std::ifstream file(params.webui_config_file);
        if (file.is_open()) {
            try {
                webui_settings = json::parse(file);
            } catch (const std::exception & e) {
                LOG_ERR("%s: failed to parse webui config file: %s\n", __func__, e.what());
                return 1;
            }
        } else {
            LOG_WRN("%s: webui config file not found: %s\n", __func__, params.webui_config_file.c_str());
        }
    }

    params.webui_config_json = webui_settings.dump();

    // TODO: should we have a separate n_parallel parameter for the server?
    //       https://github.com/ggml-org/llama.cpp/pull/16736#discussion_r2483763177

Review comment (suggestion):
IMO we should not have this `std::getenv` out of no where as it won't show up in the auto-generated docs. Also, our convention in the `arg.cpp` is to allow arg to take precedence over env (written in the docs)

Either convert it to an arg like `--chat-template` / `--chat-template-file` or to remove it altogether.

Ideally we don't need to add anything into `server.cpp`, I prefer to keep the `main()` as simple as possible

Code after:
            res->data = safe_json_to_str({{ "error", error_data }});
            SRV_WRN("got exception: %s\n", res->data.c_str());
        } catch (const std::exception & e) {
            SRV_ERR("got another exception: %s | while handling exception: %s\n", e.what(), message.c_str());
            res->data = "Internal Server Error";
        }
        return res;
    };
}

int main(int argc, char ** argv, char ** envp) {
    // own arguments required by this example
    common_params params;

    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) {
        return 1;
    }

    // validate batch size for embeddings
    // embeddings require all tokens to be processed in a single ubatch
    // see https://github.com/ggml-org/llama.cpp/issues/12836
    if (params.embedding && params.n_batch > params.n_ubatch) {
        LOG_WRN("%s: embeddings enabled with n_batch (%d) > n_ubatch (%d)\n", __func__, params.n_batch, params.n_ubatch);
        LOG_WRN("%s: setting n_batch = n_ubatch = %d to avoid assertion failure\n", __func__, params.n_ubatch);
        params.n_batch = params.n_ubatch;
    }

    if (params.n_parallel < 0) {
        LOG_INF("%s: n_parallel is set to auto, using n_parallel = 4 and kv_unified = true\n", __func__);

        params.n_parallel = 4;
        params.kv_unified = true;
    }

    // for consistency between server router mode and single-model mode, we set the same model name as alias
    if (params.model_alias.empty() && !params.model.name.empty()) {
        params.model_alias = params.model.name;
    }

    common_init();

    // struct that contains llama context and inference
    server_context ctx_server;

    llama_backend_init();
    llama_numa_init(params.numa);

    LOG_INF("system info: n_threads = %d, n_threads_batch = %d, total_threads = %d\n", params.cpuparams.n_threads, params.cpuparams_batch.n_threads, std::thread::hardware_concurrency());
    LOG_INF("\n");
    LOG_INF("%s\n", common_params_get_system_info(params).c_str());
    LOG_INF("\n");