Repository: ggml-org/llama.cpp
Pull request: server: SSL Support

## Description

This PR adds SSL support to the `examples/server` using the built-in SSL support in the vendored `httlib.h` header library.

## Changes

* Add the `CMake` build option `LLAMA_SERVER_SSL` (default `OFF`) to toggle building with/without SSL support
   * **NOTE**: Building with SSL requires linkage against `libssl`/`libcrypto`, so this adds a dependency on `openssl` and requires version >= 3 (see [note here](https://github.com/yhirose/cpp-httplib?tab=readme-ov-file#ssl-support) on the `cpp-httplib` repo)
* Hide all SSL support behind `#define CPPHTTPLIB_OPENSSL_SUPPORT` following the pattern in `httplib`
* When enabled, add `ssl_key_file` and `ssl_cert_file` to the `server_params` struct
* When enabled, add parsing logic to the command line args for `--ssl-key-file` and `--ssl-cert-file`
    * **NOTE**: `httplib` does not support inline PEM strings, so these must point to files
* Move the `svr` instance variable to be a `unique_ptr<httplib::Server>` and instantiate it as either a `httplib::Server` or `httplib::SslServer` depending on the values of `server_params.ssl_key_file` and `server_params.ssl_cert_file`.
    * **NOTE**: This caused all uses of `svr` to move from `.` to `->`, so touched a lot of lines!

## Testing

I noticed the tests in [examples/server/tests](https://github.com/ggerganov/llama.cpp/tree/master/examples/server/tests), but these all appear to be testing against the python bindings. Since building with SSL support requires additional dependencies, I did not want to add these as dependencies in the python bindings and did not attempt to make the default build for python include SSL support, therefore I did not update these tests.

I performed manual testing using locally-generated SSL files (see my utility script for generating them below). With the generated `server.key.pem` and `server.cert.pem`, I ran the following:

```sh
cd build

# Build with SSL disabled
cmake .. -DLLAMA_SERVER_SSL=OFF
make server

# Boot with SSL: verify invalid arguments
server -m my_model --ssl-key-file server.key.pem --ssl-cert-file serever.cert.pem
# -> Display help, exit code 1

# Build with SSL enabled
cmake .. -DLLAMA_SERVER_SSL=ON
make server

# Boot with SSL: verify log message
server -m my_model --ssl-key-file server.key.pem --ssl-cert-file serever.cert.pem
# {"cert":"server.cert.pem","function":"main","key":"server.key.pem","level":"INFO","line":2644,"msg":"Running with SSL","tid":"0x1d8b71000","timestamp":1709837564}

# Run http call: verify no valid response
curl http://localhost:8080

# Run insecure https call: verify valid response
curl -k https://localhost:8080

# Run https call with signing CA: verify valid response
curl --cacert ca.cert.pem https://localhost:8080

# Boot without SSL: verify log message
server -m my_model
# {"function":"main","level":"INFO","line":2649,"msg":"Running without SSL","tid":"0x1d8b71000","timestamp":1709837582}

# Run http call: verify valid response
curl http://localhost:8080

# Run insecure https call: verify invalid response
curl -k https://localhost:8080
```

<details>
<summary>gen_mtls_test_files.sh</summary>

```sh
#!/usr/bin/env bash

## Config ######################################################################

# Optional additional SANs can be set with SANS
SANS=${SANS:-""}

# CN can be overloaded
CN=${CN:-"foo.bar.com"}

set -euo pipefail

# Set up additional SANs block
IFS=' ' read -r -a sans_arr <<< "$SANS"
extra_sans=""
counter="1"
for san in "${sans_arr[@]}"
do
    counter=$(expr "$counter" "+" "1")
    echo "Adding SAN DNS.$counter [$san]"
    extra_sans="$extra_sans\nDNS.$counter = $san"
done

root_name="ca"
root_key="$root_name.key.pem"
root_crt="$root_name.cert.pem"

server_name="server"
server_key="$server_name.key.pem"
server_crt="$server_name.cert.pem"

client_name="client"
client_key="$client_name.key.pem"
client_crt="$client_name.cert.pem"

common_config='
[req]
default_bits       = 4096
default_keyfile    = server.key.pem
distinguished_name = req_distinguished_name
x509_extensions    = x509_ext
string_mask        = utf8only

[req_distinguished_name]
countryName                 = Country Name (2 letter code)
countryName_default         = US
stateOrProvinceName         = State or Province Name (full name)
stateOrProvinceName_default = Denver
localityName                = Locality Name (eg, city)
localityName_default        = Denver
organizationName            = Organization Name (eg, company)
organizationName_default    = IBM Watson
commonName                  = Common Name (eg, YOUR name)
commonName_max              = 64
'

ca_config="
$common_config

[x509_ext]
subjectKeyIdentifier   = hash
authorityKeyIdentifier = keyid:always, issuer
basicConstraints       = critical, CA:TRUE, pathlen:1
keyUsage               = keyCertSign, cRLSign
"

derived_config="
$common_config

[x509_ext]
subjectKeyIdentifier   = hash
authorityKeyIdentifier = keyid,issuer
basicConstraints       = CA:FALSE
keyUsage               = digitalSignature, keyEncipherment
subjectAltName         = @alt_names

[alt_names]
DNS.1 = localhost
$extra_sans
IP.1  = '127.0.0.1'
"

# use wild card in subject, not all clients accept that, but Java grpc client does
SUBJ="/C=US/ST=Denver/L=Denver/O=ME/OU=YOU/CN=$CN"

# Set the expiration for 10 years
expiration_days=3650

## Root ########################################################################


# Create the root key
echo "[Creating root key]"
openssl genrsa -out $root_key 2048

# create root key and cert
echo "[Creating root cert]"
openssl req \
    -config <(echo -e "$ca_config") \
    -x509 \
    -new \
    -nodes \
    -key $root_key \
    -sha256 \
    -subj "$SUBJ" \
    -out $root_crt

## Server ######################################################################

# create a new server key and certificate signing request
echo "[Creating server key and signing request]"
openssl req -config <(echo -e "$derived_config") -new -nodes -sha256 -keyout $server_key -out $server_name.csr -newkey rsa:2048 -subj "$SUBJ"

# sign the request with our root cert key
echo "[Sign server cert]"
openssl x509 -req -sha256 -extfile <(echo -e "$derived_config") -extensions x509_ext -in $server_name.csr -CA $root_crt -CAkey $root_key -CAcreateserial -out $server_crt -days $expiration_days

# write out server key in pkcs8 format, required by grpc
echo "[Convert server key to pkcs8]"
cp $server_key $server_key.tmp
openssl pkcs8 -topk8 -nocrypt -in $server_key.tmp -out $server_key

## Client ######################################################################

# create a new server key and certificate signing request
echo "[Creating client key and signing request]"
openssl req -config <(echo -e "$derived_config") -new -nodes -sha256 -keyout $client_key -out $client_name.csr -newkey rsa:2048 -subj "$SUBJ"

# sign the request with our root cert key
echo "[Sign client cert]"
openssl x509 -req -sha256 -extfile <(echo -e "$derived_config") -extensions x509_ext -in $client_name.csr -CA $root_crt -CAkey $root_key -CAcreateserial -out $client_crt -days $expiration_days

# write out client key in pkcs8 format, required by grpc
echo "[Convert client key to pkcs8]"
cp $client_key $client_key.tmp
openssl pkcs8 -topk8 -nocrypt -in $client_key.tmp -out $client_key

# Clean up
rm *.tmp
rm *.csr
rm *.srl
```

</details>

## Open Questions

* As discussed in [this issue](https://github.com/ggerganov/llama.cpp/issues/2194), for full security support, an `nginx` proxy is a more robust way to go. That said, having basic SSL support in the native server is a much simpler deployment topology, so may be worth the maintenance effort.
* The `--ssl-key-file` and `--ssl-cert-file` arguments only work if both are specified. Currently, if either is not specified, they are both ignored. Should they instead cause an error if one is specified without the other?
* Since `cpp-httlib` does not support mutual TLS, this change also does not support it. Is it worth having non-mutual support without going all the way to mTLS?


Changed files:
filename: Makefile
patch: @@ -201,6 +201,10 @@ ifdef LLAMA_SERVER_VERBOSE
 	MK_CPPFLAGS += -DSERVER_VERBOSE=$(LLAMA_SERVER_VERBOSE)
 endif
 
+ifdef LLAMA_SERVER_SSL
+	MK_CPPFLAGS += -DCPPHTTPLIB_OPENSSL_SUPPORT
+	MK_LDFLAGS += -lssl -lcrypto
+endif
 
 ifdef LLAMA_CODE_COVERAGE
 	MK_CXXFLAGS += -fprofile-arcs -ftest-coverage -dumpbase ''

filename: examples/server/CMakeLists.txt
patch: @@ -1,12 +1,18 @@
 set(TARGET server)
 option(LLAMA_SERVER_VERBOSE "Build verbose logging option for Server" ON)
+option(LLAMA_SERVER_SSL "Build SSL support for the server" OFF)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 add_executable(${TARGET} server.cpp utils.hpp json.hpp httplib.h)
 install(TARGETS ${TARGET} RUNTIME)
 target_compile_definitions(${TARGET} PRIVATE
     SERVER_VERBOSE=$<BOOL:${LLAMA_SERVER_VERBOSE}>
 )
 target_link_libraries(${TARGET} PRIVATE common ${CMAKE_THREAD_LIBS_INIT})
+if (LLAMA_SERVER_SSL)
+    find_package(OpenSSL REQUIRED)
+    target_link_libraries(${TARGET} PRIVATE OpenSSL::SSL OpenSSL::Crypto)
+    target_compile_definitions(${TARGET} PRIVATE CPPHTTPLIB_OPENSSL_SUPPORT)
+endif()
 if (WIN32)
     TARGET_LINK_LIBRARIES(${TARGET} PRIVATE ws2_32)
 endif()

filename: examples/server/README.md
patch: @@ -59,6 +59,10 @@ see https://github.com/ggerganov/llama.cpp/issues/1437
 - `--log-disable`: Output logs to stdout only, default: enabled.
 - `--log-format FORMAT`: Define the log output to FORMAT: json or text (default: json)
 
+**If compiled with `LLAMA_SERVER_SSL=ON`**
+- `--ssl-key-file FNAME`: path to file a PEM-encoded SSL private key
+- `--ssl-cert-file FNAME`: path to file a PEM-encoded SSL certificate
+
 ## Build
 
 server is build alongside everything else from the root of the project
@@ -75,6 +79,28 @@ server is build alongside everything else from the root of the project
   cmake --build . --config Release
   ```
 
+## Build with SSL
+
+server can also be built with SSL support using OpenSSL 3
+
+- Using `make`:
+
+  ```bash
+  # NOTE: For non-system openssl, use the following:
+  #   CXXFLAGS="-I /path/to/openssl/include"
+  #   LDFLAGS="-L /path/to/openssl/lib"
+  make LLAMA_SERVER_SSL=true server
+  ```
+
+- Using `CMake`:
+
+  ```bash
+  mkdir build
+  cd build
+  cmake .. -DLLAMA_SERVER_SSL=ON
+  make server
+  ```
+
 ## Quick Start
 
 To get started right away, run the following command, making sure to use the correct path for the model you have:

filename: examples/server/server.cpp
patch: @@ -27,6 +27,7 @@
 #include <mutex>
 #include <thread>
 #include <signal.h>
+#include <memory>
 
 using json = nlohmann::json;
 
@@ -118,6 +119,11 @@ struct server_params {
 
     std::vector<std::string> api_keys;
 
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+    std::string ssl_key_file = "";
+    std::string ssl_cert_file = "";
+#endif
+
     bool slots_endpoint   = true;
     bool metrics_endpoint = false;
 };
@@ -2142,6 +2148,10 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co
     printf("  --path PUBLIC_PATH        path from which to serve static files (default %s)\n", sparams.public_path.c_str());
     printf("  --api-key API_KEY         optional api key to enhance server security. If set, requests must include this key for access.\n");
     printf("  --api-key-file FNAME      path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n");
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+    printf("  --ssl-key-file FNAME      path to file a PEM-encoded SSL private key\n");
+    printf("  --ssl-cert-file FNAME     path to file a PEM-encoded SSL certificate\n");
+#endif
     printf("  -to N, --timeout N        server read/write timeout in seconds (default: %d)\n", sparams.read_timeout);
     printf("  --embeddings              enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled");
     printf("  -np N, --parallel N       number of slots for process requests (default: %d)\n", params.n_parallel);
@@ -2220,7 +2230,24 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams,
                }
             }
             key_file.close();
-        } else if (arg == "--timeout" || arg == "-to") {
+
+        }
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+        else if (arg == "--ssl-key-file") {
+            if (++i >= argc) {
+                invalid_param = true;
+                break;
+            }
+            sparams.ssl_key_file = argv[i];
+        } else if (arg == "--ssl-cert-file") {
+            if (++i >= argc) {
+                invalid_param = true;
+                break;
+            }
+            sparams.ssl_cert_file = argv[i];
+        }
+#endif
+        else if (arg == "--timeout" || arg == "-to") {
             if (++i >= argc) {
                 invalid_param = true;
                 break;
@@ -2658,21 +2685,34 @@ int main(int argc, char ** argv) {
         {"system_info",     llama_print_system_info()},
     });
 
-    httplib::Server svr;
+    std::unique_ptr<httplib::Server> svr;
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+    if (sparams.ssl_key_file != "" && sparams.ssl_cert_file != "") {
+        LOG_INFO("Running with SSL", {{"key", sparams.ssl_key_file}, {"cert", sparams.ssl_cert_file}});
+        svr.reset(
+            new httplib::SSLServer(sparams.ssl_cert_file.c_str(), sparams.ssl_key_file.c_str())
+        );
+    } else {
+        LOG_INFO("Running without SSL", {});
+        svr.reset(new httplib::Server());
+    }
+#else
+    svr.reset(new httplib::Server());
+#endif
 
     std::atomic<server_state> state{SERVER_STATE_LOADING_MODEL};
 
-    svr.set_default_headers({{"Server", "llama.cpp"}});
+    svr->set_default_headers({{"Server", "llama.cpp"}});
 
     // CORS preflight
-    svr.Options(R"(.*)", [](const httplib::Request & req, httplib::Response & res) {
+    svr->Options(R"(.*)", [](const httplib::Request & req, httplib::Response & res) {
         res.set_header("Access-Control-Allow-Origin",      req.get_header_value("Origin"));
         res.set_header("Access-Control-Allow-Credentials", "true");
         res.set_header("Access-Control-Allow-Methods",     "POST");
         res.set_header("Access-Control-Allow-Headers",     "*");
     });
 
-    svr.Get("/health", [&](const httplib::Request & req, httplib::Response & res) {
+    svr->Get("/health", [&](const httplib::Request & req, httplib::Response & res) {
         server_state current_state = state.load();
         switch (current_state) {
             case SERVER_STATE_READY:
@@ -2728,7 +2768,7 @@ int main(int argc, char ** argv) {
     });
 
     if (sparams.slots_endpoint) {
-        svr.Get("/slots", [&](const httplib::Request &, httplib::Response & res) {
+        svr->Get("/slots", [&](const httplib::Request &, httplib::Response & res) {
             // request slots data using task queue
             server_task task;
             task.id = ctx_server.queue_tasks.get_new_id();
@@ -2749,7 +2789,7 @@ int main(int argc, char ** argv) {
     }
 
     if (sparams.metrics_endpoint) {
-        svr.Get("/metrics", [&](const httplib::Request &, httplib::Response & res) {
+        svr->Get("/metrics", [&](const httplib::Request &, httplib::Response & res) {
             // request slots data using task queue
             server_task task;
             task.id = ctx_server.queue_tasks.get_new_id();
@@ -2846,9 +2886,9 @@ int main(int argc, char ** argv) {
         });
     }
 
-    svr.set_logger(log_server_request);
+    svr->set_logger(log_server_request);
 
-    svr.set_exception_handler([](const httplib::Request &, httplib::Response & res, std::exception_ptr ep) {
+    svr->set_exception_handler([](const httplib::Request &, httplib::Response & res, std::exception_ptr ep) {
         const char fmt[] = "500 Internal Server Error\n%s";
 
         char buf[BUFSIZ];
@@ -2864,7 +2904,7 @@ int main(int argc, char ** argv) {
         res.status = 500;
     });
 
-    svr.set_error_handler([](const httplib::Request &, httplib::Response & res) {
+    svr->set_error_handler([](const httplib::Request &, httplib::Response & res) {
         if (res.status == 401) {
             res.set_content("Unauthorized", "text/plain; charset=utf-8");
         }
@@ -2877,16 +2917,16 @@ int main(int argc, char ** argv) {
     });
 
     // set timeouts and change hostname and port
-    svr.set_read_timeout (sparams.read_timeout);
-    svr.set_write_timeout(sparams.write_timeout);
+    svr->set_read_timeout (sparams.read_timeout);
+    svr->set_write_timeout(sparams.write_timeout);
 
-    if (!svr.bind_to_port(sparams.hostname, sparams.port)) {
+    if (!svr->bind_to_port(sparams.hostname, sparams.port)) {
         fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", sparams.hostname.c_str(), sparams.port);
         return 1;
     }
 
     // Set the base directory for serving static files
-    svr.set_base_dir(sparams.public_path);
+    svr->set_base_dir(sparams.public_path);
 
     std::unordered_map<std::string, std::string> log_data;
 
@@ -2947,30 +2987,30 @@ int main(int argc, char ** argv) {
     };
 
     // this is only called if no index.html is found in the public --path
-    svr.Get("/", [](const httplib::Request &, httplib::Response & res) {
+    svr->Get("/", [](const httplib::Request &, httplib::Response & res) {
         res.set_content(reinterpret_cast<const char*>(&index_html), index_html_len, "text/html; charset=utf-8");
         return false;
     });
 
     // this is only called if no index.js is found in the public --path
-    svr.Get("/index.js", [](const httplib::Request &, httplib::Response & res) {
+    svr->Get("/index.js", [](const httplib::Request &, httplib::Response & res) {
         res.set_content(reinterpret_cast<const char *>(&index_js), index_js_len, "text/javascript; charset=utf-8");
         return false;
     });
 
     // this is only called if no index.html is found in the public --path
-    svr.Get("/completion.js", [](const httplib::Request &, httplib::Response & res) {
+    svr->Get("/completion.js", [](const httplib::Request &, httplib::Response & res) {
         res.set_content(reinterpret_cast<const char*>(&completion_js), completion_js_len, "application/javascript; charset=utf-8");
         return false;
     });
 
     // this is only called if no index.html is found in the public --path
-    svr.Get("/json-schema-to-grammar.mjs", [](const httplib::Request &, httplib::Response & res) {
+    svr->Get("/json-schema-to-grammar.mjs", [](const httplib::Request &, httplib::Response & res) {
         res.set_content(reinterpret_cast<const char*>(&json_schema_to_grammar_mjs), json_schema_to_grammar_mjs_len, "application/javascript; charset=utf-8");
         return false;
     });
 
-    svr.Get("/props", [&ctx_server](const httplib::Request & req, httplib::Response & res) {
+    svr->Get("/props", [&ctx_server](const httplib::Request & req, httplib::Response & res) {
         res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
         json data = {
             { "user_name",                   ctx_server.name_user.c_str() },
@@ -3062,11 +3102,11 @@ int main(int argc, char ** argv) {
         }
     };
 
-    svr.Post("/completion", completions); // legacy
-    svr.Post("/completions", completions);
-    svr.Post("/v1/completions", completions);
+    svr->Post("/completion", completions); // legacy
+    svr->Post("/completions", completions);
+    svr->Post("/v1/completions", completions);
 
-    svr.Get("/v1/models", [&params, &model_meta](const httplib::Request & req, httplib::Response & res) {
+    svr->Get("/v1/models", [&params, &model_meta](const httplib::Request & req, httplib::Response & res) {
         res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
 
         json models = {
@@ -3161,10 +3201,10 @@ int main(int argc, char ** argv) {
         }
     };
 
-    svr.Post("/chat/completions",    chat_completions);
-    svr.Post("/v1/chat/completions", chat_completions);
+    svr->Post("/chat/completions",    chat_completions);
+    svr->Post("/v1/chat/completions", chat_completions);
 
-    svr.Post("/infill", [&ctx_server, &validate_api_key](const httplib::Request & req, httplib::Response & res) {
+    svr->Post("/infill", [&ctx_server, &validate_api_key](const httplib::Request & req, httplib::Response & res) {
         res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
         if (!validate_api_key(req, res)) {
             return;
@@ -3228,11 +3268,11 @@ int main(int argc, char ** argv) {
         }
     });
 
-    svr.Options(R"(/.*)", [](const httplib::Request &, httplib::Response & res) {
+    svr->Options(R"(/.*)", [](const httplib::Request &, httplib::Response & res) {
         return res.set_content("", "application/json; charset=utf-8");
     });
 
-    svr.Post("/tokenize", [&ctx_server](const httplib::Request & req, httplib::Response & res) {
+    svr->Post("/tokenize", [&ctx_server](const httplib::Request & req, httplib::Response & res) {
         res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
         const json body = json::parse(req.body);
 
@@ -3244,7 +3284,7 @@ int main(int argc, char ** argv) {
         return res.set_content(data.dump(), "application/json; charset=utf-8");
     });
 
-    svr.Post("/detokenize", [&ctx_server](const httplib::Request & req, httplib::Response & res) {
+    svr->Post("/detokenize", [&ctx_server](const httplib::Request & req, httplib::Response & res) {
         res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
         const json body = json::parse(req.body);
 
@@ -3258,7 +3298,7 @@ int main(int argc, char ** argv) {
         return res.set_content(data.dump(), "application/json; charset=utf-8");
     });
 
-    svr.Post("/embedding", [&params, &ctx_server](const httplib::Request & req, httplib::Response & res) {
+    svr->Post("/embedding", [&params, &ctx_server](const httplib::Request & req, httplib::Response & res) {
         res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
         if (!params.embedding) {
             res.status = 501;
@@ -3289,7 +3329,7 @@ int main(int argc, char ** argv) {
         return res.set_content(result.data.dump(), "application/json; charset=utf-8");
     });
 
-    svr.Post("/v1/embeddings", [&params, &ctx_server](const httplib::Request & req, httplib::Response & res) {
+    svr->Post("/v1/embeddings", [&params, &ctx_server](const httplib::Request & req, httplib::Response & res) {
         res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
         if (!params.embedding) {
             res.status = 501;
@@ -3360,13 +3400,13 @@ int main(int argc, char ** argv) {
         sparams.n_threads_http = std::max(params.n_parallel + 2, (int32_t) std::thread::hardware_concurrency() - 1);
     }
     log_data["n_threads_http"] =  std::to_string(sparams.n_threads_http);
-    svr.new_task_queue = [&sparams] { return new httplib::ThreadPool(sparams.n_threads_http); };
+    svr->new_task_queue = [&sparams] { return new httplib::ThreadPool(sparams.n_threads_http); };
 
     LOG_INFO("HTTP server listening", log_data);
 
     // run the HTTP server in a thread - see comment below
     std::thread t([&]() {
-        if (!svr.listen_after_bind()) {
+        if (!svr->listen_after_bind()) {
             state.store(SERVER_STATE_ERROR);
             return 1;
         }
@@ -3407,7 +3447,7 @@ int main(int argc, char ** argv) {
 
     ctx_server.queue_tasks.start_loop();
 
-    svr.stop();
+    svr->stop();
     t.join();
 
     llama_backend_free();