Repository: ggerganov/llama.cpp
File: tools/server/server.cpp (C++)

Code before:
                    }
                    res_ok(res, arr);
                }
            }, [&](const json & error_data) {
                res_error(res, error_data);
            }, is_connection_closed);

            ctx_server.queue_results.remove_waiting_task_ids(task_ids);
        } else {
            const auto chunked_content_provider = [task_ids, &ctx_server, oaicompat](size_t, httplib::DataSink & sink) {
                ctx_server.receive_cmpl_results_stream(task_ids, [&](server_task_result_ptr & result) -> bool {
                    json res_json = result->to_json();
                    if (res_json.is_array()) {
                        for (const auto & res : res_json) {
                            if (!server_sent_event(sink, res)) {
                                // sending failed (HTTP connection closed), cancel the generation
                                return false;
                            }
                        }
                        return true;
                    } else {
                        return server_sent_event(sink, res_json);
                    }
                }, [&](const json & error_data) {
                    server_sent_event(sink, json{{"error", error_data}});
                }, [&sink]() {
                    // note: do not use req.is_connection_closed here because req is already destroyed
                    return !sink.is_writable();
                });
                if (oaicompat != OAICOMPAT_TYPE_NONE) {
                    static const std::string ev_done = "data: [DONE]\n\n";
                    sink.write(ev_done.data(), ev_done.size());
                }
                sink.done();
                return false;
            };

            auto on_complete = [task_ids, &ctx_server] (bool) {
                ctx_server.queue_results.remove_waiting_task_ids(task_ids);
            };

            res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);
        }
    };

    const auto handle_completions = [&handle_completions_impl](const httplib::Request & req, httplib::Response & res) {
        json data = json::parse(req.body);
        std::vector<raw_buffer> files; // dummy
        handle_completions_impl(
            SERVER_TASK_TYPE_COMPLETION,
            data,

Review comment (suggestion):
The usage of `first_result_json` here looks suspicious. It seems like you assume that it's value would be preserved across calls of the provider, but it's not the case because it's captured by value. The `first_result_json.clear();` has no purpose in this case.

Nvm: the callback is `mutable`.

Code after:
            // in streaming mode, the first error must be treated as non-stream response
            // this is to match the OAI API behavior
            // ref: https://github.com/ggml-org/llama.cpp/pull/16486#discussion_r2419657309
            server_task_result_ptr first_result = rd->next(is_connection_closed);
            if (first_result == nullptr) {
                return; // connection is closed
            } else if (first_result->is_error()) {
                res_error(res, first_result->to_json());
                return;
            } else {
                GGML_ASSERT(
                    dynamic_cast<server_task_result_cmpl_partial*>(first_result.get()) != nullptr
                    || dynamic_cast<server_task_result_cmpl_final*>(first_result.get()) != nullptr
                );
            }

            // next responses are streamed
            json first_result_json = first_result->to_json();
            const auto chunked_content_provider = [first_result_json, rd, oaicompat](size_t, httplib::DataSink & sink) mutable -> bool {
                // flush the first result as it's not an error
                if (!first_result_json.empty()) {
                    if (!server_sent_event(sink, first_result_json)) {
                        sink.done();
                        return false; // sending failed, go to on_complete()
                    }
                    first_result_json.clear(); // mark as sent
                }

                // receive subsequent results
                auto result = rd->next([&sink]{ return !sink.is_writable(); });
                if (result == nullptr) {
                    sink.done();
                    return false; // connection is closed, go to on_complete()
                }

                // send the results
                json res_json = result->to_json();
                bool ok = false;
                if (result->is_error()) {
                    ok = server_sent_event(sink, json {{ "error", result->to_json() }});
                    sink.done();
                    return false; // go to on_complete()
                } else {
                    GGML_ASSERT(
                        dynamic_cast<server_task_result_cmpl_partial*>(result.get()) != nullptr
                        || dynamic_cast<server_task_result_cmpl_final*>(result.get()) != nullptr
                    );
                    ok = server_sent_event(sink, res_json);
                }

                if (!ok) {