Repository: ggerganov/llama.cpp
File: tools/server/server.cpp (C++)

Code before:
                    arr.push_back(res->to_json());
                }
                // if single request, return single object instead of array
                res_ok(res, arr.size() == 1 ? arr[0] : arr);
            }

        } else {
            // in streaming mode, the first error must be treated as non-stream response
            // this is to match the OAI API behavior
            // ref: https://github.com/ggml-org/llama.cpp/pull/16486#discussion_r2419657309
            server_task_result_ptr first_result = gen->next(is_connection_closed);
            if (first_result == nullptr) {
                return; // connection is closed
            } else if (first_result->is_error()) {
                res_error(res, first_result->to_json());
                return;
            } else {
                GGML_ASSERT(
                    dynamic_cast<server_task_result_cmpl_partial*>(first_result.get()) != nullptr
                    || dynamic_cast<server_task_result_cmpl_final*>(first_result.get()) != nullptr
                );
            }

            // next responses are streamed
            json first_result_json = first_result->to_json();
            const auto chunked_content_provider = [first_result_json, gen, oaicompat](size_t, httplib::DataSink & sink) mutable -> bool {
                // flush the first result as it's not an error
                if (!first_result_json.empty()) {
                    if (!server_sent_event(sink, first_result_json)) {
                        sink.done();
                        return false; // sending failed, go to on_complete()
                    }
                    first_result_json.clear(); // mark as sent
                }

                // receive subsequent results
                auto result = gen->next([&sink]{ return !sink.is_writable(); });
                if (result == nullptr) {
                    sink.done();
                    return false; // connection is closed, go to on_complete()
                }

                // send the results
                json res_json = result->to_json();
                bool ok = false;
                if (result->is_error()) {
                    ok = server_sent_event(sink, json {{ "error", result->to_json() }});
                    sink.done();
                    return false; // go to on_complete()
                } else {
                    GGML_ASSERT(

Review comment (suggestion):
~~note: in the future, when we separate the HTTP implementation from the current code base, this `chunked_content_provider` callback pattern will disappear.~~

the goal is to make each server endpoint handler itself become a generator, which generate JSON response each time the `next()` function is called

Code after:
                res_ok(res, arr.size() == 1 ? arr[0] : arr);
            }

        } else {
            // in streaming mode, the first error must be treated as non-stream response
            // this is to match the OAI API behavior
            // ref: https://github.com/ggml-org/llama.cpp/pull/16486#discussion_r2419657309
            server_task_result_ptr first_result = rd->next(is_connection_closed);
            if (first_result == nullptr) {
                return; // connection is closed
            } else if (first_result->is_error()) {
                res_error(res, first_result->to_json());
                return;
            } else {
                GGML_ASSERT(
                    dynamic_cast<server_task_result_cmpl_partial*>(first_result.get()) != nullptr
                    || dynamic_cast<server_task_result_cmpl_final*>(first_result.get()) != nullptr
                );
            }

            // next responses are streamed
            json first_result_json = first_result->to_json();
            const auto chunked_content_provider = [first_result_json, rd, oaicompat](size_t, httplib::DataSink & sink) mutable -> bool {
                // flush the first result as it's not an error
                if (!first_result_json.empty()) {
                    if (!server_sent_event(sink, first_result_json)) {
                        sink.done();
                        return false; // sending failed, go to on_complete()
                    }
                    first_result_json.clear(); // mark as sent
                }

                // receive subsequent results
                auto result = rd->next([&sink]{ return !sink.is_writable(); });
                if (result == nullptr) {
                    sink.done();
                    return false; // connection is closed, go to on_complete()
                }

                // send the results
                json res_json = result->to_json();
                bool ok = false;
                if (result->is_error()) {
                    ok = server_sent_event(sink, json {{ "error", result->to_json() }});
                    sink.done();
                    return false; // go to on_complete()
                } else {
                    GGML_ASSERT(
                        dynamic_cast<server_task_result_cmpl_partial*>(result.get()) != nullptr
                        || dynamic_cast<server_task_result_cmpl_final*>(result.get()) != nullptr
                    );