Repository: mlc-ai/mlc-llm
File: python/mlc_chat/serve/entrypoints/openai_entrypoints.py (Python)

Code before:
    async_engine.record_event(request_id, event="invoke generate")
    async for delta_text, num_delta_tokens, finish_reason in async_engine.generate(
        prompt, generation_cfg, request_id
    ):
        if await raw_request.is_disconnected():
            # In non-streaming cases, the engine will not be notified
            # when the request is disconnected.
            # Therefore, we check if it is disconnected each time,
            # and abort the request from engine if so.
            await async_engine.abort(request_id)
            return entrypoint_utils.create_error_response(
                HTTPStatus.BAD_REQUEST, message="The request has disconnected"
            )
        output_text += delta_text
        num_completion_tokens += num_delta_tokens
    assert finish_reason is not None

    async_engine.record_event(request_id, event="finish")

    if conv_template.use_function_calling:
        try:
            fn_json_list = convert_function_str_to_json(output_text)
        except (SyntaxError, ValueError):
            return entrypoint_utils.create_error_response(
                HTTPStatus.BAD_GATEWAY, message="Got an invalid function call output from model"
            )
        tool_calls = [
            ChatToolCall(
                type="function",
                function=ChatFunctionCall(
                    name=fn_json_obj["name"], arguments=fn_json_obj["arguments"]
                ),
            )
            for fn_json_obj in fn_json_list
            if fn_json_obj is not None
        ]
        if len(tool_calls) == 0:
            return entrypoint_utils.create_error_response(
                HTTPStatus.BAD_GATEWAY, message="Got an invalid function call output from model"
            )

        finish_reason = "tool_calls"

    message = (
        ChatCompletionMessage(role="assistant", content=output_text)
        if not conv_template.use_function_calling
        else ChatCompletionMessage(role="assistant", content=None, tool_calls=tool_calls)
    )

    return ChatCompletionResponse(
        id=request_id,

Review comment (bug):
I'm wondering how often does the parsing fail? From the perspective of user experience, I'm thinking about say returning `output_text` as a normal assistant text when it cannot be parsed as a function call. Does this make sense?

Code after:
    async_engine.record_event(request_id, event="invoke generate")
    async for delta_text, num_delta_tokens, finish_reason in async_engine.generate(
        prompt, generation_cfg, request_id
    ):
        if await raw_request.is_disconnected():
            # In non-streaming cases, the engine will not be notified
            # when the request is disconnected.
            # Therefore, we check if it is disconnected each time,
            # and abort the request from engine if so.
            await async_engine.abort(request_id)
            return entrypoint_utils.create_error_response(
                HTTPStatus.BAD_REQUEST, message="The request has disconnected"
            )
        output_text += delta_text
        num_completion_tokens += num_delta_tokens
    assert finish_reason is not None

    async_engine.record_event(request_id, event="finish")

    if conv_template.use_function_calling:
        try:
            fn_json_list = convert_function_str_to_json(output_text)
        except (SyntaxError, ValueError):
            output_text = "Got an invalid function call output from model"
            finish_reason = "error"
        else:
            tool_calls = [
                ChatToolCall(
                    type="function",
                    function=ChatFunctionCall(
                        name=fn_json_obj["name"], arguments=fn_json_obj["arguments"]
                    ),
                )
                for fn_json_obj in fn_json_list
                if fn_json_obj is not None
            ]
            if len(tool_calls) == 0:
                output_text = "Got an invalid function call output from model"
                finish_reason = "error"
            else:
                finish_reason = "tool_calls"

    message = (
        ChatCompletionMessage(role="assistant", content=output_text)
        if (not conv_template.use_function_calling or finish_reason == "error")
        else ChatCompletionMessage(role="assistant", content=None, tool_calls=tool_calls)
    )

    return ChatCompletionResponse(
        id=request_id,
        choices=[