Repository: mlc-ai/mlc-llm
File: python/mlc_chat/serve/entrypoints/openai_entrypoints.py (Python)

Code before:
        assert role != "tool", "Internal error: tool role."
        conv_template.messages.append((conv_template.roles[role == "assistant"], content))
    conv_template.messages.append((conv_template.roles[1], None))

    # - Get the prompt from template, and encode to token ids.
    # - Check prompt length
    async_engine.record_event(request_id, event="start tokenization")
    prompts = entrypoint_utils.process_prompts(
        conv_template.as_prompt(), async_engine.tokenizer.encode
    )
    async_engine.record_event(request_id, event="finish tokenization")
    assert isinstance(prompts, list) and len(prompts) == 1, "Internal error"
    if conv_template.system_prefix_token_ids is not None:
        prompts[0] = conv_template.system_prefix_token_ids + prompts[0]
    error = entrypoint_utils.check_prompts_length(prompts, async_engine.max_single_sequence_length)
    if error is not None:
        return error
    prompt = prompts[0]

    # Process generation config. Create request id.
    generation_cfg = protocol_utils.get_generation_config(
        request,
        extra_stop_token_ids=conv_template.stop_token_ids,
        extra_stop_str=conv_template.stop_str,
    )
    request_id = f"chatcmpl-{entrypoint_utils.random_uuid()}"

    # Streaming response.
    if request.stream:

        async def completion_stream_generator() -> AsyncGenerator[str, None]:
            assert request.n == 1
            async_engine.record_event(request_id, event="invoke generate")
            async for delta_text, _, finish_reason in async_engine.generate(
                prompt, generation_cfg, request_id
            ):
                if delta_text == "":
                    # Ignore empty delta text -- do not yield.
                    continue

                response = ChatCompletionStreamResponse(
                    id=request_id,
                    choices=[
                        ChatCompletionStreamResponseChoice(
                            finish_reason=finish_reason,
                            delta=ChatCompletionMessage(content=delta_text, role="assistant"),
                        )
                    ],
                    model=request.model,
                    system_fingerprint="",
                )

Review comment (suggestion):
Would you mind removing this line (likely caused by rebase)? `request_id` is already set a few lines above.

Code after:
        assert role != "tool", "Internal error: tool role."
        conv_template.messages.append((conv_template.roles[role == "assistant"], content))
    conv_template.messages.append((conv_template.roles[1], None))

    # - Get the prompt from template, and encode to token ids.
    # - Check prompt length
    async_engine.record_event(request_id, event="start tokenization")
    prompts = entrypoint_utils.process_prompts(
        conv_template.as_prompt(), async_engine.tokenizer.encode
    )
    async_engine.record_event(request_id, event="finish tokenization")
    assert isinstance(prompts, list) and len(prompts) == 1, "Internal error"
    if conv_template.system_prefix_token_ids is not None:
        prompts[0] = conv_template.system_prefix_token_ids + prompts[0]
    error = entrypoint_utils.check_prompts_length(prompts, async_engine.max_single_sequence_length)
    if error is not None:
        return error
    prompt = prompts[0]

    # Process generation config. Create request id.
    generation_cfg = protocol_utils.get_generation_config(
        request,
        extra_stop_token_ids=conv_template.stop_token_ids,
        extra_stop_str=conv_template.stop_str,
    )

    # Streaming response.
    if request.stream:

        async def completion_stream_generator() -> AsyncGenerator[str, None]:
            assert request.n == 1
            async_engine.record_event(request_id, event="invoke generate")
            async for delta_text, _, finish_reason in async_engine.generate(
                prompt, generation_cfg, request_id
            ):
                if delta_text == "":
                    # Ignore empty delta text -- do not yield.
                    continue

                response = ChatCompletionStreamResponse(
                    id=request_id,
                    choices=[
                        ChatCompletionStreamResponseChoice(
                            finish_reason=finish_reason,
                            delta=ChatCompletionMessage(content=delta_text, role="assistant"),
                        )
                    ],
                    model=request.model,
                    system_fingerprint="",
                )
                yield f"data: {response.model_dump_json()}\n\n"