Repository: hiyouga/LLaMA-Factory
File: src/llamafactory/chat/sglang_engine.py (Python)

Code before:

        if videos is not None:
            mm_input_dict.update({"videos": videos, "vidlens": [len(videos)]})
            if not any(VIDEO_PLACEHOLDER in message["content"] for message in messages):
                messages[0]["content"] = VIDEO_PLACEHOLDER * len(videos) + messages[0]["content"]

        if audios is not None:
            mm_input_dict.update({"audios": audios, "audlens": [len(audios)]})
            if not any(AUDIO_PLACEHOLDER in message["content"] for message in messages):
                messages[0]["content"] = AUDIO_PLACEHOLDER * len(audios) + messages[0]["content"]

        # Process messages with multimodal plugin if needed
        messages = self.template.mm_plugin.process_messages(
            messages, mm_input_dict["images"], mm_input_dict["videos"], mm_input_dict["audios"], self.processor
        )

        # Prepare the input for generation
        paired_messages = messages + [{"role": "assistant", "content": ""}]
        system = system or self.generating_args["default_system"]

        # Convert to a properly formatted prompt for the model
        prompt_ids, _ = self.template.encode_oneturn(self.tokenizer, paired_messages, system, tools)
        prompt_text = self.tokenizer.decode(prompt_ids, skip_special_tokens=True)

        # Extract generation parameters from input_kwargs or use defaults
        temperature = input_kwargs.pop("temperature", self.generating_args["temperature"])
        top_p = input_kwargs.pop("top_p", self.generating_args["top_p"])
        top_k = input_kwargs.pop("top_k", self.generating_args["top_k"])
        input_kwargs.pop("num_return_sequences", 1)  # Still pop it but don't assign to variable
        repetition_penalty = input_kwargs.pop("repetition_penalty", self.generating_args["repetition_penalty"])
        max_new_tokens = input_kwargs.pop("max_new_tokens", self.generating_args.get("max_new_tokens", 2048))
        stop = input_kwargs.pop("stop", None)

        # Prepare sampling parameters for SGLang
        sampling_params = {
            "temperature": temperature if temperature > 0 else 0.01,
            "top_p": top_p if top_p < 1.0 else 0.95,
            "max_new_tokens": max_new_tokens,
        }

        # Add additional parameters
        if top_k > 0:
            sampling_params["top_k"] = top_k
        if repetition_penalty != 1.0:
            sampling_params["repetition_penalty"] = repetition_penalty
        if stop:
            sampling_params["stop"] = [stop] if isinstance(stop, str) else stop

        # Create an async generator to yield results
        async def generate():
            try:

Review comment (suggestion):
Please use
```python
temperature = input_kwargs.pop("temperature", None)
temperature = temperature if temperature is not None else generating_args["temperature"]
```
to handle the situation of `input_kwargs["temperature"] = None`

https://github.com/hiyouga/LLaMA-Factory/blob/0dbce72fb8b03ea2c20809f3334c54d2106ff5c2/src/llamafactory/chat/vllm_engine.py#L166-L181

Code after:
        ]
        return results

    @override
    async def stream_chat(
        self,
        messages: list[dict[str, str]],
        system: Optional[str] = None,
        tools: Optional[str] = None,
        images: Optional[list["ImageInput"]] = None,
        videos: Optional[list["VideoInput"]] = None,
        audios: Optional[list["AudioInput"]] = None,
        **input_kwargs,
    ) -> AsyncGenerator[str, None]:
        generated_text = ""
        generator = await self._generate(messages, system, tools, images, videos, audios, **input_kwargs)
        for result in generator:
            delta_text = result["text"][len(generated_text) :]
            generated_text = result["text"]
            yield delta_text

    @override
    async def get_scores(
        self,
        batch_input: list[str],
        **input_kwargs,
    ) -> list[float]:
        raise NotImplementedError("SGLang engine does not support `get_scores`.")

    def __del__(self):
        r"""Ensure server is cleaned up when object is deleted."""
        self._cleanup_server()
        try:
            atexit.unregister(self._cleanup_server)
        except Exception:
            pass