relace huggingface_hub by vllm

app.py

@@ -1,5 +1,13 @@
 import gradio as gr
-from huggingface_hub import InferenceClient
+from vllm import LLM, SamplingParams
+
+llm = LLM(
+    model="stepfun-ai/Step-Audio-2-mini-Think",  # 修改为你需要的模型
+    trust_remote_code=True,
+    tensor_parallel_size=2,  # 如果有多张GPU，设置并行数量
+    # gpu_memory_utilization=0.9,  # GPU显存利用率
+    max_model_len=8192, 
+)
 
 
 def respond(
@@ -12,32 +20,33 @@ def respond(
     hf_token: gr.OAuthToken,
 ):
     """
-    For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
+    使用 vllm 在本地进行推理
     """
-    client = InferenceClient(token=hf_token.token, model="openai/gpt-oss-20b")
-
+    # 构建对话消息
     messages = [{"role": "system", "content": system_message}]
-
     messages.extend(history)
-
     messages.append({"role": "user", "content": message})
 
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
+    # 设置采样参数
+    sampling_params = SamplingParams(
         temperature=temperature,
         top_p=top_p,
-    ):
-        choices = message.choices
-        token = ""
-        if len(choices) and choices[0].delta.content:
-            token = choices[0].delta.content
+        max_tokens=max_tokens,
+    )
+
+    # 使用 vllm 的 chat 接口进行推理
+    outputs = llm.chat(
+        messages=messages,
+        sampling_params=sampling_params,
+        use_tqdm=False,
+    )
+
+    # 获取生成的文本
+    response = outputs[0].outputs[0].text
 
-        response += token
-        yield response
+    # 模拟流式输出效果（逐字符yield）
+    for i in range(1, len(response) + 1):
+        yield response[:i]
 
 
 """