Repository: xorbitsai/inference

Issue:
部署本地qwen-vl-chat模型调用报错'list' object has no attribute 'lstrip'
```python
response = client.chat.completions.create(
    model="test1",
    messages=[
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "What’s in this image?"},
                {
                    "type": "image_url",
                    "image_url": {
                        "url": "http://i.epochtimes.com/assets/uploads/2020/07/shutterstock_675595789-600x400.jpg",
                    },
                },
            ],
        }
    ],
)
print(response.choices[0])

```

报错信息
```shell
---------------------------------------------------------------------------
InternalServerError                       Traceback (most recent call last)
Cell In[7], line 1
----> 1 response = client.chat.completions.create(
      2     model="qwen-vl-o",
      3     messages=[
      4         {
      5             "role": "user",
      6             "content": [
      7                 {"type": "text", "text": "What’s in this image?"},
      8                 {
      9                     "type": "image_url",
     10                     "image_url": {
     11                         "url": "http://i.epochtimes.com/assets/uploads/2020/07/shutterstock_675595789-600x400.jpg",
     12                     },
     13                 },
     14             ],
     15         }
     16     ],
     17 )
     18 print(response.choices[0])

File /hy-tmp/conda/envs/swift/lib/python3.10/site-packages/openai/_utils/_utils.py:275, in required_args.<locals>.inner.<locals>.wrapper(*args, **kwargs)
    273             msg = f"Missing required argument: {quote(missing[0])}"
    274     raise TypeError(msg)
--> 275 return func(*args, **kwargs)

File /hy-tmp/conda/envs/swift/lib/python3.10/site-packages/openai/resources/chat/completions.py:667, in Completions.create(self, messages, model, frequency_penalty, function_call, functions, logit_bias, logprobs, max_tokens, n, presence_penalty, response_format, seed, stop, stream, temperature, tool_choice, tools, top_logprobs, top_p, user, extra_headers, extra_query, extra_body, timeout)
    615 @required_args(["messages", "model"], ["messages", "model", "stream"])
    616 def create(
    617     self,
   (...)
    665     timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    666 ) -> ChatCompletion | Stream[ChatCompletionChunk]:
--> 667     return self._post(
    668         "/chat/completions",
    669         body=maybe_transform(
    670             {
    671                 "messages": messages,
    672                 "model": model,
    673                 "frequency_penalty": frequency_penalty,
    674                 "function_call": function_call,
    675                 "functions": functions,
    676                 "logit_bias": logit_bias,
    677                 "logprobs": logprobs,
    678                 "max_tokens": max_tokens,
    679                 "n": n,
    680                 "presence_penalty": presence_penalty,
    681                 "response_format": response_format,
    682                 "seed": seed,
    683                 "stop": stop,
    684                 "stream": stream,
    685                 "temperature": temperature,
    686                 "tool_choice": tool_choice,
    687                 "tools": tools,
    688                 "top_logprobs": top_logprobs,
    689                 "top_p": top_p,
    690                 "user": user,
    691             },
    692             completion_create_params.CompletionCreateParams,
    693         ),
    694         options=make_request_options(
    695             extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
    696         ),
    697         cast_to=ChatCompletion,
    698         stream=stream or False,
    699         stream_cls=Stream[ChatCompletionChunk],
    700     )

File /hy-tmp/conda/envs/swift/lib/python3.10/site-packages/openai/_base_client.py:1208, in SyncAPIClient.post(self, path, cast_to, body, options, files, stream, stream_cls)
   1194 def post(
   1195     self,
   1196     path: str,
   (...)
   1203     stream_cls: type[_StreamT] | None = None,
   1204 ) -> ResponseT | _StreamT:
   1205     opts = FinalRequestOptions.construct(
   1206         method="post", url=path, json_data=body, files=to_httpx_files(files), **options
   1207     )
-> 1208     return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))

File /hy-tmp/conda/envs/swift/lib/python3.10/site-packages/openai/_base_client.py:897, in SyncAPIClient.request(self, cast_to, options, remaining_retries, stream, stream_cls)
    888 def request(
    889     self,
    890     cast_to: Type[ResponseT],
   (...)
    895     stream_cls: type[_StreamT] | None = None,
    896 ) -> ResponseT | _StreamT:
--> 897     return self._request(
    898         cast_to=cast_to,
    899         options=options,
    900         stream=stream,
    901         stream_cls=stream_cls,
    902         remaining_retries=remaining_retries,
    903     )

File /hy-tmp/conda/envs/swift/lib/python3.10/site-packages/openai/_base_client.py:973, in SyncAPIClient._request(self, cast_to, options, remaining_retries, stream, stream_cls)
    971 if retries > 0 and self._should_retry(err.response):
    972     err.response.close()
--> 973     return self._retry_request(
    974         options,
    975         cast_to,
    976         retries,
    977         err.response.headers,
    978         stream=stream,
    979         stream_cls=stream_cls,
    980     )
    982 # If the response is streamed then we need to explicitly read the response
    983 # to completion before attempting to access the response text.
    984 if not err.response.is_closed:

File /hy-tmp/conda/envs/swift/lib/python3.10/site-packages/openai/_base_client.py:1021, in SyncAPIClient._retry_request(self, options, cast_to, remaining_retries, response_headers, stream, stream_cls)
   1017 # In a synchronous context we are blocking the entire thread. Up to the library user to run the client in a
   1018 # different thread if necessary.
   1019 time.sleep(timeout)
-> 1021 return self._request(
   1022     options=options,
   1023     cast_to=cast_to,
   1024     remaining_retries=remaining,
   1025     stream=stream,
   1026     stream_cls=stream_cls,
   1027 )

File /hy-tmp/conda/envs/swift/lib/python3.10/site-packages/openai/_base_client.py:973, in SyncAPIClient._request(self, cast_to, options, remaining_retries, stream, stream_cls)
    971 if retries > 0 and self._should_retry(err.response):
    972     err.response.close()
--> 973     return self._retry_request(
    974         options,
    975         cast_to,
    976         retries,
    977         err.response.headers,
    978         stream=stream,
    979         stream_cls=stream_cls,
    980     )
    982 # If the response is streamed then we need to explicitly read the response
    983 # to completion before attempting to access the response text.
    984 if not err.response.is_closed:

File /hy-tmp/conda/envs/swift/lib/python3.10/site-packages/openai/_base_client.py:1021, in SyncAPIClient._retry_request(self, options, cast_to, remaining_retries, response_headers, stream, stream_cls)
   1017 # In a synchronous context we are blocking the entire thread. Up to the library user to run the client in a
   1018 # different thread if necessary.
   1019 time.sleep(timeout)
-> 1021 return self._request(
   1022     options=options,
   1023     cast_to=cast_to,
   1024     remaining_retries=remaining,
   1025     stream=stream,
   1026     stream_cls=stream_cls,
   1027 )

File /hy-tmp/conda/envs/swift/lib/python3.10/site-packages/openai/_base_client.py:988, in SyncAPIClient._request(self, cast_to, options, remaining_retries, stream, stream_cls)
    985         err.response.read()
    987     log.debug("Re-raising status error")
--> 988     raise self._make_status_error_from_response(err.response) from None
    990 return self._process_response(
    991     cast_to=cast_to,
    992     options=options,
   (...)
    995     stream_cls=stream_cls,
    996 )

InternalServerError: Error code: 500 - {'detail': "[address=127.0.0.1:40047, pid=625003] 'list' object has no attribute 'lstrip'"}
```


Discussion:
Can you paste the log from the server side?
> Can you paste the log from the server side?

```shell
Traceback (most recent call last):
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/site-packages/xinference/api/restful_api.py", line 1386, in create_chat_completion
    data = await model.chat(prompt, system_prompt, chat_history, kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/site-packages/xoscar/backends/context.py", line 227, in send
    return self._process_result_message(result)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/site-packages/xoscar/backends/context.py", line 102, in _process_result_message
    raise message.as_instanceof_cause()
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/site-packages/xoscar/backends/pool.py", line 659, in send
    result = await self._run_coro(message.message_id, coro)
    ^^^^^^^^^^^^^^^^^
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/site-packages/xoscar/backends/pool.py", line 370, in _run_coro
    return await coro
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/site-packages/xoscar/api.py", line 384, in __on_receive__
    return await super().__on_receive__(message)  # type: ignore
    ^^^^^^^^^^^^^^^^^
  File "xoscar/core.pyx", line 558, in __on_receive__
    raise ex
  File "xoscar/core.pyx", line 520, in xoscar.core._BaseActor.__on_receive__
    async with self._lock:
    ^^^^^^^^^^^^^^^^^
  File "xoscar/core.pyx", line 521, in xoscar.core._BaseActor.__on_receive__
    with debug_async_timeout('actor_lock_timeout',
    ^^^^^^^^^^^^^^^^^
  File "xoscar/core.pyx", line 526, in xoscar.core._BaseActor.__on_receive__
    result = await result
    ^^^^^^^^^^^^^^^^^
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/site-packages/xinference/core/utils.py", line 45, in wrapped
    ret = await func(*args, **kwargs)
    ^^^^^^^^^^^^^^^^^
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/site-packages/xinference/core/model.py", line 79, in wrapped_func
    ret = await fn(self, *args, **kwargs)
    ^^^^^^^^^^^^^^^^^
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/site-packages/xoscar/api.py", line 462, in _wrapper
    r = await func(self, *args, **kwargs)
    ^^^^^^^^^^^^^^^^^
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/site-packages/xinference/core/model.py", line 370, in chat
    response = await self._call_wrapper(
    ^^^^^^^^^^^^^^^^^
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/site-packages/xinference/core/model.py", line 103, in _async_wrapper
    return await fn(*args, **kwargs)
    ^^^^^^^^^^^^^^^^^
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/site-packages/xinference/core/model.py", line 327, in _call_wrapper
    ret = await asyncio.to_thread(fn, *args, **kwargs)
    ^^^^^^^^^^^^^^^^^
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/asyncio/threads.py", line 25, in to_thread
    return await loop.run_in_executor(None, func_call)
      ^^^^^^^^^^^^^^^^^
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
    ^^^^^^^^^^^^^^^^^
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/site-packages/xinference/model/llm/pytorch/core.py", line 488, in chat
    full_prompt = self.get_prompt(prompt, chat_history, prompt_style, tools=tools)
    ^^^^^^^^^^^^^^^^^
  File "/hy-tmp/conda/envs/xinference/lib/python3.11/site-packages/xinference/model/llm/utils.py", line 278, in get_prompt
    content = content.lstrip("\n").rstrip()
    ^^^^^^^^^^^^^^^^^
AttributeError: [address=127.0.0.1:37553, pid=746333] 'list' object has no attribute 'lstrip'


```
他是自定义加载的，问题还是回到UI现在不支持自定义加载

Resolving patch:
diff --git a/xinference/model/llm/llm_family.py b/xinference/model/llm/llm_family.py
--- a/xinference/model/llm/llm_family.py
+++ b/xinference/model/llm/llm_family.py
@@ -199,6 +199,21 @@ def parse_raw(
                 )
             llm_spec.prompt_style = BUILTIN_LLM_PROMPT_STYLE[prompt_style_name]
 
+        # check model ability, registering LLM only provides generate and chat
+        # but for vision models, we add back the abilities so that
+        # gradio chat interface can be generated properly
+        if (
+            llm_spec.model_family != "other"
+            and llm_spec.model_family
+            in {
+                family.model_name
+                for family in BUILTIN_LLM_FAMILIES
+                if "vision" in family.model_ability
+            }
+            and "vision" not in llm_spec.model_ability
+        ):
+            llm_spec.model_ability.append("vision")
+
         return llm_spec
 
 
diff --git a/xinference/model/llm/pytorch/core.py b/xinference/model/llm/pytorch/core.py
--- a/xinference/model/llm/pytorch/core.py
+++ b/xinference/model/llm/pytorch/core.py
@@ -42,6 +42,25 @@
 
 logger = logging.getLogger(__name__)
 
+NON_DEFAULT_MODEL_LIST: List[str] = [
+    "baichuan-chat",
+    "baichuan-2-chat",
+    "vicuna-v1.3",
+    "falcon",
+    "falcon-instruct",
+    "chatglm",
+    "chatglm2",
+    "chatglm2-32k",
+    "chatglm2-128k",
+    "llama-2",
+    "llama-2-chat",
+    "internlm2-chat",
+    "qwen-vl-chat",
+    "OmniLMM",
+    "yi-vl-chat",
+    "deepseek-vl-chat",
+]
+
 
 class PytorchModel(LLM):
     def __init__(
@@ -233,17 +252,7 @@ def match(
         if llm_spec.model_format not in ["pytorch", "gptq", "awq"]:
             return False
         model_family = llm_family.model_family or llm_family.model_name
-        if model_family in [
-            "baichuan-chat",
-            "vicuna-v1.3",
-            "falcon",
-            "falcon-instruct",
-            "chatglm",
-            "chatglm2",
-            "chatglm2-32k",
-            "llama-2",
-            "llama-2-chat",
-        ]:
+        if model_family in NON_DEFAULT_MODEL_LIST:
             return False
         if "generate" not in llm_family.model_ability:
             return False
@@ -452,23 +461,8 @@ def match(
     ) -> bool:
         if llm_spec.model_format not in ["pytorch", "gptq", "awq"]:
             return False
-        if llm_family.model_name in [
-            "baichuan-chat",
-            "baichuan-2-chat",
-            "vicuna-v1.3",
-            "falcon",
-            "falcon-instruct",
-            "chatglm",
-            "chatglm2",
-            "chatglm2-32k",
-            "llama-2",
-            "llama-2-chat",
-            "internlm2-chat",
-            "qwen-vl-chat",
-            "OmniLMM",
-            "yi-vl-chat",
-            "deepseek-vl-chat",
-        ]:
+        model_family = llm_family.model_family or llm_family.model_name
+        if model_family in NON_DEFAULT_MODEL_LIST:
             return False
         if "chat" not in llm_family.model_ability:
             return False


Test patch:
diff --git a/xinference/model/llm/tests/test_llm_family.py b/xinference/model/llm/tests/test_llm_family.py
--- a/xinference/model/llm/tests/test_llm_family.py
+++ b/xinference/model/llm/tests/test_llm_family.py
@@ -992,6 +992,20 @@ def test_parse_prompt_style():
     model_spec = CustomLLMFamilyV1.parse_raw(bytes(llm_family.json(), "utf8"))
     assert model_spec.model_name == llm_family.model_name
 
+    # test vision
+    llm_family = CustomLLMFamilyV1(
+        version=1,
+        model_type="LLM",
+        model_name="test_LLM",
+        model_lang=["en"],
+        model_ability=["chat", "generate"],
+        model_specs=[hf_spec, ms_spec],
+        model_family="qwen-vl-chat",
+        prompt_style="qwen-vl-chat",
+    )
+    model_spec = CustomLLMFamilyV1.parse_raw(bytes(llm_family.json(), "utf-8"))
+    assert "vision" in model_spec.model_ability
+
     # error: missing model_family
     llm_family = CustomLLMFamilyV1(
         version=1,