internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: xorbitsai/inference

Issue:
useless fstring
https://github.com/xorbitsai/inference/blob/8b5cd165f545a649ce6ac91a4345a6a80fbbfdf0/xinference/core/supervisor.py#L725-L736

debug function uses fstring, however none of placeholders is provided.


Discussion:
Could you open a PR to fix it?
sure

Resolving patch:
diff --git a/xinference/core/supervisor.py b/xinference/core/supervisor.py
--- a/xinference/core/supervisor.py
+++ b/xinference/core/supervisor.py
@@ -722,17 +722,10 @@ async def launch_builtin_model(
         if model_uid is None:
             model_uid = self._gen_model_uid(model_name)
 
+        model_size = str(model_size_in_billions) if model_size_in_billions else ""
         logger.debug(
-            (
-                f"Enter launch_builtin_model, model_uid: %s, model_name: %s, model_size: %s, "
-                f"model_format: %s, quantization: %s, replica: %s"
-            ),
-            model_uid,
-            model_name,
-            str(model_size_in_billions) if model_size_in_billions else "",
-            model_format,
-            quantization,
-            replica,
+            f"Enter launch_builtin_model, model_uid: {model_uid}, model_name: {model_name}, model_size: {model_size}, "
+            f"model_format: {model_format}, quantization: {quantization}, replica: {replica}"
         )
 
         async def _launch_one_model(_replica_model_uid):


Test patch:
Repository: xorbitsai/inference

Issue:
BUG: [UI] image model does not show `cache` status
Note that the issue tracker is NOT the place for general support.



Discussion:

Resolving patch:
diff --git a/xinference/model/image/__init__.py b/xinference/model/image/__init__.py
--- a/xinference/model/image/__init__.py
+++ b/xinference/model/image/__init__.py
@@ -18,7 +18,9 @@
 from itertools import chain
 
 from .core import (
+    BUILTIN_IMAGE_MODELS,
     IMAGE_MODEL_DESCRIPTIONS,
+    MODELSCOPE_IMAGE_MODELS,
     ImageModelFamilyV1,
     generate_image_description,
     get_cache_status,
@@ -29,14 +31,18 @@
 _model_spec_modelscope_json = os.path.join(
     os.path.dirname(__file__), "model_spec_modelscope.json"
 )
-BUILTIN_IMAGE_MODELS = dict(
-    (spec["model_name"], ImageModelFamilyV1(**spec))
-    for spec in json.load(codecs.open(_model_spec_json, "r", encoding="utf-8"))
+BUILTIN_IMAGE_MODELS.update(
+    dict(
+        (spec["model_name"], ImageModelFamilyV1(**spec))
+        for spec in json.load(codecs.open(_model_spec_json, "r", encoding="utf-8"))
+    )
 )
-MODELSCOPE_IMAGE_MODELS = dict(
-    (spec["model_name"], ImageModelFamilyV1(**spec))
-    for spec in json.load(
-        codecs.open(_model_spec_modelscope_json, "r", encoding="utf-8")
+MODELSCOPE_IMAGE_MODELS.update(
+    dict(
+        (spec["model_name"], ImageModelFamilyV1(**spec))
+        for spec in json.load(
+            codecs.open(_model_spec_modelscope_json, "r", encoding="utf-8")
+        )
     )
 )
 
diff --git a/xinference/model/image/core.py b/xinference/model/image/core.py
--- a/xinference/model/image/core.py
+++ b/xinference/model/image/core.py
@@ -27,6 +27,8 @@
 logger = logging.getLogger(__name__)
 
 IMAGE_MODEL_DESCRIPTIONS: Dict[str, List[Dict]] = defaultdict(list)
+BUILTIN_IMAGE_MODELS: Dict[str, "ImageModelFamilyV1"] = {}
+MODELSCOPE_IMAGE_MODELS: Dict[str, "ImageModelFamilyV1"] = {}
 
 
 def get_image_model_descriptions():
@@ -151,7 +153,21 @@ def get_cache_status(
 ) -> bool:
     cache_dir = get_cache_dir(model_spec)
     meta_path = os.path.join(cache_dir, "__valid_download")
-    return valid_model_revision(meta_path, model_spec.model_revision)
+
+    model_name = model_spec.model_name
+    if model_name in BUILTIN_IMAGE_MODELS and model_name in MODELSCOPE_IMAGE_MODELS:
+        hf_spec = BUILTIN_IMAGE_MODELS[model_name]
+        ms_spec = MODELSCOPE_IMAGE_MODELS[model_name]
+
+        return any(
+            [
+                valid_model_revision(meta_path, hf_spec.model_revision),
+                valid_model_revision(meta_path, ms_spec.model_revision),
+            ]
+        )
+    else:  # Usually for UT
+        logger.warning(f"Cannot find builtin image model spec: {model_name}")
+        return valid_model_revision(meta_path, model_spec.model_revision)
 
 
 def create_image_model_instance(


Test patch:
Repository: xorbitsai/inference

Issue:
FEAT: Support stopping stream output when client interrupts
### Is your feature request related to a problem? Please describe

Now when client stop generating, the background stream will continue, we should ad support for stopping generating at server side.


Discussion:

Resolving patch:
diff --git a/xinference/api/restful_api.py b/xinference/api/restful_api.py
--- a/xinference/api/restful_api.py
+++ b/xinference/api/restful_api.py
@@ -891,11 +891,17 @@ async def stream_results():
                         self.handle_request_limit_error(re)
                     async for item in iterator:
                         yield item
+                except asyncio.CancelledError:
+                    logger.info(
+                        f"Disconnected from client (via refresh/close) {request.client} during generate."
+                    )
+                    return
                 except Exception as ex:
                     logger.exception("Completion stream got an error: %s", ex)
                     await self._report_error_event(model_uid, str(ex))
                     # https://github.com/openai/openai-python/blob/e0aafc6c1a45334ac889fe3e54957d309c3af93f/src/openai/_streaming.py#L107
                     yield dict(data=json.dumps({"error": str(ex)}))
+                    return
 
             return EventSourceResponse(stream_results())
         else:
@@ -1273,11 +1279,23 @@ async def stream_results():
                     async for item in iterator:
                         yield item
                     yield "[DONE]"
+                # Note that asyncio.CancelledError does not inherit from Exception.
+                # When the user uses ctrl+c to cancel the streaming chat, asyncio.CancelledError would be triggered.
+                # See https://github.com/sysid/sse-starlette/blob/main/examples/example.py#L48
+                except asyncio.CancelledError:
+                    logger.info(
+                        f"Disconnected from client (via refresh/close) {request.client} during chat."
+                    )
+                    # See https://github.com/sysid/sse-starlette/blob/main/examples/error_handling.py#L13
+                    # Use return to stop the generator from continuing.
+                    # TODO: Cannot yield here. Yield here would leads to error for the next streaming request.
+                    return
                 except Exception as ex:
                     logger.exception("Chat completion stream got an error: %s", ex)
                     await self._report_error_event(model_uid, str(ex))
                     # https://github.com/openai/openai-python/blob/e0aafc6c1a45334ac889fe3e54957d309c3af93f/src/openai/_streaming.py#L107
                     yield dict(data=json.dumps({"error": str(ex)}))
+                    return
 
             return EventSourceResponse(stream_results())
         else:


Test patch:
Repository: xorbitsai/inference

Issue:
是否考虑支持CodeShell？
是否考虑支持CodeShell？
北大开源的多语言代码大模型基座：https://github.com/WisdomShell/codeshell


Discussion:

Resolving patch:
diff --git a/xinference/model/llm/utils.py b/xinference/model/llm/utils.py
--- a/xinference/model/llm/utils.py
+++ b/xinference/model/llm/utils.py
@@ -411,6 +411,16 @@ def get_role(role_name: str):
                 if content:
                     ret += content + "<end_of_turn>\n"
             return ret
+        elif prompt_style.style_name == "CodeShell":
+            ret = ""
+            for message in chat_history:
+                content = message["content"]
+                role = get_role(message["role"])
+                if content:
+                    ret += f"{role}{content}|<end>|"
+                else:
+                    ret += f"{role}".rstrip()
+            return ret
         else:
             raise ValueError(f"Invalid prompt style: {prompt_style.style_name}")
 


Test patch:
Repository: xorbitsai/inference

Issue:
QUESTION:是否考虑支持MiniCPM-2B
如题，谢谢！


Discussion:
[OmniLMM](https://github.com/OpenBMB/OmniLMM)还有这个，3B的多模态模型

Resolving patch:
diff --git a/xinference/model/llm/utils.py b/xinference/model/llm/utils.py
--- a/xinference/model/llm/utils.py
+++ b/xinference/model/llm/utils.py
@@ -421,6 +421,16 @@ def get_role(role_name: str):
                 else:
                     ret += f"{role}".rstrip()
             return ret
+        elif prompt_style.style_name == "MINICPM-2B":
+            ret = ""
+            for message in chat_history:
+                content = message["content"] or ""
+                role = get_role(message["role"])
+                if role == "user":
+                    ret += "<用户>" + content.strip()
+                else:
+                    ret += "<AI>" + content.strip()
+            return ret
         else:
             raise ValueError(f"Invalid prompt style: {prompt_style.style_name}")
 
@@ -585,7 +595,7 @@ def _eval_qwen_chat_arguments(c, tools):
     def _tool_calls_completion(cls, model_family, model_uid, c, tools):
         _id = str(uuid.uuid4())
         family = model_family.model_family or model_family.model_name
-        if "gorilla-openfunctions-v1" == family:
+        if family in ["gorilla-openfunctions-v1", "gorilla-openfunctions-v2"]:
             content, func, args = cls._eval_gorilla_openfunctions_arguments(c, tools)
         elif "chatglm3" == family:
             content, func, args = cls._eval_chatglm3_arguments(c, tools)


Test patch:
Repository: xorbitsai/inference

Issue:
Support CodeQwen1.5 request
request to support CodeQwen1.5 https://modelscope.cn/models/qwen/CodeQwen1.5-7B-Chat/summary


Discussion:
try using `register model`, model family is qwen1.5-chat.

Resolving patch:
diff --git a/xinference/model/llm/vllm/core.py b/xinference/model/llm/vllm/core.py
--- a/xinference/model/llm/vllm/core.py
+++ b/xinference/model/llm/vllm/core.py
@@ -116,6 +116,7 @@ class VLLMGenerateConfig(TypedDict, total=False):
 ]
 if VLLM_INSTALLED and vllm.__version__ >= "0.3.0":
     VLLM_SUPPORTED_CHAT_MODELS.append("qwen1.5-chat")
+    VLLM_SUPPORTED_CHAT_MODELS.append("codeqwen1.5-chat")
 
 if VLLM_INSTALLED and vllm.__version__ >= "0.3.2":
     VLLM_SUPPORTED_CHAT_MODELS.append("gemma-it")


Test patch:
Repository: xorbitsai/inference

Issue:
FEAT: support phi-3 model
### Is your feature request related to a problem? Please describe
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]

### Describe the solution you'd like
A clear and concise description of what you want to happen.

### Describe alternatives you've considered
A clear and concise description of any alternative solutions or features you've considered.

### Additional context
Add any other context or screenshots about the feature request here.



Discussion:

Resolving patch:
diff --git a/xinference/model/llm/utils.py b/xinference/model/llm/utils.py
--- a/xinference/model/llm/utils.py
+++ b/xinference/model/llm/utils.py
@@ -447,6 +447,17 @@ def get_role(role_name: str):
                 else:
                     ret += "<AI>" + content.strip()
             return ret
+        elif prompt_style.style_name == "PHI3":
+            ret = f"
{prompt_style.intra_message_sep}{prompt_style.system_prompt}{prompt_style.inter_message_sep}"
+            for message in chat_history:
+                content = message["content"] or ""
+                role = get_role(message["role"])
+                if content:
+                    ret += f"<|{role}|>{prompt_style.intra_message_sep}{content}{prompt_style.inter_message_sep}"
+                else:
+                    ret += f"<|{role}|>{prompt_style.intra_message_sep}"
+            ret += "
\n"
+            return ret
         else:
             raise ValueError(f"Invalid prompt style: {prompt_style.style_name}")
 


Test patch:
Repository: xorbitsai/inference

Issue:
BUG: there is no tool_call ability
https://github.com/xorbitsai/inference/blob/944bf6c033e2cd60d0eb22edcf15e2f4ffdb0765/xinference/model/llm/llm_family.py#L169

Here it is compared with `tool_call`, but there is only `tools` ability in model abilities.

https://github.com/xorbitsai/inference/blob/944bf6c033e2cd60d0eb22edcf15e2f4ffdb0765/xinference/model/llm/llm_family.py#L117


Discussion:
Yes, this must be a bug, can you fix it?
> Yes, this must be a bug, can you fix it?

Sure, but which is right? `tool_call` or `tools`, I notice that the source code use `tool_call` everywhere, maybe it is better to unify the term first.
@codingl2k1 What is your opinion?

Resolving patch:
diff --git a/xinference/core/supervisor.py b/xinference/core/supervisor.py
--- a/xinference/core/supervisor.py
+++ b/xinference/core/supervisor.py
@@ -284,7 +284,7 @@ async def get_builtin_families() -> Dict[str, List[str]]:
         return {
             "chat": list(BUILTIN_LLM_MODEL_CHAT_FAMILIES),
             "generate": list(BUILTIN_LLM_MODEL_GENERATE_FAMILIES),
-            "tool_call": list(BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES),
+            "tools": list(BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES),
         }
 
     async def get_devices_count(self) -> int:
diff --git a/xinference/model/llm/__init__.py b/xinference/model/llm/__init__.py
--- a/xinference/model/llm/__init__.py
+++ b/xinference/model/llm/__init__.py
@@ -174,7 +174,7 @@ def _install():
             BUILTIN_LLM_MODEL_CHAT_FAMILIES.add(model_spec.model_name)
         else:
             BUILTIN_LLM_MODEL_GENERATE_FAMILIES.add(model_spec.model_name)
-        if "tool_call" in model_spec.model_ability:
+        if "tools" in model_spec.model_ability:
             BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES.add(model_spec.model_name)
 
     modelscope_json_path = os.path.join(
@@ -197,7 +197,7 @@ def _install():
             BUILTIN_LLM_MODEL_CHAT_FAMILIES.add(model_spec.model_name)
         else:
             BUILTIN_LLM_MODEL_GENERATE_FAMILIES.add(model_spec.model_name)
-        if "tool_call" in model_spec.model_ability:
+        if "tools" in model_spec.model_ability:
             BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES.add(model_spec.model_name)
 
     for llm_specs in [BUILTIN_LLM_FAMILIES, BUILTIN_MODELSCOPE_LLM_FAMILIES]:
diff --git a/xinference/model/llm/llm_family.py b/xinference/model/llm/llm_family.py
--- a/xinference/model/llm/llm_family.py
+++ b/xinference/model/llm/llm_family.py
@@ -166,7 +166,7 @@ def parse_raw(
             )
         if (
             llm_spec.model_family != "other"
-            and "tool_call" in llm_spec.model_ability
+            and "tools" in llm_spec.model_ability
             and llm_spec.model_family not in BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES
         ):
             raise ValueError(


Test patch:
Repository: celery/kombu

Issue:
Kombu compression "encodes" the body ?
At https://github.com/ask/kombu/blob/master/kombu/compression.py#L61, we see:

```
return encoder(body.encode("utf-8")), content_type
```

Same for decoding. What gives? It crashes with msgpack (which can't be encoded). Is there any reason for that ?



Discussion:
I can't remember why it's using `encode(utf-8)`, but I'm pretty sure the reason is that it needs to convert it to bytes for Python3.

It should probably use `utils.encoding.ensure_bytes` instead.


Resolving patch:
diff --git a/kombu/compression.py b/kombu/compression.py
--- a/kombu/compression.py
+++ b/kombu/compression.py
@@ -10,6 +10,8 @@
 """
 from __future__ import absolute_import
 
+from kombu.utils.encoding import ensure_bytes, bytes_to_str
+
 import zlib
 
 _aliases = {}
@@ -58,7 +60,7 @@ def compress(body, content_type):
 
     """
     encoder, content_type = get_encoder(content_type)
-    return encoder(body.encode("utf-8")), content_type
+    return encoder(ensure_bytes(body)), content_type
 
 
 def decompress(body, content_type):
@@ -68,7 +70,7 @@ def decompress(body, content_type):
     :param content_type: mime-type of compression method used.
 
     """
-    return get_decoder(content_type)(body).decode("utf-8")
+    return bytes_to_str(get_decoder(content_type)(body))
 
 
 register(zlib.compress,


Test patch:
Repository: celery/kombu

Issue:
autoretry assumes function has __name__, __module__ and __doc__
https://github.com/celery/kombu/blob/master/kombu/connection.py#L498

This means it completely breaks when passed a partial.

Maybe just `.getattr`?



Discussion:

Resolving patch:
diff --git a/kombu/connection.py b/kombu/connection.py
--- a/kombu/connection.py
+++ b/kombu/connection.py
@@ -495,9 +495,9 @@ def autoretry(self, fun, channel=None, **ensure_options):
         create_channel = self.channel
 
         class Revival(object):
-            __name__ = fun.__name__
-            __module__ = fun.__module__
-            __doc__ = fun.__doc__
+            __name__ = getattr(fun, '__name__', None)
+            __module__ = getattr(fun, '__module__', None)
+            __doc__ = getattr(fun, '__doc__', None)
 
             def revive(self, channel):
                 channels[0] = channel


Test patch:
diff --git a/kombu/tests/test_connection.py b/kombu/tests/test_connection.py
--- a/kombu/tests/test_connection.py
+++ b/kombu/tests/test_connection.py
@@ -418,7 +418,6 @@ def publish():
 
     def test_autoretry(self):
         myfun = Mock()
-        myfun.__name__ = 'test_autoretry'
 
         self.conn.transport.connection_errors = (KeyError, )
Repository: celery/kombu

Issue:
Django Kombu app conflicts with Django app with Django 1.7
``` python
Traceback (most recent call last):
  File "manage.py", line 13, in <module>
    execute_from_command_line(sys.argv)
  File "/.../.env/lib/python2.7/site-packages/django/core/management/__init__.py", line 385, in execute_from_command_line
    utility.execute()
  File "/.../.env/lib/python2.7/site-packages/django/core/management/__init__.py", line 354, in execute
    django.setup()
  File "/.../.env/lib/python2.7/site-packages/django/__init__.py", line 21, in setup
    apps.populate(settings.INSTALLED_APPS)
  File "/.../.env/lib/python2.7/site-packages/django/apps/registry.py", line 89, in populate
    "duplicates: %s" % app_config.label)
django.core.exceptions.ImproperlyConfigured: Application labels aren't unique, duplicates: django
```

More info [here](https://docs.djangoproject.com/en/1.7/releases/1.7/#app-loading-refactor)



Discussion:

Resolving patch:
diff --git a/kombu/transport/django/__init__.py b/kombu/transport/django/__init__.py
--- a/kombu/transport/django/__init__.py
+++ b/kombu/transport/django/__init__.py
@@ -18,7 +18,7 @@
     pass
 else:
     class KombuAppConfig(AppConfig):
-        name = __name__
+        name = 'kombu.transport.django'
         label = name.replace('.', '_')
         verbose_name = 'Message queue'
     default_app_config = 'kombu.transport.django.KombuAppConfig'


Test patch:
Repository: celery/kombu

Issue:
The qpid transport - Transport class leaks (2) file descriptors
The _Transport_ class provided by the qpid transport creates a pipe in **init**().  The pipe file descriptors are never closed.  As a result, each connect or re-connect causes these two file descriptors to be leaked.

kombu 3.0.20 on f21.



Discussion:

Resolving patch:
diff --git a/kombu/transport/qpid.py b/kombu/transport/qpid.py
--- a/kombu/transport/qpid.py
+++ b/kombu/transport/qpid.py
@@ -1707,3 +1707,14 @@ def default_connection_params(self):
         return {'userid': 'guest', 'password': '',
                 'port': self.default_port, 'virtual_host': '',
                 'hostname': 'localhost', 'sasl_mechanisms': 'PLAIN ANONYMOUS'}
+
+    def __del__(self):
+        """
+        Ensure file descriptors opened in __init__() are closed.
+        """
+        for fd in (self.r, self._w):
+            try:
+                os.close(fd)
+            except OSError:
+                # ignored
+                pass


Test patch:
diff --git a/kombu/tests/transport/test_qpid.py b/kombu/tests/transport/test_qpid.py
--- a/kombu/tests/transport/test_qpid.py
+++ b/kombu/tests/transport/test_qpid.py
@@ -2023,3 +2023,26 @@ def test_default_connection_params(self):
         my_transport = Transport(self.mock_client)
         result_params = my_transport.default_connection_params
         self.assertDictEqual(correct_params, result_params)
+
+    @patch('os.close')
+    def test_del(self, close):
+        my_transport = Transport(self.mock_client)
+        my_transport.__del__()
+        self.assertEqual(
+            close.call_args_list,
+            [
+                ((my_transport.r,), {}),
+                ((my_transport._w,), {}),
+            ])
+
+    @patch('os.close')
+    def test_del_failed(self, close):
+        close.side_effect = OSError()
+        my_transport = Transport(self.mock_client)
+        my_transport.__del__()
+        self.assertEqual(
+            close.call_args_list,
+            [
+                ((my_transport.r,), {}),
+                ((my_transport._w,), {}),
+            ])
Repository: celery/kombu

Issue:
Qpid Transport deadlocks when a ConnectionError occurs
If using the asynchonous interface of the Qpid transport, when a `ConnectionError` occurs your connection will be unusable and no Exception will be raised. This issue is better described in the upstream Qpid issue tracker as [QPID-7053](https://issues.apache.org/jira/browse/QPID-7053).

There needs to be a callback created so that the Kombu transport can write to the epoll monitor causing `Celery` to call back into `qpid.messaging`.



Discussion:

Resolving patch:
diff --git a/kombu/transport/qpid.py b/kombu/transport/qpid.py
--- a/kombu/transport/qpid.py
+++ b/kombu/transport/qpid.py
@@ -1437,6 +1437,9 @@ def verify_runtime_environment(self):
     def _qpid_session_ready(self):
         os.write(self._w, '0')
 
+    def _qpid_exception(self, obj_with_exception):
+        os.write(self._w, 'e')
+
     def on_readable(self, connection, loop):
         """Handle any messages associated with this Transport.
 
@@ -1594,6 +1597,12 @@ def establish_connection(self):
         conn.client = self.client
         self.session = conn.get_qpid_connection().session()
         self.session.set_message_received_handler(self._qpid_session_ready)
+        conn.get_qpid_connection().set_exception_notify_handler(
+            self._qpid_exception
+        )
+        self.session.set_exception_notify_handler(
+            self._qpid_exception
+        )
         return conn
 
     def close_connection(self, connection):


Test patch: