Repository: xorbitsai/inference

Issue:
BUG: Register Custom Model Docs
### Describe the bug
When running the code as provided in the [docs](https://inference.readthedocs.io/en/latest/models/custom.html#define-a-custom-model) to register a custom model, the following error was raised:

<details>
  <summary>Full Error Stack</summary>

```
---------------------------------------------------------------------------
ValidationError                           Traceback (most recent call last)
Cell In[1], line 11
      9 endpoint = 'http://localhost:9997'
     10 client = Client(endpoint)
---> 11 client.register_model(model_type="LLM", model=model, persist=False)
     13 client.list_model_registrations(model_type="LLM")

File ~/Desktop/cs/inference/xinference/client.py:502, in Client.register_model(self, model_type, model, persist)
    490 """
    491 Register a custom model.
    492 
   (...)
    499 persist: bool
    500 """
    501 coro = self._supervisor_ref.register_model(model_type, model, persist)
--> 502 self._isolation.call(coro)

File ~/Desktop/cs/inference/xinference/isolation.py:44, in Isolation.call(self, coro)
     42 def call(self, coro: Coroutine) -> Any:
     43     fut = asyncio.run_coroutine_threadsafe(coro, self._loop)
---> 44     return fut.result()

File ~/anaconda3/lib/python3.11/concurrent/futures/_base.py:456, in Future.result(self, timeout)
    454     raise CancelledError()
    455 elif self._state == FINISHED:
--> 456     return self.__get_result()
    457 else:
    458     raise TimeoutError()

File ~/anaconda3/lib/python3.11/concurrent/futures/_base.py:401, in Future.__get_result(self)
    399 if self._exception:
    400     try:
--> 401         raise self._exception
    402     finally:
    403         # Break a reference cycle with the exception in self._exception
    404         self = None

File ~/anaconda3/lib/python3.11/site-packages/xoscar/backends/context.py:227, in IndigenActorContext.send(self, actor_ref, message, wait_response, profiling_context)
    225 if wait_response:
    226     result = await self._wait(future, actor_ref.address, send_message)  # type: ignore
--> 227     return self._process_result_message(result)
    228 else:
    229     return future

File ~/anaconda3/lib/python3.11/site-packages/xoscar/backends/context.py:102, in IndigenActorContext._process_result_message(message)
    100     return message.result
    101 else:
--> 102     raise message.as_instanceof_cause()

File ~/anaconda3/lib/python3.11/site-packages/xoscar/backends/pool.py:657, in send()
    655         raise ActorNotExist(f"Actor {actor_id} does not exist")
    656     coro = self._actors[actor_id].__on_receive__(message.content)
--> 657     result = await self._run_coro(message.message_id, coro)
    658     processor.result = ResultMessage(
    659         message.message_id,
    660         result,
    661         protocol=message.protocol,
    662         profiling_context=message.profiling_context,
    663     )
    664 return processor.result

File ~/anaconda3/lib/python3.11/site-packages/xoscar/backends/pool.py:368, in _run_coro()
    366 self._process_messages[message_id] = asyncio.tasks.current_task()
    367 try:
--> 368     return await coro
    369 finally:
    370     self._process_messages.pop(message_id, None)

File ~/anaconda3/lib/python3.11/site-packages/xoscar/api.py:306, in __on_receive__()
    297 async def __on_receive__(self, message: Tuple[Any]):
    298     """
    299     Handle message from other actors and dispatch them to user methods
    300 
   (...)
    304         Message shall be (method_name,) + args + (kwargs,)
    305     """
--> 306     return await super().__on_receive__(message)

File ~/anaconda3/lib/python3.11/site-packages/xoscar/core.pyx:527, in __on_receive__()
    525     debug_logger.exception('Got unhandled error when handling message %.500r '
    526                            'in actor %s at %s', message, self.uid, self.address)
--> 527 raise ex
    528 
    529 

File ~/anaconda3/lib/python3.11/site-packages/xoscar/core.pyx:497, in xoscar.core._BaseActor.__on_receive__()
    495 if call_method == CALL_METHOD_DEFAULT:
    496     func = getattr(self, method)
--> 497     async with self._lock:
    498         with debug_async_timeout('actor_lock_timeout',
    499                                  "Method %s of actor %s hold lock timeout.",

File ~/anaconda3/lib/python3.11/site-packages/xoscar/core.pyx:498, in xoscar.core._BaseActor.__on_receive__()
    496 func = getattr(self, method)
    497 async with self._lock:
--> 498     with debug_async_timeout('actor_lock_timeout',
    499                              "Method %s of actor %s hold lock timeout.",
    500                              method, self.uid):

File ~/anaconda3/lib/python3.11/site-packages/xoscar/core.pyx:503, in xoscar.core._BaseActor.__on_receive__()
    501             result = func(*args, **kwargs)
    502             if asyncio.iscoroutine(result):
--> 503                 result = await result
    504 elif call_method == CALL_METHOD_BATCH:
    505     func = getattr(self, method)

File ~/Desktop/cs/inference/xinference/core/utils.py:25, in wrapped()
     23 logger.debug(f"Enter {func.__name__}, args: {args}, kwargs: {kwargs}")
     24 start = time.time()
---> 25 ret = await func(*args, **kwargs)
     26 logger.debug(
     27     f"Leave {func.__name__}, elapsed time: {int(time.time() - start)} ms"
     28 )
     29 return ret

File ~/Desktop/cs/inference/xinference/core/supervisor.py:118, in register_model()
    115 if model_type == "LLM":
    116     from ..model.llm import LLMFamilyV1, register_llm
--> 118     llm_family = LLMFamilyV1.parse_raw(model)
    119     register_llm(llm_family, persist)
    121     if not self.is_local_deployment:

File ~/anaconda3/lib/python3.11/site-packages/pydantic/main.py:548, in pydantic.main.BaseModel.parse_raw()
    539     obj = load_str_bytes(
    540         b,
    541         proto=proto,
   (...)
    545         json_loads=cls.__config__.json_loads,
    546     )
    547 except (ValueError, TypeError, UnicodeDecodeError) as e:
--> 548     raise ValidationError([ErrorWrapper(e, loc=ROOT_KEY)], cls)
    549 return cls.parse_obj(obj)

ValidationError: [address=127.0.0.1:19955, pid=57934] 1 validation error for LLMFamilyV1
__root__
  Expecting property name enclosed in double quotes: line 33 column 1 (char 667) (type=value_error.jsondecode; msg=Expecting property name enclosed in double quotes; doc={
  "version": 1,
  "model_name": "custom-llama-2",
  "model_lang": [
    "en"
  ],
  "model_ability": [
    "generate"
  ],
  "model_specs": [
    {
      "model_format": "pytorch",
      "model_size_in_billions": 7,
      "quantizations": [
        "4-bit",
        "8-bit",
        "none"
      ],
      "model_id": "meta-llama/Llama-2-7b",
      "model_uri": "file:///path/to/llama-2-7b"
    },
    {
      "model_format": "ggmlv3",
      "model_size_in_billions": 7,
      "quantizations": [
        "q4_0",
        "q8_0"
      ],
      "model_id": "TheBloke/Llama-2-7B-GGML",
      "model_file_name_template": "llama-2-7b.ggmlv3.{quantization}.bin"
    }
  ],
}
; pos=667; lineno=33; colno=1)

```
</details>

It seems to be caused by a trailing comma in the provided json file, as discussed in this [StackOverflow post](https://stackoverflow.com/a/55117031). 


Discussion:

Resolving patch:
diff --git a/xinference/model/llm/llm_family.py b/xinference/model/llm/llm_family.py
--- a/xinference/model/llm/llm_family.py
+++ b/xinference/model/llm/llm_family.py
@@ -27,6 +27,7 @@
 logger = logging.getLogger(__name__)
 
 MAX_ATTEMPTS = 3
+DEFAULT_CONTEXT_LENGTH = 2048
 
 
 class GgmlLLMSpecV1(BaseModel):
@@ -60,7 +61,7 @@ class PromptStyleV1(BaseModel):
 
 class LLMFamilyV1(BaseModel):
     version: Literal[1]
-    context_length: int
+    context_length: Optional[int] = DEFAULT_CONTEXT_LENGTH
     model_name: str
     model_lang: List[Literal["en", "zh"]]
     model_ability: List[Literal["embed", "generate", "chat"]]


Test patch:
diff --git a/xinference/model/llm/tests/test_llm_family.py b/xinference/model/llm/tests/test_llm_family.py
--- a/xinference/model/llm/tests/test_llm_family.py
+++ b/xinference/model/llm/tests/test_llm_family.py
@@ -120,7 +120,6 @@ def test_serialize_llm_family_v1():
     )
     llm_family = LLMFamilyV1(
         version=1,
-        context_length=2048,
         model_type="LLM",
         model_name="TestModel",
         model_lang=["en"],
@@ -132,6 +131,19 @@ def test_serialize_llm_family_v1():
     expected = """{"version": 1, "context_length": 2048, "model_name": "TestModel", "model_lang": ["en"], "model_ability": ["embed", "generate"], "model_description": null, "model_specs": [{"model_format": "ggmlv3", "model_size_in_billions": 2, "quantizations": ["q4_0", "q4_1"], "model_id": "example/TestModel", "model_revision": "123", "model_file_name_template": "TestModel.{quantization}.ggmlv3.bin", "model_uri": null}, {"model_format": "pytorch", "model_size_in_billions": 3, "quantizations": ["int8", "int4", "none"], "model_id": "example/TestModel", "model_revision": "456", "model_uri": null}], "prompt_style": {"style_name": "ADD_COLON_SINGLE", "system_prompt": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["user", "assistant"], "intra_message_sep": "\\n### ", "inter_message_sep": "\\n### ", "stop": null, "stop_token_ids": null}}"""
     assert json.loads(llm_family.json()) == json.loads(expected)
 
+    llm_family_context_length = LLMFamilyV1(
+        version=1,
+        context_length=2048,
+        model_type="LLM",
+        model_name="TestModel",
+        model_lang=["en"],
+        model_ability=["embed", "generate"],
+        model_specs=[ggml_spec, pytorch_spec],
+        prompt_style=prompt_style,
+    )
+
+    assert json.loads(llm_family_context_length.json()) == json.loads(expected)
+
 
 def test_builtin_llm_families():
     import os