Repository: xorbitsai/inference

Issue:
ENH: including default context length into model family
### Is your feature request related to a problem? Please describe
The context length differs from one model to another. Need to specify it in the config file.

### Describe the solution you'd like
A clear and concise description of what you want to happen.

### Describe alternatives you've considered
A clear and concise description of any alternative solutions or features you've considered.

### Additional context
Add any other context or screenshots about the feature request here.



Discussion:

Resolving patch:
diff --git a/xinference/client.py b/xinference/client.py
--- a/xinference/client.py
+++ b/xinference/client.py
@@ -850,6 +850,8 @@ def describe_model(self, model_uid: str):
                the quantization applied to the model
             - "revision": str
                the revision number of the LLM model specification
+            - "context_length": int
+               the maximum text length the LLM model can accommodate (include all input & output)
 
         Raises
         ------
diff --git a/xinference/core/worker.py b/xinference/core/worker.py
--- a/xinference/core/worker.py
+++ b/xinference/core/worker.py
@@ -110,6 +110,7 @@ def _to_llm_description(
             "model_size_in_billions": llm_spec.model_size_in_billions,
             "quantization": quantization,
             "revision": llm_spec.model_revision,
+            "context_length": llm_family.context_length,
         }
 
     @log_sync(logger=logger)
diff --git a/xinference/model/llm/llm_family.py b/xinference/model/llm/llm_family.py
--- a/xinference/model/llm/llm_family.py
+++ b/xinference/model/llm/llm_family.py
@@ -58,6 +58,7 @@ class PromptStyleV1(BaseModel):
 
 class LLMFamilyV1(BaseModel):
     version: Literal[1]
+    context_length: int
     model_name: str
     model_lang: List[Literal["en", "zh"]]
     model_ability: List[Literal["embed", "generate", "chat"]]


Test patch:
diff --git a/xinference/core/tests/test_restful_api.py b/xinference/core/tests/test_restful_api.py
--- a/xinference/core/tests/test_restful_api.py
+++ b/xinference/core/tests/test_restful_api.py
@@ -193,6 +193,7 @@ async def test_restful_api(setup):
 
     model = """{
   "version": 1,
+  "context_length":2048,
   "model_name": "custom_model",
   "model_lang": [
     "en", "zh"
diff --git a/xinference/deploy/test/test_cmdline.py b/xinference/deploy/test/test_cmdline.py
--- a/xinference/deploy/test/test_cmdline.py
+++ b/xinference/deploy/test/test_cmdline.py
@@ -141,6 +141,7 @@ def test_cmdline_of_custom_model(setup):
     # register custom model
     custom_model_desc = """{
   "version": 1,
+  "context_length":2048,
   "model_name": "custom_model",
   "model_lang": [
     "en", "zh"
diff --git a/xinference/model/llm/ggml/tests/test_chatglm.py b/xinference/model/llm/ggml/tests/test_chatglm.py
--- a/xinference/model/llm/ggml/tests/test_chatglm.py
+++ b/xinference/model/llm/ggml/tests/test_chatglm.py
@@ -50,6 +50,7 @@ def load(self):
 
 serialized = """{
    "version":1,
+   "context_length":2048,
    "model_name":"TestModel",
    "model_lang":[
       "en"
diff --git a/xinference/model/llm/ggml/tests/test_ctransformers.py b/xinference/model/llm/ggml/tests/test_ctransformers.py
--- a/xinference/model/llm/ggml/tests/test_ctransformers.py
+++ b/xinference/model/llm/ggml/tests/test_ctransformers.py
@@ -30,6 +30,7 @@
 
 test_model_spec = """{
    "version":1,
+   "context_length":2048,
    "model_name":"TestModel",
    "model_lang":[
       "en"
diff --git a/xinference/model/llm/tests/test_llm_family.py b/xinference/model/llm/tests/test_llm_family.py
--- a/xinference/model/llm/tests/test_llm_family.py
+++ b/xinference/model/llm/tests/test_llm_family.py
@@ -26,6 +26,7 @@
 def test_deserialize_llm_family_v1():
     serialized = """{
    "version":1,
+   "context_length":2048,
    "model_name":"TestModel",
    "model_lang":[
       "en"
@@ -61,6 +62,7 @@ def test_deserialize_llm_family_v1():
     model_family = LLMFamilyV1.parse_raw(serialized)
     assert isinstance(model_family, LLMFamilyV1)
     assert model_family.version == 1
+    assert model_family.context_length == 2048
     assert model_family.model_name == "TestModel"
     assert model_family.model_lang == ["en"]
     assert model_family.model_ability == ["embed", "generate"]
@@ -118,6 +120,7 @@ def test_serialize_llm_family_v1():
     )
     llm_family = LLMFamilyV1(
         version=1,
+        context_length=2048,
         model_type="LLM",
         model_name="TestModel",
         model_lang=["en"],
@@ -126,7 +129,7 @@ def test_serialize_llm_family_v1():
         prompt_style=prompt_style,
     )
 
-    expected = """{"version": 1, "model_name": "TestModel", "model_lang": ["en"], "model_ability": ["embed", "generate"], "model_description": null, "model_specs": [{"model_format": "ggmlv3", "model_size_in_billions": 2, "quantizations": ["q4_0", "q4_1"], "model_id": "example/TestModel", "model_revision": "123", "model_file_name_template": "TestModel.{quantization}.ggmlv3.bin", "model_uri": null}, {"model_format": "pytorch", "model_size_in_billions": 3, "quantizations": ["int8", "int4", "none"], "model_id": "example/TestModel", "model_revision": "456", "model_uri": null}], "prompt_style": {"style_name": "ADD_COLON_SINGLE", "system_prompt": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["user", "assistant"], "intra_message_sep": "\\n### ", "inter_message_sep": "\\n### ", "stop": null, "stop_token_ids": null}}"""
+    expected = """{"version": 1, "context_length": 2048, "model_name": "TestModel", "model_lang": ["en"], "model_ability": ["embed", "generate"], "model_description": null, "model_specs": [{"model_format": "ggmlv3", "model_size_in_billions": 2, "quantizations": ["q4_0", "q4_1"], "model_id": "example/TestModel", "model_revision": "123", "model_file_name_template": "TestModel.{quantization}.ggmlv3.bin", "model_uri": null}, {"model_format": "pytorch", "model_size_in_billions": 3, "quantizations": ["int8", "int4", "none"], "model_id": "example/TestModel", "model_revision": "456", "model_uri": null}], "prompt_style": {"style_name": "ADD_COLON_SINGLE", "system_prompt": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["user", "assistant"], "intra_message_sep": "\\n### ", "inter_message_sep": "\\n### ", "stop": null, "stop_token_ids": null}}"""
     assert json.loads(llm_family.json()) == json.loads(expected)
 
 
@@ -151,6 +154,7 @@ def test_cache_from_huggingface_pytorch():
     )
     family = LLMFamilyV1(
         version=1,
+        context_length=2048,
         model_type="LLM",
         model_name="opt",
         model_lang=["en"],
@@ -180,6 +184,7 @@ def test_cache_from_huggingface_ggml():
     )
     family = LLMFamilyV1(
         version=1,
+        context_length=2048,
         model_type="LLM",
         model_name="orca",
         model_lang=["en"],
@@ -215,6 +220,7 @@ def test_cache_from_uri_local():
     )
     family = LLMFamilyV1(
         version=1,
+        context_length=2048,
         model_type="LLM",
         model_name="test",
         model_lang=["en"],
@@ -262,6 +268,7 @@ def test_cache_from_uri_remote():
     )
     family = LLMFamilyV1(
         version=1,
+        context_length=2048,
         model_type="LLM",
         model_name="test",
         model_lang=["en"],
@@ -309,6 +316,7 @@ def test_legacy_cache():
     )
     family = LLMFamilyV1(
         version=1,
+        context_length=2048,
         model_type="LLM",
         model_name="orca",
         model_lang=["en"],
@@ -349,6 +357,7 @@ def test_custom_llm():
     )
     family = LLMFamilyV1(
         version=1,
+        context_length=2048,
         model_type="LLM",
         model_name="custom_model",
         model_lang=["en"],
@@ -380,6 +389,7 @@ def test_persistent_custom_llm():
     )
     family = LLMFamilyV1(
         version=1,
+        context_length=2048,
         model_type="LLM",
         model_name="custom_model",
         model_lang=["en"],
diff --git a/xinference/tests/test_client.py b/xinference/tests/test_client.py
--- a/xinference/tests/test_client.py
+++ b/xinference/tests/test_client.py
@@ -62,6 +62,7 @@ def test_client_custom_model(setup):
 
     model = """{
   "version": 1,
+  "context_length":2048,
   "model_name": "custom_model",
   "model_lang": [
     "en", "zh"
@@ -193,6 +194,7 @@ def test_RESTful_client_custom_model(setup):
 
     model = """{
   "version": 1,
+  "context_length":2048,
   "model_name": "custom_model",
   "model_lang": [
     "en", "zh"