Repository: xorbitsai/inference
Pull request: FEAT: Support Ctransformers
Resolves #220 

Problem statement:
FEAT: support starcoder


ENH: incorporate ctransformer
### Is your feature request related to a problem? Please describe
We need a backend to support StarCoder, MPT, and falcon in GGML.

### Describe the solution you'd like
A clear and concise description of what you want to happen.

### Describe alternatives you've considered
A clear and concise description of any alternative solutions or features you've considered.

### Additional context
Add any other context or screenshots about the feature request here.

Hints from issue discussion:

Commit message: add GPU check to make sure only supported model can initialize Cuda.

Patch to review:
diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml
index 810939afbd..28741ea347 100644
--- a/.github/workflows/python.yaml
+++ b/.github/workflows/python.yaml
@@ -92,6 +92,7 @@ jobs:
           pip install sentencepiece
           pip install transformers_stream_generator
           pip install bitsandbytes
+          pip install ctransformers
           pip install -e ".[dev]"
         working-directory: .
 
diff --git a/setup.cfg b/setup.cfg
index 143cef1acb..6b81d1fb19 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -52,6 +52,7 @@ dev =
     pytest-timeout>=1.2.0
     pytest-forked>=1.0
     pytest-asyncio>=0.14.0
+    pytest-mock>=3.11.1
     ipython>=6.5.0
     sphinx>=3.0.0,<5.0.0
     pydata-sphinx-theme>=0.3.0
@@ -60,6 +61,7 @@ dev =
     flake8>=3.8.0
     black
 all =
+    ctransformers
     llama-cpp-python>=0.1.77
     transformers>=4.31.0
     torch
@@ -72,6 +74,7 @@ all =
     tiktoken
 ggml =
     llama-cpp-python>=0.1.77
+    ctransformers
 pytorch =
     transformers>=4.31.0
     torch
diff --git a/xinference/model/llm/__init__.py b/xinference/model/llm/__init__.py
index 89d9c1a0b4..14ca84554d 100644
--- a/xinference/model/llm/__init__.py
+++ b/xinference/model/llm/__init__.py
@@ -35,6 +35,7 @@
 
 def _install():
     from .ggml.chatglm import ChatglmCppChatModel
+    from .ggml.ctransformers import CtransformersModel
     from .ggml.llamacpp import LlamaCppChatModel, LlamaCppModel
     from .pytorch.baichuan import BaichuanPytorchChatModel
     from .pytorch.chatglm import ChatglmPytorchChatModel
@@ -54,6 +55,7 @@ def _install():
             FalconPytorchModel,
             FalconPytorchChatModel,
             ChatglmPytorchChatModel,
+            CtransformersModel,
         ]
     )
 
diff --git a/xinference/model/llm/ggml/ctransformers.py b/xinference/model/llm/ggml/ctransformers.py
new file mode 100644
index 0000000000..3a5984dac2
--- /dev/null
+++ b/xinference/model/llm/ggml/ctransformers.py
@@ -0,0 +1,277 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+from typing import TYPE_CHECKING, Iterator, Optional, Sequence, TypedDict, Union
+
+if TYPE_CHECKING:
+    from ctransformers import AutoConfig
+
+from ....types import Completion, CompletionChunk
+from ..core import LLM
+from ..llm_family import LLMFamilyV1, LLMSpecV1
+from .ctransformers_util import generate_stream
+
+logger = logging.getLogger(__name__)
+
+# all supported models for Ctransformers with their model type.
+# Please Strictly follows this name format when inputting new model to model_family.
+MODEL_TYPE_FOR_CTRANSFORMERS = {
+    "gpt-2": "gpt2",
+    "gpt-j": "gptj",
+    "gpt4all-j": "gptj",
+    "gpt-neox": "gpt_neox",
+    "stablelm": "gpt_neox",
+    "llama": "llama",
+    "llama-2": "llama",
+    "mpt": "mpt",
+    "dolly-v2": "dolly-v2",
+    "replit": "replit",
+    "starcoder": "starcoder",
+    "starchat": "starcoder",
+    "falcon": "falcon",
+}
+
+# these two constants subjects to change for future development and ctransformers updates.
+CTRANSFORMERS_SUPPORTED_MODEL = ["starcoder", "gpt-2"]
+
+CTRANSFORMERS_GPU_SUPPORT = ["llama", "llama-2", "mpt", "falcon"]
+
+SIZE_TO_GPU_LAYERS = {
+    3: 26,
+    7: 32,
+    13: 40,
+    30: 60,
+    65: 80,
+}
+
+
+class CtransformersModelConfig(TypedDict, total=False):
+    n_ctx: int
+    n_gpu_layers: int
+
+
+class CtransformersGenerateConfig(TypedDict, total=False):
+    max_tokens: Optional[int]
+    top_k: Optional[int]
+    top_p: Optional[float]
+    temperature: Optional[float]
+    repetition_penalty: Optional[float]
+    last_n_tokens: Optional[int]
+    seed: Optional[int]
+    batch_size: Optional[int]
+    threads: Optional[int]
+    stop: Optional[Sequence[str]]
+    stream: Optional[bool]
+    reset: Optional[bool]
+
+
+def _has_cuda_device():
+    from xorbits._mars.resource import cuda_count
+
+    return cuda_count() > 0
+
+
+class CtransformersModel(LLM):
+    def __init__(
+        self,
+        model_uid: str,
+        model_family: "LLMFamilyV1",
+        model_spec: "LLMSpecV1",
+        quantization: str,
+        model_path: str,
+        ctransformers_Model_Config: Optional[CtransformersModelConfig],
+    ):
+        super().__init__(model_uid, model_family, model_spec, quantization, model_path)
+
+        self._model_type = None
+        closest_size = min(
+            SIZE_TO_GPU_LAYERS.keys(),
+            key=lambda x: abs(x - model_spec.model_size_in_billions),
+        )
+
+        self._model_family = model_family
+        self._model_uid = model_uid
+        self._llm = None
+
+        self._gpu_layers = SIZE_TO_GPU_LAYERS[closest_size]
+        self._ctransformer_model_config = self._sanitize_model_config(
+            model_path, ctransformers_Model_Config
+        )
+
+    def _sanitize_model_config(
+        self, model_path, ctransformers_model_config: Optional[CtransformersModelConfig]
+    ) -> "AutoConfig":
+        try:
+            from ctransformers import AutoConfig, Config
+        except ImportError:
+            error_message = (
+                "Failed to import module 'ctransformers - AutoConfig and Config'"
+            )
+
+            installation_guide = [
+                f"Please make sure 'ctransformers' is installed.",
+                f"You can install it by checking out the repository for command:"
+                f"https://github.com/marella/ctransformers",
+            ]
+
+            raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+
+        # if the model have customized config, we update it.
+        ctransformers_model_config_returned = Config()
+        potential_gpu_layers = None
+        if ctransformers_model_config:
+            potential_context_length = ctransformers_model_config.pop("n_ctx", None)
+            potential_gpu_layers = ctransformers_model_config.pop("n_gpu_layers", None)
+
+            ctransformers_model_config_returned.context_length = (
+                potential_context_length
+            )
+            ctransformers_model_config_returned.gpu_layers = potential_gpu_layers
+
+        # if user does not define gpu layers, we have to set it with our system if applicable.
+        if potential_gpu_layers is None:
+            if self._model_family.model_name not in CTRANSFORMERS_GPU_SUPPORT:
+                ctransformers_model_config_returned.gpu_layers = -1
+            elif self._is_darwin_and_apple_silicon():
+                ctransformers_model_config_returned.gpu_layers = 1
+            elif _has_cuda_device():
+                ctransformers_model_config_returned.gpu_layers = self._gpu_layers
+
+        return AutoConfig(ctransformers_model_config_returned)
+
+    def _sanitize_generate_config(
+        self,
+        ctransformers_generate_config: Optional[CtransformersGenerateConfig],
+    ) -> CtransformersGenerateConfig:
+        # if the input config is not None, we try to copy the selected attributes to the ctransformersGenerateConfig.
+        if ctransformers_generate_config is None:
+            ctransformers_generate_config = CtransformersGenerateConfig()
+
+        # for our system, the threads will have to be set to 4
+        # all other parameters, if not specified, will be set to default when generate.
+        ctransformers_generate_config.setdefault("threads", 4)
+
+        return ctransformers_generate_config
+
+    def load(self):
+        try:
+            from ctransformers import AutoModelForCausalLM
+        except ImportError:
+            error_message = "Failed to import module 'ctransformers'"
+
+            installation_guide = [
+                f"Please make sure 'ctransformers' is installed.",
+                f"You can install it by checking out the repository for command."
+                f"https://github.com/marella/ctransformers",
+            ]
+
+            raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+
+        model_path = os.path.join(
+            self.model_path,
+            self.model_spec.model_file_name_template.format(
+                quantization=self.quantization
+            ),
+        )
+
+        self._model_type = self._determine_model_type()
+        self._llm = AutoModelForCausalLM.from_pretrained(
+            model_path_or_repo_id=model_path,
+            model_type=self._model_type,
+            config=self._ctransformer_model_config,
+        )
+
+    @classmethod
+    def match(cls, llm_family: LLMFamilyV1, llm_spec: LLMSpecV1) -> bool:
+        if llm_spec.model_format != "ggmlv3":
+            return False
+        if llm_family.model_name not in CTRANSFORMERS_SUPPORTED_MODEL:
+            return False
+        if "generate" not in llm_family.model_ability:
+            return False
+        return True
+
+    def _determine_model_type(self):
+        if self._model_family.model_name not in MODEL_TYPE_FOR_CTRANSFORMERS:
+            raise ValueError(
+                f"The current model {self._model_family.model_name} is not supported, check your model name. "
+            )
+        return MODEL_TYPE_FOR_CTRANSFORMERS[self._model_family.model_name]
+
+    def generate(
+        self, prompt: str, generate_config_raw: CtransformersGenerateConfig
+    ) -> Union[Completion, Iterator[CompletionChunk]]:
+        def generator_wrapper(
+            _prompt: str,
+            _max_new_tokens: Union[int, None],
+            _generate_config: CtransformersGenerateConfig,
+        ) -> Iterator[CompletionChunk]:
+            assert self._model_uid is not None
+            for _completion_chunk, _ in generate_stream(
+                model=self._model_uid,
+                model_ref=self._llm,
+                prompt=_prompt,
+                max_new_tokens=_max_new_tokens,
+                **_generate_config,
+            ):
+                yield _completion_chunk
+
+        generate_config = self._sanitize_generate_config(generate_config_raw)
+        max_new_tokens = generate_config.pop("max_tokens", None)
+
+        logger.debug(
+            "Enter generate, prompt: %s, generate config: %s", prompt, generate_config
+        )
+
+        stream_or_not = generate_config.get("stream", False)
+        if stream_or_not:
+            return generator_wrapper(
+                _prompt=prompt,
+                _max_new_tokens=max_new_tokens,
+                _generate_config=generate_config,
+            )
+        else:
+            assert self.model_uid is not None
+            completion_chunk = None
+            completion_usage = None
+            for completion_chunk, completion_usage in generate_stream(
+                model=self.model_uid,
+                model_ref=self._llm,
+                prompt=prompt,
+                max_new_tokens=max_new_tokens,
+                **generate_config,
+            ):
+                pass
+
+            assert completion_chunk is not None
+            assert completion_usage is not None
+
+            completion = Completion(
+                id=completion_chunk["id"],
+                object=completion_chunk["object"],
+                created=completion_chunk["created"],
+                model=completion_chunk["model"],
+                choices=completion_chunk["choices"],
+                usage=completion_usage,
+            )
+
+            logger.debug(
+                "Generated, completion: %s, generate config: %s",
+                completion,
+                generate_config,
+            )
+
+            return completion
diff --git a/xinference/model/llm/ggml/ctransformers_util.py b/xinference/model/llm/ggml/ctransformers_util.py
new file mode 100644
index 0000000000..33a14705be
--- /dev/null
+++ b/xinference/model/llm/ggml/ctransformers_util.py
@@ -0,0 +1,165 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import re
+import time
+import uuid
+from typing import Iterator, Optional, Sequence, Tuple
+
+from ....types import CompletionChoice, CompletionChunk, CompletionUsage
+
+logger = logging.getLogger(__name__)
+
+
+def generate_stream(
+    model,
+    model_ref,
+    prompt: str,
+    *,
+    max_new_tokens: Optional[int] = None,
+    top_k: Optional[int] = None,
+    top_p: Optional[float] = None,
+    temperature: Optional[float] = None,
+    repetition_penalty: Optional[float] = None,
+    last_n_tokens: Optional[int] = None,
+    seed: Optional[int] = None,
+    batch_size: Optional[int] = None,
+    stream: Optional[bool] = False,
+    threads: Optional[int] = None,
+    stop: Optional[Sequence[str]] = None,
+    reset: Optional[bool] = None,
+    **kwargs,
+) -> Iterator[Tuple[CompletionChunk, CompletionUsage]]:
+    stop = stop or []
+    if isinstance(stop, str):
+        stop = [stop]
+
+    tokens = model_ref.tokenize(prompt)
+
+    stop_regex = re.compile("|".join(map(re.escape, stop)))
+    count = 0
+    text = ""
+    total_text = ""
+    incomplete = b""
+
+    # parameters needed for Xinference.
+    finish_reason = None
+
+    for token in model_ref.generate(
+        tokens,
+        top_k=top_k,
+        top_p=top_p,
+        temperature=temperature,
+        repetition_penalty=repetition_penalty,
+        last_n_tokens=last_n_tokens,
+        seed=seed,
+        batch_size=batch_size,
+        threads=threads,
+        reset=reset,
+    ):
+        # Handle incomplete UTF-8 multi-byte characters.
+        try:
+            from ctransformers.utils import utf8_split_incomplete
+        except ImportError:
+            error_message = (
+                "Failed to import module 'ctransformers - utf8_split_incomplete'"
+            )
+
+            installation_guide = [
+                "Please make sure 'ctransformers' is installed. You can install it by checking out the repository: "
+                "https://github.com/marella/ctransformers",
+            ]
+
+            raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+
+        incomplete += model_ref.detokenize([token], decode=False)
+        complete, incomplete = utf8_split_incomplete(incomplete)
+        output = complete.decode(errors="ignore")
+        text += output
+        total_text += output
+
+        logger.debug("Output, completion: %s", text)
+
+        # https://github.com/abetlen/llama-cpp-python/blob/1a13d76c487df1c8560132d10bda62d6e2f4fa93/llama_cpp/llama.py#L686-L706
+        # Check if one of the stop sequences is part of the text.
+        # Note that the stop sequence may not always be at the end of text.
+        if stop:
+            match = stop_regex.search(text)
+            if match:
+                text = text[: match.start()]
+                finish_reason = "stop"
+                break
+
+        # Avoid sending the longest suffix of text which is also a prefix
+        # of a stop sequence, as it can form a stop sequence with the text
+        # generated later.
+        longest = 0
+        for s in stop:
+            for i in range(len(s), 0, -1):
+                if text.endswith(s[:i]):
+                    longest = max(i, longest)
+                    break
+
+        end = len(text) - longest
+        if end > 0:
+            output = text[:end]
+            completion_choice = CompletionChoice(
+                text=output, index=0, logprobs=None, finish_reason=None
+            )
+            completion_chunk = CompletionChunk(
+                id=str(uuid.uuid1()),
+                object="text_completion",
+                created=int(time.time()),
+                model=model,
+                choices=[completion_choice],
+            )
+            completion_usage = CompletionUsage(
+                prompt_tokens=len(tokens),
+                completion_tokens=count + 1,
+                total_tokens=count + 1 + len(tokens),
+            )
+
+            yield completion_chunk, completion_usage
+            text = text[end:]
+
+        count += 1
+        if max_new_tokens is not None and count >= max_new_tokens:
+            finish_reason = "length"
+            break
+
+    if stream is False:
+        completion_choice = CompletionChoice(
+            text=total_text, index=0, logprobs=None, finish_reason=finish_reason
+        )
+    else:
+        completion_choice = CompletionChoice(
+            text=text, index=0, logprobs=None, finish_reason=finish_reason
+        )
+
+    completion_chunk = CompletionChunk(
+        id=str(uuid.uuid1()),
+        object="text_completion",
+        created=int(time.time()),
+        model=model,
+        choices=[completion_choice],
+    )
+    completion_usage = CompletionUsage(
+        prompt_tokens=len(tokens),
+        completion_tokens=count,
+        total_tokens=count + len(tokens),
+    )
+
+    logger.debug("Completionchoice: %s", completion_choice)
+
+    yield completion_chunk, completion_usage
diff --git a/xinference/model/llm/ggml/llamacpp.py b/xinference/model/llm/ggml/llamacpp.py
index bea3e4744e..4bcee28059 100644
--- a/xinference/model/llm/ggml/llamacpp.py
+++ b/xinference/model/llm/ggml/llamacpp.py
@@ -28,6 +28,7 @@
 from ..core import LLM
 from ..llm_family import LLMFamilyV1, LLMSpecV1
 from ..utils import ChatModelMixin
+from .ctransformers import CTRANSFORMERS_SUPPORTED_MODEL
 
 if TYPE_CHECKING:
     from llama_cpp import LogitsProcessorList, StoppingCriteriaList
@@ -187,7 +188,10 @@ def load(self):
     def match(cls, llm_family: LLMFamilyV1, llm_spec: LLMSpecV1) -> bool:
         if llm_spec.model_format != "ggmlv3":
             return False
-        if "chatglm" in llm_family.model_name:
+        if (
+            "chatglm" in llm_family.model_name
+            or llm_family.model_name in CTRANSFORMERS_SUPPORTED_MODEL
+        ):
             return False
         if "generate" not in llm_family.model_ability:
             return False
@@ -258,7 +262,10 @@ def __init__(
     def match(cls, llm_family: LLMFamilyV1, llm_spec: LLMSpecV1) -> bool:
         if llm_spec.model_format != "ggmlv3":
             return False
-        if "chatglm" in llm_family.model_name:
+        if (
+            "chatglm" in llm_family.model_name
+            or llm_family.model_name in CTRANSFORMERS_SUPPORTED_MODEL
+        ):
             return False
         if "chat" not in llm_family.model_ability:
             return False
diff --git a/xinference/model/llm/ggml/tests/test_ctransformers.py b/xinference/model/llm/ggml/tests/test_ctransformers.py
new file mode 100644
index 0000000000..d13b2377fb
--- /dev/null
+++ b/xinference/model/llm/ggml/tests/test_ctransformers.py
@@ -0,0 +1,166 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import random
+import string
+
+import pytest
+
+from .....client import Client, GenerateModelHandle
+from ....llm import GgmlLLMSpecV1, LLMFamilyV1
+from ..ctransformers import CtransformersModel
+
+
+class MockCtransformersModel(CtransformersModel):
+    def load(self):
+        pass
+
+
+mock_model_spec = GgmlLLMSpecV1(
+    model_format="ggmlv3",
+    model_size_in_billions=6,
+    quantizations=["q2_k", "q4_0"],
+    model_id="test_id",
+    model_file_name_template="TestModel.{quantization}.ggmlv3.bin",
+)
+
+test_model_spec = """{
+   "version":1,
+   "model_name":"TestModel",
+   "model_lang":[
+      "en"
+   ],
+   "model_ability":[
+      "embed", "generate"
+   ],
+   "model_specs":[
+      {
+         "model_format":"ggmlv3",
+         "model_size_in_billions":6,
+         "quantizations": ["q2_k", "q4_0"],
+         "model_id":"test_id",
+         "model_file_name_template":"TestModel.{quantization}.ggmlv3.bin"
+      },
+      {
+         "model_format":"pytorch",
+         "model_size_in_billions":3,
+         "quantizations": ["int8", "int4", "none"],
+         "model_id":"example/TestModel"
+      }
+   ],
+   "prompt_style": null
+}"""
+
+mock_model_family = LLMFamilyV1.parse_raw(test_model_spec)
+
+
+@pytest.mark.parametrize(
+    "model_spec, model_family", [(mock_model_spec, mock_model_family)]
+)
+def test_ctransformer_init(model_spec, model_family):
+    from ctransformers import AutoConfig
+
+    quantization = "q4_0"
+    uid = "".join(random.choice(string.digits) for i in range(15))
+    path = "".join(
+        random.choice(string.ascii_letters + string.punctuation) for i in range(100)
+    )
+    model = MockCtransformersModel(
+        model_uid=uid,
+        model_family=model_family,
+        model_spec=model_spec,
+        quantization=quantization,
+        model_path=path,
+        ctransformers_Model_Config=None,
+    )
+
+    assert model.model_uid == uid
+    assert model.quantization == quantization
+    assert model.model_path == path
+    assert model._ctransformer_model_config is not None
+    assert isinstance(model._ctransformer_model_config, AutoConfig)
+
+    assert isinstance(model.model_spec, GgmlLLMSpecV1)
+    assert isinstance(model.model_family, LLMFamilyV1)
+    assert isinstance(model.model_family.model_specs[0], GgmlLLMSpecV1)
+
+    assert (
+        model.model_family.model_specs[0].model_format == model.model_spec.model_format
+    )
+    assert model.model_family.model_specs[0].model_format == model_spec.model_format
+    assert (
+        model.model_family.model_specs[0].model_size_in_billions
+        == model.model_spec.model_size_in_billions
+    )
+    assert (
+        model.model_family.model_specs[0].model_size_in_billions
+        == model_spec.model_size_in_billions
+    )
+    assert (
+        model.model_family.model_specs[0].quantizations
+        == model.model_spec.quantizations
+    )
+    assert model.model_family.model_specs[0].quantizations == model_spec.quantizations
+    assert model.model_family.model_specs[0].model_id == model.model_spec.model_id
+    assert model.model_family.model_specs[0].model_id == model_spec.model_id
+    assert (
+        model.model_family.model_specs[0].model_file_name_template
+        == model.model_spec.model_file_name_template
+    )
+    assert (
+        model.model_family.model_specs[0].model_file_name_template
+        == model_spec.model_file_name_template
+    )
+    assert model._llm is None
+
+
+@pytest.mark.asyncio
+async def test_ctransformers_generate(setup):
+    endpoint, _ = setup
+    client = Client(endpoint)
+    assert len(client.list_models()) == 0
+
+    model_uid = client.launch_model(
+        model_name="gpt-2",
+        model_size_in_billions=1,
+        model_format="ggmlv3",
+        quantization="none",
+    )
+
+    assert len(client.list_models()) == 1
+
+    model = client.get_model(model_uid=model_uid)
+    assert isinstance(model, GenerateModelHandle)
+
+    completion = model.generate("AI is going to", generate_config={"max_tokens": 5})
+    print(completion)
+    assert "id" in completion
+    assert "text" in completion["choices"][0]
+    assert len(completion["choices"][0]["text"]) > 0
+
+    assert completion["model"] == model_uid
+
+    assert "finish_reason" in completion["choices"][0]
+    assert completion["choices"][0]["finish_reason"] == "length"
+
+    assert "prompt_tokens" in completion["usage"]
+    assert completion["usage"]["prompt_tokens"] == 4
+
+    assert "completion_tokens" in completion["usage"]
+    assert completion["usage"]["completion_tokens"] == 5
+
+    assert "total_tokens" in completion["usage"]
+    assert completion["usage"]["total_tokens"] == 9
+
+    client.terminate_model(model_uid=model_uid)
+    assert len(client.list_models()) == 0
diff --git a/xinference/model/llm/llm_family.json b/xinference/model/llm/llm_family.json
index ae481f2245..8e709182f2 100644
--- a/xinference/model/llm/llm_family.json
+++ b/xinference/model/llm/llm_family.json
@@ -744,7 +744,8 @@
     "version": 1,
     "model_name": "qwen-chat",
     "model_lang": [
-      "en", "zh"
+      "en",
+      "zh"
     ],
     "model_ability": [
       "embed",
@@ -774,5 +775,53 @@
         151643
       ]
     }
+  },
+  {
+    "version": 1,
+    "model_name": "starcoder",
+    "model_lang": [
+      "en"
+    ],
+    "model_ability":[
+      "generate"
+    ],
+    "model_specs": [
+      {
+        "model_format": "ggmlv3",
+        "model_size_in_billions": 16,
+        "quantizations": [
+          "q4_0",
+          "q4_1",
+          "q5_0",
+          "q5_1",
+          "q8_0"
+        ],
+        "model_id": "TheBloke/starcoder-GGML",
+        "model_file_name_template": "starcoder.ggmlv3.{quantization}.bin"
+      }
+    ],
+    "prompt_style": null
+  },
+  {
+    "version": 1,
+    "model_name": "gpt-2",
+    "model_lang": [
+      "en"
+    ],
+    "model_ability":[
+      "generate"
+    ],
+    "model_specs": [
+      {
+        "model_format": "ggmlv3",
+        "model_size_in_billions": 1,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "marella/gpt-2-ggml",
+        "model_file_name_template": "ggml-model.bin"
+      }
+    ],
+    "prompt_style": null
   }
 ]


Code review comments:
diff_hunk: @@ -0,0 +1,277 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+from typing import TYPE_CHECKING, Iterator, Optional, Sequence, TypedDict, Union
+
+if TYPE_CHECKING:
+    from ctransformers import AutoConfig
+
+from ....types import Completion, CompletionChunk
+from ..core import LLM
+from ..llm_family import LLMFamilyV1, LLMSpecV1
+from .ctransformers_util import generate_stream
+
+logger = logging.getLogger(__name__)
+
+# all supported models for Ctransformers with their model type.
+# Please Strictly follows this name format when inputting new model to model_family.
+MODEL_TYPE_FOR_CTRANSFORMERS = {
+    "gpt-2": "gpt2",
+    "gpt-j": "gptj",
+    "gpt4all-j": "gptj",
+    "gpt-neox": "gpt_neox",
+    "stablelm": "gpt_neox",
+    "llama": "llama",
+    "llama-2": "llama",
+    "mpt": "mpt",
+    "dolly-v2": "dolly-v2",
+    "replit": "replit",
+    "starcoder": "starcoder",
+    "starchat": "starcoder",
+    "falcon": "falcon",
+}
+
+# these two constants subjects to change for future development and ctransformers updates.
+CTRANSFORMERS_SUPPORTED_MODEL = ["starcoder", "gpt-2"]
+
+CTRANSFORMERS_GPU_SUPPORT = ["llama", "llama-2", "mpt", "falcon"]
+
+SIZE_TO_GPU_LAYERS = {
+    3: 26,
+    7: 32,
+    13: 40,
+    30: 60,
+    65: 80,
+}
+
+
+class CtransformersModelConfig(TypedDict, total=False):
+    n_ctx: int
+    n_gpu_layers: int
+
+
+class CtransformersGenerateConfig(TypedDict, total=False):
+    max_tokens: Optional[int]
+    top_k: Optional[int]
+    top_p: Optional[float]
+    temperature: Optional[float]
+    repetition_penalty: Optional[float]
+    last_n_tokens: Optional[int]
+    seed: Optional[int]
+    batch_size: Optional[int]
+    threads: Optional[int]
+    stop: Optional[Sequence[str]]
+    stream: Optional[bool]
+    reset: Optional[bool]
+
+
+def _has_cuda_device():
+    from xorbits._mars.resource import cuda_count
+
+    return cuda_count() > 0
+
+
+class CtransformersModel(LLM):
+    def __init__(
+        self,
+        model_uid: str,
+        model_family: "LLMFamilyV1",
+        model_spec: "LLMSpecV1",
+        quantization: str,
+        model_path: str,
+        ctransformers_Model_Config: Optional[CtransformersModelConfig],
+    ):
+        super().__init__(model_uid, model_family, model_spec, quantization, model_path)
+
+        self._model_type = None
+        closest_size = min(
+            SIZE_TO_GPU_LAYERS.keys(),
+            key=lambda x: abs(x - model_spec.model_size_in_billions),
+        )
+
+        self._model_family = model_family
+        self._model_uid = model_uid
+        self._llm = None
+
+        self._gpu_layers = SIZE_TO_GPU_LAYERS[closest_size]
+        self._ctransformer_model_config = self._sanitize_model_config(
+            model_path, ctransformers_Model_Config
+        )
+
+    def _sanitize_model_config(
+        self, model_path, ctransformers_model_config: Optional[CtransformersModelConfig]
+    ) -> "AutoConfig":
+        try:
+            from ctransformers import AutoConfig, Config
+        except ImportError:
+            error_message = (
+                "Failed to import module 'ctransformers - AutoConfig and Config'"
+            )
+
+            installation_guide = [
+                f"Please make sure 'ctransformers' is installed.",
+                f"You can install it by checking out the repository for command:"
+                f"https://github.com/marella/ctransformers",
+            ]
+
+            raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+
+        # if the model have customized config, we update it.
+        ctransformers_model_config_returned = Config()
+        potential_gpu_layers = None
+        if ctransformers_model_config:
+            potential_context_length = ctransformers_model_config.pop("n_ctx", None)
+            potential_gpu_layers = ctransformers_model_config.pop("n_gpu_layers", None)
+
+            ctransformers_model_config_returned.context_length = (
path: xinference/model/llm/ggml/ctransformers.py
text: @user1:
The redundant parentheses could be misleading.

@author:
changed in the latest version

diff_hunk: @@ -0,0 +1,277 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+from typing import TYPE_CHECKING, Iterator, Optional, Sequence, TypedDict, Union
+
+if TYPE_CHECKING:
+    from ctransformers import AutoConfig
+
+from ....types import Completion, CompletionChunk
+from ..core import LLM
+from ..llm_family import LLMFamilyV1, LLMSpecV1
+from .ctransformers_util import generate_stream
+
+logger = logging.getLogger(__name__)
+
+# all supported models for Ctransformers with their model type.
+# Please Strictly follows this name format when inputting new model to model_family.
+MODEL_TYPE_FOR_CTRANSFORMERS = {
+    "gpt-2": "gpt2",
+    "gpt-j": "gptj",
+    "gpt4all-j": "gptj",
+    "gpt-neox": "gpt_neox",
+    "stablelm": "gpt_neox",
+    "llama": "llama",
+    "llama-2": "llama",
+    "mpt": "mpt",
+    "dolly-v2": "dolly-v2",
+    "replit": "replit",
+    "starcoder": "starcoder",
+    "starchat": "starcoder",
+    "falcon": "falcon",
+}
+
+# these two constants subjects to change for future development and ctransformers updates.
+CTRANSFORMERS_SUPPORTED_MODEL = ["starcoder", "gpt-2"]
+
+CTRANSFORMERS_GPU_SUPPORT = ["llama", "llama-2", "mpt", "falcon"]
+
+SIZE_TO_GPU_LAYERS = {
+    3: 26,
+    7: 32,
+    13: 40,
+    30: 60,
+    65: 80,
+}
+
+
+class CtransformersModelConfig(TypedDict, total=False):
+    n_ctx: int
+    n_gpu_layers: int
+
+
+class CtransformersGenerateConfig(TypedDict, total=False):
+    max_tokens: Optional[int]
+    top_k: Optional[int]
+    top_p: Optional[float]
+    temperature: Optional[float]
+    repetition_penalty: Optional[float]
+    last_n_tokens: Optional[int]
+    seed: Optional[int]
+    batch_size: Optional[int]
+    threads: Optional[int]
+    stop: Optional[Sequence[str]]
+    stream: Optional[bool]
+    reset: Optional[bool]
+
+
+def _has_cuda_device():
+    from xorbits._mars.resource import cuda_count
+
+    return cuda_count() > 0
+
+
+class CtransformersModel(LLM):
+    def __init__(
+        self,
+        model_uid: str,
+        model_family: "LLMFamilyV1",
+        model_spec: "LLMSpecV1",
+        quantization: str,
+        model_path: str,
+        ctransformers_Model_Config: Optional[CtransformersModelConfig],
path: xinference/model/llm/ggml/ctransformers.py
text: @user1:
`ctransformers_Model_Config` -> `ctransformers_model_config`

@author:
changed in the latest version