Repository: xorbitsai/inference

Issue:
Llama-3-8B支持可以搞起来了
### Is your feature request related to a problem? Please describe
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]

### Describe the solution you'd like
A clear and concise description of what you want to happen.

### Describe alternatives you've considered
A clear and concise description of any alternative solutions or features you've considered.

### Additional context
Add any other context or screenshots about the feature request here.



Discussion:

Resolving patch:
diff --git a/xinference/model/llm/utils.py b/xinference/model/llm/utils.py
--- a/xinference/model/llm/utils.py
+++ b/xinference/model/llm/utils.py
@@ -114,6 +114,22 @@ def get_role(role_name: str):
                 else:
                     ret += role
             return ret
+        elif prompt_style.style_name == "LLAMA3":
+            ret = (
+                f"

system
"
+                f"{prompt_style.intra_message_sep}{prompt_style.system_prompt}{prompt_style.inter_message_sep}"
+            )
+            for i, message in enumerate(chat_history):
+                role = get_role(message["role"])
+                content = message["content"]
+                if content:
+                    ret += (
+                        f"
{role}
"
+                        f"{prompt_style.intra_message_sep}{content}{prompt_style.inter_message_sep}"
+                    )
+                else:
+                    ret += f"
{role}
{prompt_style.intra_message_sep}"
+            return ret
         elif prompt_style.style_name == "FALCON":
             ret = prompt_style.system_prompt
             for message in chat_history:
diff --git a/xinference/model/llm/vllm/core.py b/xinference/model/llm/vllm/core.py
--- a/xinference/model/llm/vllm/core.py
+++ b/xinference/model/llm/vllm/core.py
@@ -85,6 +85,7 @@ class VLLMGenerateConfig(TypedDict, total=False):
 
 VLLM_SUPPORTED_MODELS = [
     "llama-2",
+    "llama-3",
     "baichuan",
     "internlm-16k",
     "mistral-v0.1",
@@ -94,6 +95,7 @@ class VLLMGenerateConfig(TypedDict, total=False):
 ]
 VLLM_SUPPORTED_CHAT_MODELS = [
     "llama-2-chat",
+    "llama-3-instruct",
     "vicuna-v1.3",
     "vicuna-v1.5",
     "baichuan-chat",


Test patch:
diff --git a/xinference/model/llm/tests/test_utils.py b/xinference/model/llm/tests/test_utils.py
--- a/xinference/model/llm/tests/test_utils.py
+++ b/xinference/model/llm/tests/test_utils.py
@@ -140,6 +140,48 @@ def test_prompt_style_llama2():
     )
 
 
+def test_prompt_style_llama3():
+    prompt_style = PromptStyleV1(
+        style_name="LLAMA3",
+        system_prompt=(
+            "You are a helpful, respectful and honest assistant. Always answer"
+            " as helpfully as possible, while being safe. Your answers should not include any"
+            " harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please"
+            " ensure that your responses are socially unbiased and positive in nature.\n\nIf a"
+            " question does not make any sense, or is not factually coherent, explain why instead"
+            " of answering something not correct. If you don't know the answer to a question,"
+            " please don't share false information"
+        ),
+        roles=["user", "assistant"],
+        intra_message_sep="\n\n",
+        inter_message_sep="
",
+        stop_token_ids=[128001, 128009],
+    )
+    chat_history = [
+        ChatCompletionMessage(role=prompt_style.roles[0], content="Hi there."),
+        ChatCompletionMessage(
+            role=prompt_style.roles[1], content="Hello, how may I help you?"
+        ),
+    ]
+    expected = (
+        "

system
\n\n"
+        "You are a helpful, respectful and honest assistant. Always answer"
+        " as helpfully as possible, while being safe. Your answers should not include any"
+        " harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please"
+        " ensure that your responses are socially unbiased and positive in nature.\n\nIf a"
+        " question does not make any sense, or is not factually coherent, explain why instead"
+        " of answering something not correct. If you don't know the answer to a question,"
+        " please don't share false information
"
+        "
user
\n\nHi there.
"
+        "
assistant
\n\nHello, how may I help you?
"
+        "
user
\n\nWrite a poem.
"
+        "
assistant
\n\n"
+    )
+    assert expected == ChatModelMixin.get_prompt(
+        "Write a poem.", chat_history, prompt_style
+    )
+
+
 def test_prompt_style_falcon():
     prompt_style = PromptStyleV1(
         style_name="FALCON",