Spaces:
Sleeping
Sleeping
Commit
•
6f13f13
1
Parent(s):
b7ebcbb
Update main.py
Browse files
main.py
CHANGED
@@ -4,12 +4,15 @@ import requests
|
|
4 |
from ctransformers import AutoModelForCausalLM
|
5 |
|
6 |
llms = {
|
7 |
-
"
|
|
|
|
|
|
|
8 |
"orca2":{"name": "TheBloke/Orca-2-7B-GGUF", "file":"orca-2-7b.Q4_K_M.gguf", "suffix":"<|im_end|><|im_start|>assistant", "prefix":"<|im_start|>system You are a helpful assistant<|im_end|><|im_start|>user "},
|
9 |
"zephyr":{"name": "TheBloke/zephyr-7B-beta-GGUF", "file":"zephyr-7b-beta.Q4_K_M.gguf", "suffix":"</s><|assistant|>", "prefix":"<|system|>You are a helpful assistant</s><|user|> "},
|
10 |
"mixtral":{"name": "TheBloke/Mistral-7B-Instruct-v0.1-GGUF", "file":"mistral-7b-instruct-v0.1.Q4_K_M.gguf", "suffix":"[/INST]", "prefix":"<s>[INST] "},
|
11 |
"llama2":{"name": "TheBloke/Llama-2-7B-Chat-GGUF", "file":"llama-2-7b-chat.Q4_K_M.gguf", "suffix":"[/INST]", "prefix":"[INST] <<SYS>> You are a helpful assistant <</SYS>>"},
|
12 |
-
"
|
13 |
}
|
14 |
|
15 |
for k in llms.keys():
|
@@ -24,8 +27,6 @@ app = FastAPI()
|
|
24 |
|
25 |
@app.post("/llm_on_cpu")
|
26 |
async def stream(item: validation):
|
27 |
-
|
28 |
-
|
29 |
|
30 |
prefix=llms[item.llm]['prefix']
|
31 |
suffix=llms[item.llm]['suffix']
|
|
|
4 |
from ctransformers import AutoModelForCausalLM
|
5 |
|
6 |
llms = {
|
7 |
+
"tinyllama":{"name": "TheBloke/TinyLlama-1.1B-1T-OpenOrca-GGUF", "file":"tinyllama-1.1b-1t-openorca.Q4_K_M.gguf", "suffix":"<|im_end|><|im_start|>assistant", "prefix":"<|im_start|>system You are a helpful assistant <|im_end|><|im_start|>user"},
|
8 |
+
"tinyllama2":{"name": "TheBloke/TinyLlama-1.1B-1T-OpenOrca-GGUF", "file":"tinyllama-1.1b-1t-openorca.Q3_K_M.gguf", "suffix":"<|im_end|><|im_start|>assistant", "prefix":"<|im_start|>system You are a helpful assistant <|im_end|><|im_start|>user"},
|
9 |
+
"tinyllama3":{"name": "TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF", "file":"tinyllama-1.1b-chat-v0.3.Q4_K_M.gguf", "suffix":"<|im_end|><|im_start|>assistant", "prefix":"<|im_start|>system You are a helpful assistant <|im_end|><|im_start|>user"},
|
10 |
+
"tinyllama4":{"name": "TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF", "file":"tinyllama-1.1b-chat-v0.3.Q2_K.gguf", "suffix":"<|im_end|><|im_start|>assistant", "prefix":"<|im_start|>system You are a helpful assistant <|im_end|><|im_start|>user"},
|
11 |
"orca2":{"name": "TheBloke/Orca-2-7B-GGUF", "file":"orca-2-7b.Q4_K_M.gguf", "suffix":"<|im_end|><|im_start|>assistant", "prefix":"<|im_start|>system You are a helpful assistant<|im_end|><|im_start|>user "},
|
12 |
"zephyr":{"name": "TheBloke/zephyr-7B-beta-GGUF", "file":"zephyr-7b-beta.Q4_K_M.gguf", "suffix":"</s><|assistant|>", "prefix":"<|system|>You are a helpful assistant</s><|user|> "},
|
13 |
"mixtral":{"name": "TheBloke/Mistral-7B-Instruct-v0.1-GGUF", "file":"mistral-7b-instruct-v0.1.Q4_K_M.gguf", "suffix":"[/INST]", "prefix":"<s>[INST] "},
|
14 |
"llama2":{"name": "TheBloke/Llama-2-7B-Chat-GGUF", "file":"llama-2-7b-chat.Q4_K_M.gguf", "suffix":"[/INST]", "prefix":"[INST] <<SYS>> You are a helpful assistant <</SYS>>"},
|
15 |
+
"solar":{"name": "TheBloke/SOLAR-10.7B-Instruct-v1.0-GGUF", "file":"solar-10.7b-instruct-v1.0.Q4_K_M.gguf", "suffix":"\n### Assistant:\n", "prefix":"### User:\n"}
|
16 |
}
|
17 |
|
18 |
for k in llms.keys():
|
|
|
27 |
|
28 |
@app.post("/llm_on_cpu")
|
29 |
async def stream(item: validation):
|
|
|
|
|
30 |
|
31 |
prefix=llms[item.llm]['prefix']
|
32 |
suffix=llms[item.llm]['suffix']
|