Spaces:
Runtime error
Runtime error
Update app.py
Browse filesattempting to get model to work
app.py
CHANGED
@@ -6,13 +6,22 @@ from langchain.embeddings import HuggingFaceBgeEmbeddings
|
|
6 |
from langchain.chains import RetrievalQA
|
7 |
import gradio as gr
|
8 |
|
9 |
-
local_llm = "https://huggingface.co/TheBloke/zephyr-7B-beta-GGUF/blob/main/zephyr-7b-beta.Q5_K_S.gguf"
|
10 |
|
11 |
# Load model directly
|
12 |
# from transformers import AutoModel
|
13 |
# local_llm = AutoModel.from_pretrained("TheBloke/zephyr-7B-beta-GGUF")
|
14 |
|
15 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
16 |
config = {
|
17 |
"max_new_token": 1024,
|
18 |
"repetition_penalty": 1.1,
|
|
|
6 |
from langchain.chains import RetrievalQA
|
7 |
import gradio as gr
|
8 |
|
9 |
+
# local_llm = "https://huggingface.co/TheBloke/zephyr-7B-beta-GGUF/blob/main/zephyr-7b-beta.Q5_K_S.gguf"
|
10 |
|
11 |
# Load model directly
|
12 |
# from transformers import AutoModel
|
13 |
# local_llm = AutoModel.from_pretrained("TheBloke/zephyr-7B-beta-GGUF")
|
14 |
|
15 |
|
16 |
+
local_llm = CTransformers(
|
17 |
+
model = "TheBloke/zephyr-7B-beta-GGUF",
|
18 |
+
model_file = "zephyr-7b-beta.Q4_0.gguf",
|
19 |
+
model_type="mistral",
|
20 |
+
lib="avx2", #for CPU use
|
21 |
+
**config
|
22 |
+
)
|
23 |
+
|
24 |
+
|
25 |
config = {
|
26 |
"max_new_token": 1024,
|
27 |
"repetition_penalty": 1.1,
|