Update app.py
Browse files
app.py
CHANGED
@@ -19,6 +19,28 @@ For more information on `huggingface_hub` Inference API support, please check th
|
|
19 |
"""
|
20 |
client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
|
21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
22 |
def respond(message, history, system_message, max_tokens, temperature, top_p):
|
23 |
|
24 |
URL = "https://www.esmo.org/content/download/6594/114963/1/ES-Cancer-de-Mama-Guia-para-Pacientes.pdf"
|
@@ -36,6 +58,7 @@ def respond(message, history, system_message, max_tokens, temperature, top_p):
|
|
36 |
|
37 |
vectordb = Chroma.from_documents(documents=all_splits, embedding=embeddings, persist_directory="chroma_db")
|
38 |
|
|
|
39 |
query = message
|
40 |
docs = vectordb.similarity_search_with_score(query)
|
41 |
context = []
|
|
|
19 |
"""
|
20 |
client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
|
21 |
|
22 |
+
model_id = 'mistralai/Mistral-7B-Instruct-v0.1'
|
23 |
+
model_config = transformers.AutoConfig.from_pretrained(
|
24 |
+
model_id,
|
25 |
+
max_new_tokens=200
|
26 |
+
)
|
27 |
+
|
28 |
+
model = transformers.AutoModelForCausalLM.from_pretrained(
|
29 |
+
model_id,
|
30 |
+
trust_remote_code=True,
|
31 |
+
config=model_config,
|
32 |
+
quantization_config=bnb_config,
|
33 |
+
device_map='auto',
|
34 |
+
)
|
35 |
+
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
36 |
+
query_pipeline = transformers.pipeline(
|
37 |
+
"text-generation",
|
38 |
+
model=model,
|
39 |
+
tokenizer=tokenizer,
|
40 |
+
torch_dtype=torch.float16,
|
41 |
+
device_map="auto", max_new_tokens=200)
|
42 |
+
|
43 |
+
|
44 |
def respond(message, history, system_message, max_tokens, temperature, top_p):
|
45 |
|
46 |
URL = "https://www.esmo.org/content/download/6594/114963/1/ES-Cancer-de-Mama-Guia-para-Pacientes.pdf"
|
|
|
58 |
|
59 |
vectordb = Chroma.from_documents(documents=all_splits, embedding=embeddings, persist_directory="chroma_db")
|
60 |
|
61 |
+
pipeline=query_pipeline
|
62 |
query = message
|
63 |
docs = vectordb.similarity_search_with_score(query)
|
64 |
context = []
|