File size: 1,976 Bytes
30d62b2 e95d4c9 30d62b2 e95d4c9 2917bf2 30d62b2 e95d4c9 30d62b2 e95d4c9 30d62b2 e95d4c9 30d62b2 e95d4c9 30d62b2 e95d4c9 30d62b2 e95d4c9 30d62b2 e95d4c9 30d62b2 e95d4c9 1de93e3 e95d4c9 30d62b2 e95d4c9 30d62b2 e95d4c9 30d62b2 e95d4c9 57d9bfd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 |
import gradio as gr
from llama_cpp import Llama
from huggingface_hub import hf_hub_download
import threading
# Title and description
TITLE = "AI Copilot for Patients"
DESCRIPTION = "I provide answers to concerns related to Health"
# Globals
llm_llama_cpp = None
model_ready = False
# Download and initialize model in background
def load_model():
global llm_llama_cpp, model_ready
try:
print("Downloading model...")
model_file_path = hf_hub_download(
repo_id="TheBloke/Llama-2-7B-GGUF",
filename="llama-2-7b.Q4_0.gguf"
)
print("Initializing model...")
llm_llama_cpp = Llama(
model_path=model_file_path,
verbose=False,
n_ctx=4096
)
model_ready = True
print("Model is ready.")
except Exception as e:
print(f"Failed to load model: {e}")
# Background thread for model loading
threading.Thread(target=load_model).start()
# Chatbot logic
def talk(prompt, history):
if not model_ready:
return "⏳ Please wait, the model is still loading..."
try:
response = ""
response_stream = llm_llama_cpp.create_completion(
prompt=prompt,
max_tokens=200,
stream=True
)
for chunk in response_stream:
if 'choices' in chunk and 'text' in chunk['choices'][0]:
response += chunk['choices'][0]['text']
return response
except Exception as e:
print(f"Error in generating response: {e}")
return f"Error during response generation: {e}"
# Gradio interface
demo = gr.ChatInterface(
fn=talk,
chatbot=gr.Chatbot(
show_label=True,
show_share_button=True,
show_copy_button=True,
layout="bubble",
type="messages",
),
theme="Soft",
examples=[["what is Diabetes?"]],
title=TITLE,
description=DESCRIPTION,
)
# Launch the UI
demo.launch(share=True) |