WEVE / app.py
dineth554's picture
Create app.py
deaf847 verified
import os
import logging
import gradio as gr
from huggingface_hub import hf_hub_download
# Install necessary libraries using os.system
os.system("pip install --upgrade pip")
os.system("pip install llama-cpp-agent huggingface_hub trafilatura beautifulsoup4 requests duckduckgo-search googlesearch-python")
# Attempt to import all required modules
try:
from llama_cpp import Llama
from llama_cpp_agent.providers import LlamaCppPythonProvider
from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType
from llama_cpp_agent.chat_history import BasicChatHistory
from llama_cpp_agent.chat_history.messages import Roles
from llama_cpp_agent.llm_output_settings import (
LlmStructuredOutputSettings,
LlmStructuredOutputType,
)
from llama_cpp_agent.tools import WebSearchTool
from llama_cpp_agent.prompt_templates import web_search_system_prompt, research_system_prompt
from utils import CitingSources
from settings import get_context_by_model, get_messages_formatter_type
except ImportError as e:
raise ImportError(f"Error importing modules: {e}")
# Download the models
hf_hub_download(
repo_id="bartowski/Mistral-7B-Instruct-v0.3-GGUF",
filename="Mistral-7B-Instruct-v0.3-Q6_K.gguf",
local_dir="./models"
)
hf_hub_download(
repo_id="bartowski/Meta-Llama-3-8B-Instruct-GGUF",
filename="Meta-Llama-3-8B-Instruct-Q6_K.gguf",
local_dir="./models"
)
hf_hub_download(
repo_id="TheBloke/Mixtral-8x7B-Instruct-v0.1-GGUF",
filename="mixtral-8x7b-instruct-v0.1.Q5_K_M.gguf",
local_dir="./models"
)
# Function to respond to user messages
def respond(message, temperature, top_p, top_k, repeat_penalty):
try:
model = "mixtral-8x7b-instruct-v0.1.Q5_K_M.gguf"
max_tokens = 3000
chat_template = get_messages_formatter_type(model)
llm = Llama(
model_path=f"models/{model}",
flash_attn=True,
n_gpu_layers=81,
n_batch=1024,
n_ctx=get_context_by_model(model),
)
provider = LlamaCppPythonProvider(llm)
logging.info(f"Loaded chat examples: {chat_template}")
search_tool = WebSearchTool(
llm_provider=provider,
message_formatter_type=chat_template,
max_tokens_search_results=12000,
max_tokens_per_summary=2048,
)
web_search_agent = LlamaCppAgent(
provider,
system_prompt=web_search_system_prompt,
predefined_messages_formatter_type=chat_template,
debug_output=True,
)
answer_agent = LlamaCppAgent(
provider,
system_prompt=research_system_prompt,
predefined_messages_formatter_type=chat_template,
debug_output=True,
)
settings = provider.get_provider_default_settings()
settings.stream = False
settings.temperature = temperature
settings.top_k = top_k
settings.top_p = top_p
settings.max_tokens = max_tokens
settings.repeat_penalty = repeat_penalty
output_settings = LlmStructuredOutputSettings.from_functions(
[search_tool.get_tool()]
)
messages = BasicChatHistory()
result = web_search_agent.get_chat_response(
message,
llm_sampling_settings=settings,
structured_output_settings=output_settings,
add_message_to_chat_history=False,
add_response_to_chat_history=False,
print_output=False,
)
outputs = ""
settings.stream = True
response_text = answer_agent.get_chat_response(
f"Write a detailed and complete research document that fulfills the following user request: '{message}', based on the information from the web below.\n\n" +
result[0]["return_value"],
role=Roles.tool,
llm_sampling_settings=settings,
chat_history=messages,
returns_streaming_generator=True,
print_output=False,
)
for text in response_text:
outputs += text
output_settings = LlmStructuredOutputSettings.from_pydantic_models(
[CitingSources], LlmStructuredOutputType.object_instance
)
citing_sources = answer_agent.get_chat_response(
"Cite the sources you used in your response.",
role=Roles.tool,
llm_sampling_settings=settings,
chat_history=messages,
returns_streaming_generator=False,
structured_output_settings=output_settings,
print_output=False,
)
outputs += "\n\nSources:\n"
outputs += "\n".join(citing_sources.sources)
return outputs
except Exception as e:
return f"An error occurred: {e}"
# Gradio interface
demo = gr.Interface(
fn=respond,
inputs=[
gr.Textbox(label="Enter your message:"),
gr.Slider(minimum=0.1, maximum=1.0, value=0.45, step=0.1, label="Temperature"),
gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p"),
gr.Slider(minimum=0, maximum=100, value=40, step=1, label="Top-k"),
gr.Slider(minimum=0.0, maximum=2.0, value=1.1, step=0.1, label="Repetition penalty")
],
outputs="text",
title="Novav2 Web Engine"
)
if __name__ == "__main__":
demo.launch()