Create app.py

app.py

@@ -0,0 +1,155 @@
+import os
+import logging
+import gradio as gr
+from huggingface_hub import hf_hub_download
+
+# Install necessary libraries using os.system
+os.system("pip install --upgrade pip")
+os.system("pip install llama-cpp-agent huggingface_hub trafilatura beautifulsoup4 requests duckduckgo-search googlesearch-python")
+
+# Attempt to import all required modules
+try:
+    from llama_cpp import Llama
+    from llama_cpp_agent.providers import LlamaCppPythonProvider
+    from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType
+    from llama_cpp_agent.chat_history import BasicChatHistory
+    from llama_cpp_agent.chat_history.messages import Roles
+    from llama_cpp_agent.llm_output_settings import (
+        LlmStructuredOutputSettings,
+        LlmStructuredOutputType,
+    )
+    from llama_cpp_agent.tools import WebSearchTool
+    from llama_cpp_agent.prompt_templates import web_search_system_prompt, research_system_prompt
+    from utils import CitingSources
+    from settings import get_context_by_model, get_messages_formatter_type
+except ImportError as e:
+    raise ImportError(f"Error importing modules: {e}")
+
+# Download the models
+hf_hub_download(
+    repo_id="bartowski/Mistral-7B-Instruct-v0.3-GGUF",
+    filename="Mistral-7B-Instruct-v0.3-Q6_K.gguf",
+    local_dir="./models"
+)
+hf_hub_download(
+    repo_id="bartowski/Meta-Llama-3-8B-Instruct-GGUF",
+    filename="Meta-Llama-3-8B-Instruct-Q6_K.gguf",
+    local_dir="./models"
+)
+hf_hub_download(
+    repo_id="TheBloke/Mixtral-8x7B-Instruct-v0.1-GGUF",
+    filename="mixtral-8x7b-instruct-v0.1.Q5_K_M.gguf",
+    local_dir="./models"
+)
+
+# Function to respond to user messages
+def respond(message, temperature, top_p, top_k, repeat_penalty):
+    try:
+        model = "mixtral-8x7b-instruct-v0.1.Q5_K_M.gguf"
+        max_tokens = 3000
+        chat_template = get_messages_formatter_type(model)
+        llm = Llama(
+            model_path=f"models/{model}",
+            flash_attn=True,
+            n_gpu_layers=81,
+            n_batch=1024,
+            n_ctx=get_context_by_model(model),
+        )
+        provider = LlamaCppPythonProvider(llm)
+        logging.info(f"Loaded chat examples: {chat_template}")
+        search_tool = WebSearchTool(
+            llm_provider=provider,
+            message_formatter_type=chat_template,
+            max_tokens_search_results=12000,
+            max_tokens_per_summary=2048,
+        )
+
+        web_search_agent = LlamaCppAgent(
+            provider,
+            system_prompt=web_search_system_prompt,
+            predefined_messages_formatter_type=chat_template,
+            debug_output=True,
+        )
+
+        answer_agent = LlamaCppAgent(
+            provider,
+            system_prompt=research_system_prompt,
+            predefined_messages_formatter_type=chat_template,
+            debug_output=True,
+        )
+
+        settings = provider.get_provider_default_settings()
+        settings.stream = False
+        settings.temperature = temperature
+        settings.top_k = top_k
+        settings.top_p = top_p
+        settings.max_tokens = max_tokens
+        settings.repeat_penalty = repeat_penalty
+
+        output_settings = LlmStructuredOutputSettings.from_functions(
+            [search_tool.get_tool()]
+        )
+
+        messages = BasicChatHistory()
+
+        result = web_search_agent.get_chat_response(
+            message,
+            llm_sampling_settings=settings,
+            structured_output_settings=output_settings,
+            add_message_to_chat_history=False,
+            add_response_to_chat_history=False,
+            print_output=False,
+        )
+
+        outputs = ""
+
+        settings.stream = True
+        response_text = answer_agent.get_chat_response(
+            f"Write a detailed and complete research document that fulfills the following user request: '{message}', based on the information from the web below.\n\n" +
+            result[0]["return_value"],
+            role=Roles.tool,
+            llm_sampling_settings=settings,
+            chat_history=messages,
+            returns_streaming_generator=True,
+            print_output=False,
+        )
+
+        for text in response_text:
+            outputs += text
+
+        output_settings = LlmStructuredOutputSettings.from_pydantic_models(
+            [CitingSources], LlmStructuredOutputType.object_instance
+        )
+
+        citing_sources = answer_agent.get_chat_response(
+            "Cite the sources you used in your response.",
+            role=Roles.tool,
+            llm_sampling_settings=settings,
+            chat_history=messages,
+            returns_streaming_generator=False,
+            structured_output_settings=output_settings,
+            print_output=False,
+        )
+        outputs += "\n\nSources:\n"
+        outputs += "\n".join(citing_sources.sources)
+        return outputs
+
+    except Exception as e:
+        return f"An error occurred: {e}"
+
+# Gradio interface
+demo = gr.Interface(
+    fn=respond,
+    inputs=[
+        gr.Textbox(label="Enter your message:"),
+        gr.Slider(minimum=0.1, maximum=1.0, value=0.45, step=0.1, label="Temperature"),
+        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p"),
+        gr.Slider(minimum=0, maximum=100, value=40, step=1, label="Top-k"),
+        gr.Slider(minimum=0.0, maximum=2.0, value=1.1, step=0.1, label="Repetition penalty")
+    ],
+    outputs="text",
+    title="Novav2 Web Engine"
+)
+
+if __name__ == "__main__":
+    demo.launch()