app.py

from llm import get_agent, get_openai_model
import gradio as gr



def process_url(url):
    model = get_openai_model()
    agent = get_agent(model)

    prompt_content = open("prompt.txt", "r").read()
    prompt = f"""
    You are a helpful assistant that can generate a full llm txt file from a website.
    Step by step:
    1. Generate a sitemap for the website
    2. Crawl the website and generate a description of each sitemap page
    3. Generate a full llm txt file from the website based on the sitemap and the descriptions of each page

    The generated llm txt file should be super comprehensive and should include all the information of the website.
    It is used for a chatbot that will answer questions about the website.

    You can use the following tools to help you:
    - VisitWebpageTool: to visit a website
    - GoogleSearchTool: to search the web
    - SitemapTools: to generate a sitemap for a website
    - FirecrawlTools: to crawl a website

    The generated llm txt file should be in the following format:
    {prompt_content}

    Example:
    - Sitemap: https://docs.anthropic.com/llms-full.txt
    - Description: This is a full llm txt file from the website
    - Full llm txt file: https://docs.anthropic.com/llms-full.txt

    Do that for the following website: {url}
    """

    result = agent.run(prompt)
    return result


demo = gr.Interface(
    fn=process_url,
    inputs=gr.Textbox(label="Enter website URL", placeholder="https://example.com"),
    outputs=gr.Textbox(label="Generated LLM Text", lines=10),
    title="LLM Text Generator",
    description="Enter a website URL to generate LLM text based on its content.",
)

if __name__ == "__main__":
    demo.launch()