app-old.py

import gradio as gr
import requests
from bs4 import BeautifulSoup
import re
from urllib.parse import urljoin, urlparse
import asyncio
import aiohttp
from collections import defaultdict
import unicodedata
import logging

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


class WebsiteCrawler:
    def __init__(self, max_depth=3, max_pages=50):
        self.max_depth = max_depth
        self.max_pages = max_pages
        self.visited_urls = set()
        self.url_metadata = defaultdict(dict)
        self.homepage_metadata = None
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        }

    def determine_category_importance(self, url, title, desc):
        url_lower = url.lower()
        path = urlparse(url).path.lower()

        if path == "/" or path == "":
            return "Main", 10

        if any(x in url_lower for x in ["/docs", "/faq", "/help"]):
            return "Documentation", 8

        elif any(x in url_lower for x in ["/api", "/developer"]):
            return "API", 8

        elif any(x in url_lower for x in ["/about", "/company", "/contact"]):
            return "About", 7

        elif any(x in url_lower for x in ["/news", "/blog", "/events"]):
            return "News", 5

        elif any(x in url_lower for x in ["/tools", "/pricing"]):
            return "Tools", 6

        return "Optional", 1

    def clean_text(self, text, is_title=False):
        if not text:
            return ""
        text = unicodedata.normalize("NFKD", text)
        text = re.sub(r"[^\x00-\x7F]+", "", text)
        text = " ".join(text.split()).strip()

        if is_title:
            text = re.sub(r"^\s*Welcome to\s+", "", text)
        return text

    async def crawl_page(self, url, depth, base_domain):
        if (
            depth > self.max_depth
            or url in self.visited_urls
            or len(self.visited_urls) >= self.max_pages
        ):
            return []

        try:
            async with aiohttp.ClientSession(
                timeout=aiohttp.ClientTimeout(total=20)
            ) as session:
                async with session.get(
                    url, headers=self.headers, allow_redirects=True
                ) as response:
                    if response.status != 200:
                        return []
                    text = await response.text()
                    self.visited_urls.add(url)

                    soup = BeautifulSoup(text, "html.parser")
                    title_tag = soup.find("title")
                    title = (
                        self.clean_text(title_tag.text)
                        if title_tag
                        else url.split("/")[-1]
                    )

                    desc_tag = soup.find("meta", {"name": "description"})
                    desc = (
                        self.clean_text(desc_tag["content"])
                        if desc_tag and desc_tag.get("content")
                        else ""
                    )

                    category, importance = self.determine_category_importance(
                        url, title, desc
                    )

                    self.url_metadata[url] = {
                        "title": title,
                        "description": desc,
                        "category": category,
                        "importance": importance,
                    }

                    links = []
                    for a in soup.find_all("a", href=True):
                        next_url = urljoin(url, a["href"])
                        if urlparse(next_url).netloc == base_domain:
                            links.append(next_url)

                    return links
        except Exception as e:
            logger.error(f"Error crawling {url}: {str(e)}")
            return []

    async def process_homepage(self, url):
        try:
            async with aiohttp.ClientSession(
                timeout=aiohttp.ClientTimeout(total=20)
            ) as session:
                async with session.get(
                    url, headers=self.headers, allow_redirects=True
                ) as response:
                    if response.status != 200:
                        return
                    text = await response.text()
                    soup = BeautifulSoup(text, "html.parser")

                    site_name = (
                        soup.find("title").text.split("|")[0].strip()
                        if soup.find("title")
                        else urlparse(url).netloc
                    )
                    description = soup.find("meta", {"name": "description"})
                    description = (
                        description["content"].strip()
                        if description and description.get("content")
                        else None
                    )

                    self.homepage_metadata = {
                        "site_name": self.clean_text(site_name, is_title=True),
                        "description": (
                            self.clean_text(description) if description else None
                        ),
                    }
        except Exception as e:
            logger.error(f"Error processing homepage {url}: {str(e)}")

    async def crawl_website(self, start_url):
        try:
            await self.process_homepage(start_url)
            base_domain = urlparse(start_url).netloc
            queue = [(start_url, 0)]
            seen = {start_url}

            while queue and len(self.visited_urls) < self.max_pages:
                current_url, depth = queue.pop(0)
                if depth > self.max_depth:
                    continue
                links = await self.crawl_page(current_url, depth, base_domain)
                for link in links:
                    if link not in seen:
                        seen.add(link)
                        queue.append((link, depth + 1))

        except Exception as e:
            logger.error(f"Error during crawl: {str(e)}")
            raise

    def generate_llms_txt(self):
        if not self.url_metadata:
            return "No content available."

        content = []
        homepage_title = self.homepage_metadata.get("site_name", "Website")
        homepage_description = self.homepage_metadata.get(
            "description", "No description available."
        )
        content.append(f"# {homepage_title}\n\n> {homepage_description}\n")

        categories = defaultdict(list)
        for url, metadata in self.url_metadata.items():
            categories[metadata["category"]].append((url, metadata))

        category_order = [
            "Main",
            "Documentation",
            "API",
            "About",
            "News",
            "Tools",
            "Optional",
        ]
        for category in category_order:
            if category in categories:
                content.append(f"## {category}")
                for url, metadata in categories[category]:
                    content.append(
                        f"- [{metadata['title']}]({url}): {metadata['description']}"
                    )

        return "\n".join(content)


async def process_url(url, max_depth, max_pages):
    try:
        if not url.startswith(("http://", "https://")):
            url = "https://" + url
        result = urlparse(url)
        if not result.scheme or not result.netloc:
            return "", "Invalid URL format. Please enter a valid URL."

        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
        await crawler.crawl_website(url)
        content = crawler.generate_llms_txt()

        return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
    except Exception as e:
        logger.error(f"Error processing URL {url}: {str(e)}")
        return "", f"Error: {str(e)}"


# Gradio interface
theme = gr.themes.Soft(primary_hue="blue", font="Open Sans")

with gr.Blocks(theme=theme) as iface:
    with gr.Row():
        gr.Markdown("## Website Crawler - Generate llms.txt")

    with gr.Row():
        url_input = gr.Textbox(
            label="Website URL",
            placeholder="Enter the website URL (e.g., example.com)",
            info="The URL will be automatically prefixed with https:// if not provided",
            lines=1,
        )

    with gr.Row():
        depth_input = gr.Slider(
            minimum=1, maximum=5, value=3, step=1, label="Maximum Crawl Depth"
        )
        pages_input = gr.Slider(
            minimum=10, maximum=100, value=50, step=10, label="Maximum Pages"
        )

    with gr.Row():
        generate_btn = gr.Button("Generate llms.txt", variant="primary")

    with gr.Row():
        output = gr.Textbox(
            label="Generated llms.txt Content",
            lines=15,
            show_copy_button=True,
            container=True,
        )

    with gr.Row():
        status = gr.Textbox(label="Status", interactive=False)

    # Pass the asynchronous function directly
    generate_btn.click(
        fn=process_url,
        inputs=[url_input, depth_input, pages_input],
        outputs=[output, status],
    )

if __name__ == "__main__":
    iface.launch(asyncio_mode="auto")