create-llms-txt / app-old.py
cyberandy's picture
Rename app.py to app-old.py
f06e7da verified
raw
history blame
9.32 kB
import gradio as gr
import requests
from bs4 import BeautifulSoup
import re
from urllib.parse import urljoin, urlparse
import asyncio
import aiohttp
from collections import defaultdict
import unicodedata
import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class WebsiteCrawler:
def __init__(self, max_depth=3, max_pages=50):
self.max_depth = max_depth
self.max_pages = max_pages
self.visited_urls = set()
self.url_metadata = defaultdict(dict)
self.homepage_metadata = None
self.headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
}
def determine_category_importance(self, url, title, desc):
url_lower = url.lower()
path = urlparse(url).path.lower()
if path == "/" or path == "":
return "Main", 10
if any(x in url_lower for x in ["/docs", "/faq", "/help"]):
return "Documentation", 8
elif any(x in url_lower for x in ["/api", "/developer"]):
return "API", 8
elif any(x in url_lower for x in ["/about", "/company", "/contact"]):
return "About", 7
elif any(x in url_lower for x in ["/news", "/blog", "/events"]):
return "News", 5
elif any(x in url_lower for x in ["/tools", "/pricing"]):
return "Tools", 6
return "Optional", 1
def clean_text(self, text, is_title=False):
if not text:
return ""
text = unicodedata.normalize("NFKD", text)
text = re.sub(r"[^\x00-\x7F]+", "", text)
text = " ".join(text.split()).strip()
if is_title:
text = re.sub(r"^\s*Welcome to\s+", "", text)
return text
async def crawl_page(self, url, depth, base_domain):
if (
depth > self.max_depth
or url in self.visited_urls
or len(self.visited_urls) >= self.max_pages
):
return []
try:
async with aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=20)
) as session:
async with session.get(
url, headers=self.headers, allow_redirects=True
) as response:
if response.status != 200:
return []
text = await response.text()
self.visited_urls.add(url)
soup = BeautifulSoup(text, "html.parser")
title_tag = soup.find("title")
title = (
self.clean_text(title_tag.text)
if title_tag
else url.split("/")[-1]
)
desc_tag = soup.find("meta", {"name": "description"})
desc = (
self.clean_text(desc_tag["content"])
if desc_tag and desc_tag.get("content")
else ""
)
category, importance = self.determine_category_importance(
url, title, desc
)
self.url_metadata[url] = {
"title": title,
"description": desc,
"category": category,
"importance": importance,
}
links = []
for a in soup.find_all("a", href=True):
next_url = urljoin(url, a["href"])
if urlparse(next_url).netloc == base_domain:
links.append(next_url)
return links
except Exception as e:
logger.error(f"Error crawling {url}: {str(e)}")
return []
async def process_homepage(self, url):
try:
async with aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=20)
) as session:
async with session.get(
url, headers=self.headers, allow_redirects=True
) as response:
if response.status != 200:
return
text = await response.text()
soup = BeautifulSoup(text, "html.parser")
site_name = (
soup.find("title").text.split("|")[0].strip()
if soup.find("title")
else urlparse(url).netloc
)
description = soup.find("meta", {"name": "description"})
description = (
description["content"].strip()
if description and description.get("content")
else None
)
self.homepage_metadata = {
"site_name": self.clean_text(site_name, is_title=True),
"description": (
self.clean_text(description) if description else None
),
}
except Exception as e:
logger.error(f"Error processing homepage {url}: {str(e)}")
async def crawl_website(self, start_url):
try:
await self.process_homepage(start_url)
base_domain = urlparse(start_url).netloc
queue = [(start_url, 0)]
seen = {start_url}
while queue and len(self.visited_urls) < self.max_pages:
current_url, depth = queue.pop(0)
if depth > self.max_depth:
continue
links = await self.crawl_page(current_url, depth, base_domain)
for link in links:
if link not in seen:
seen.add(link)
queue.append((link, depth + 1))
except Exception as e:
logger.error(f"Error during crawl: {str(e)}")
raise
def generate_llms_txt(self):
if not self.url_metadata:
return "No content available."
content = []
homepage_title = self.homepage_metadata.get("site_name", "Website")
homepage_description = self.homepage_metadata.get(
"description", "No description available."
)
content.append(f"# {homepage_title}\n\n> {homepage_description}\n")
categories = defaultdict(list)
for url, metadata in self.url_metadata.items():
categories[metadata["category"]].append((url, metadata))
category_order = [
"Main",
"Documentation",
"API",
"About",
"News",
"Tools",
"Optional",
]
for category in category_order:
if category in categories:
content.append(f"## {category}")
for url, metadata in categories[category]:
content.append(
f"- [{metadata['title']}]({url}): {metadata['description']}"
)
return "\n".join(content)
async def process_url(url, max_depth, max_pages):
try:
if not url.startswith(("http://", "https://")):
url = "https://" + url
result = urlparse(url)
if not result.scheme or not result.netloc:
return "", "Invalid URL format. Please enter a valid URL."
crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
await crawler.crawl_website(url)
content = crawler.generate_llms_txt()
return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
except Exception as e:
logger.error(f"Error processing URL {url}: {str(e)}")
return "", f"Error: {str(e)}"
# Gradio interface
theme = gr.themes.Soft(primary_hue="blue", font="Open Sans")
with gr.Blocks(theme=theme) as iface:
with gr.Row():
gr.Markdown("## Website Crawler - Generate llms.txt")
with gr.Row():
url_input = gr.Textbox(
label="Website URL",
placeholder="Enter the website URL (e.g., example.com)",
info="The URL will be automatically prefixed with https:// if not provided",
lines=1,
)
with gr.Row():
depth_input = gr.Slider(
minimum=1, maximum=5, value=3, step=1, label="Maximum Crawl Depth"
)
pages_input = gr.Slider(
minimum=10, maximum=100, value=50, step=10, label="Maximum Pages"
)
with gr.Row():
generate_btn = gr.Button("Generate llms.txt", variant="primary")
with gr.Row():
output = gr.Textbox(
label="Generated llms.txt Content",
lines=15,
show_copy_button=True,
container=True,
)
with gr.Row():
status = gr.Textbox(label="Status", interactive=False)
# Pass the asynchronous function directly
generate_btn.click(
fn=process_url,
inputs=[url_input, depth_input, pages_input],
outputs=[output, status],
)
if __name__ == "__main__":
iface.launch(asyncio_mode="auto")