Update app.py

app.py

@@ -3,435 +3,39 @@ import requests
 from bs4 import BeautifulSoup
 import re
 from urllib.parse import urljoin, urlparse
-import markdown
-from concurrent.futures import ThreadPoolExecutor
 import asyncio
 from collections import defaultdict
-import time
-import logging
 import unicodedata
+import logging
 
-# Set up logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 class WebsiteCrawler:
-    def __init__(self, max_depth=3, max_pages=50, timeout=30):
+    def __init__(self, max_depth=3, max_pages=50):
         self.max_depth = max_depth
         self.max_pages = max_pages
-        self.timeout = timeout
         self.visited_urls = set()
-        self.url_content = {}
         self.url_metadata = defaultdict(dict)
         self.headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
         }
 
-    def normalize_text(self, text):
-        """Normalize text to handle encoding issues"""
+    def clean_text(self, text, is_title=False):
+        """Clean and normalize text"""
         if not text:
             return ""
         # Normalize unicode characters
         text = unicodedata.normalize('NFKD', text)
-        # Replace special quotes and dashes with standard characters
-        text = text.replace('\u201c', '"').replace('\u201d', '"')  # smart quotes
-        text = text.replace('\u2018', "'").replace('\u2019', "'")  # smart single quotes
-        text = text.replace('\u2013', '-').replace('\u2014', '-')  # en and em dashes
-        # Remove any remaining non-ASCII characters
-        text = text.encode('ascii', 'ignore').decode('ascii')
-        # Clean up extra whitespace and ensure proper sentence spacing
-        text = ' '.join(text.split())
-        return text
-
-    def clean_url(self, url):
-        """Clean URL by removing fragments and unnecessary parameters"""
-        # Remove fragments (everything after #)
-        url = re.sub(r'#.*
-
-    def is_valid_url(self, url, base_domain):
-        """Check if URL is valid and belongs to the same domain"""
-        try:
-            parsed = urlparse(url)
-            base_parsed = urlparse(base_domain)
-            return (parsed.netloc == base_parsed.netloc and
-                   parsed.scheme in ['http', 'https'] and
-                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
-        except:
-            return False
-
-    def extract_content(self, soup):
-        """Extract meaningful content from HTML"""
-        # Remove script and style elements
-        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
-            element.decompose()
-
-        # Get main content
-        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
-        if main_content:
-            return self.normalize_text(main_content.get_text(strip=True))
-        return self.normalize_text(soup.get_text(strip=True))
-
-    def get_page_metadata(self, soup, url):
-        """Extract metadata from the page"""
-        metadata = {
-            'title': None,
-            'description': None,
-            'importance': 0,
-            'category': 'Optional'
-        }
-
-        # Title extraction with cleaning
-        title = (
-            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
-            soup.find('title').text if soup.find('title') else
-            soup.find('h1').text if soup.find('h1') else
-            url.split('/')[-1]
-        )
-        metadata['title'] = self.clean_title(title)
-
-        # Description extraction with cleaning
-        description = (
-            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
-            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
-            ""
-        )
-        metadata['description'] = self.clean_description(description)
-
-        # Calculate importance and category
-        url_lower = url.lower()
-        if 'docs' in url_lower or 'documentation' in url_lower:
-            metadata['importance'] = 5
-            metadata['category'] = 'Docs'
-        elif 'api' in url_lower:
-            metadata['importance'] = 4
-            metadata['category'] = 'API'
-        elif 'guide' in url_lower or 'tutorial' in url_lower:
-            metadata['importance'] = 3
-            metadata['category'] = 'Guides'
-        elif 'example' in url_lower:
-            metadata['importance'] = 2
-            metadata['category'] = 'Examples'
-        elif 'blog' in url_lower:
-            metadata['importance'] = 1
-            metadata['category'] = 'Blog'
-
-        return metadata
-
-    async def crawl_page(self, url, depth, base_domain):
-        """Crawl a single page and extract information"""
-        if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages:
-            return []
-
-        try:
-            response = requests.get(url, headers=self.headers, timeout=self.timeout)
-            response.encoding = 'utf-8'
-            response.raise_for_status()
-            self.visited_urls.add(url)
-
-            soup = BeautifulSoup(response.text, 'html.parser')
-            content = self.extract_content(soup)
-            metadata = self.get_page_metadata(soup, url)
-            
-            self.url_content[url] = content
-            self.url_metadata[url] = metadata
-
-            # Find all links
-            links = []
-            for a in soup.find_all('a', href=True):
-                next_url = urljoin(url, a['href'])
-                if self.is_valid_url(next_url, base_domain):
-                    links.append(next_url)
-
-            return links
-
-        except Exception as e:
-            logger.error(f"Error crawling {url}: {str(e)}")
-            return []
-
-    async def crawl_website(self, start_url):
-        """Crawl website starting from the given URL"""
-        base_domain = start_url
-        queue = [(start_url, 0)]
-        seen = {start_url}
-
-        while queue and len(self.visited_urls) < self.max_pages:
-            current_url, depth = queue.pop(0)
-            
-            if depth > self.max_depth:
-                continue
-
-            links = await self.crawl_page(current_url, depth, base_domain)
-            
-            for link in links:
-                if link not in seen:
-                    seen.add(link)
-                    queue.append((link, depth + 1))
-
-    def generate_llms_txt(self):
-        """Generate llms.txt content from crawled data"""
-        # Clean and deduplicate metadata
-        cleaned_metadata = self.remove_duplicate_content(self.url_metadata)
-        
-        # Sort URLs by importance
-        sorted_urls = sorted(
-            cleaned_metadata.items(),
-            key=lambda x: (x[1]['importance'], x[0]),
-            reverse=True
-        )
-
-        if not sorted_urls:
-            return "No content was found to generate llms.txt"
-
-        # Group URLs by category
-        categorized_urls = defaultdict(list)
-        for url, metadata in sorted_urls:
-            categorized_urls[metadata['category']].append((url, metadata))
-
-        # Generate content
-        content = []
-        
-        # Add main title and description
-        main_metadata = sorted_urls[0][1]
-        content.append(f"# {main_metadata['title']}")
-        if main_metadata['description']:
-            content.append(f"\n> {main_metadata['description']}")
-        
-        # Add categorized sections
-        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
-        
-        for category in priority_order:
-            if category in categorized_urls:
-                content.append(f"\n## {category}")
-                for url, metadata in categorized_urls[category]:
-                    if metadata['description']:
-                        content.append(f"\n- [{metadata['title']}]({url}): {metadata['description']}")
-                    else:
-                        content.append(f"\n- [{metadata['title']}]({url})")
-
-        return "\n".join(content)
-
-async def process_url(url, max_depth, max_pages):
-    """Process URL and generate llms.txt"""
-    try:
-        # Add https:// if not present
-        if not url.startswith(('http://', 'https://')):
-            url = 'https://' + url
-
-        # Validate URL format
-        try:
-            result = urlparse(url)
-            if not all([result.scheme, result.netloc]):
-                return "", "Invalid URL format. Please enter a valid URL."
-        except:
-            return "", "Invalid URL format. Please enter a valid URL."
-
-        # Create crawler and process
-        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
-        await crawler.crawl_website(url)
-        content = crawler.generate_llms_txt()
-        
-        return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
-    
-    except Exception as e:
-        return "", f"Error: {str(e)}"
-
-# Create custom theme
-theme = gr.themes.Soft(
-    primary_hue="blue",
-    font="Open Sans"
-)
-
-# Create the Gradio interface
-with gr.Blocks(theme=theme,     css="""
-    @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');
-    
-    .gradio-container {
-        font-family: 'Open Sans', sans-serif !important;
-    }
-    
-    .gr-button {
-        font-family: 'Open Sans', sans-serif !important;
-        font-weight: 600 !important;
-    }
-
-    /* Primary color customization */
-    .primary-btn {
-        background-color: #2436d4 !important;
-        color: white !important;
-    }
-
-    .primary-btn:hover {
-        background-color: #1c2aa8 !important;
-    }
-    
-    [data-testid="textbox"] {
-        font-family: 'Open Sans', sans-serif !important;
-    }
-""") as iface:
-    gr.Markdown("# llms.txt Generator")
-    gr.Markdown("Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.")
-    
-    with gr.Row():
-        url_input = gr.Textbox(
-            label="Website URL", 
-            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
-            info="The URL will be automatically prefixed with https:// if no protocol is specified."
-        )
-    
-    with gr.Row():
-        with gr.Column():
-            depth_input = gr.Slider(
-                minimum=1, 
-                maximum=5, 
-                value=3, 
-                step=1, 
-                label="Maximum Crawl Depth",
-                info="Higher values will result in more thorough but slower crawling"
-            )
-        with gr.Column():
-            pages_input = gr.Slider(
-                minimum=10, 
-                maximum=100, 
-                value=50, 
-                step=10, 
-                label="Maximum Pages to Crawl",
-                info="Higher values will result in more comprehensive but slower results"
-            )
-    
-    generate_btn = gr.Button("Generate llms.txt", variant="primary")
-    
-    with gr.Row():
-        output = gr.Textbox(
-            label="Generated llms.txt Content",
-            lines=20,
-            max_lines=30,
-            show_copy_button=True,
-            container=True,
-            scale=2,
-            interactive=True
-        )
-    
-    status = gr.Textbox(label="Status")
-    
-    generate_btn.click(
-        fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)),
-        inputs=[url_input, depth_input, pages_input],
-        outputs=[output, status]
-    )
-
-# Launch the app
-if __name__ == "__main__":
-    iface.launch()
-, '', url)
-        # Remove trailing slashes
-        url = url.rstrip('/')
-        return url
-
-    def remove_duplicate_content(self, urls_metadata):
-        """Remove duplicate content based on similar titles and URLs"""
-        seen_content = {}
-        cleaned_metadata = {}
-        
-        for url, metadata in urls_metadata.items():
-            clean_url = self.clean_url(url)
-            base_url = clean_url.split('#')[0]  # Remove hash fragments
-            
-            # Create a content signature based on title and base URL
-            title = metadata['title'].lower()
-            
-            # Skip entries that are just fragments of the same page
-            if base_url in seen_content:
-                # Keep the one with the shortest URL (usually the main page)
-                if len(clean_url) < len(seen_content[base_url]):
-                    cleaned_metadata[clean_url] = metadata
-                    cleaned_metadata.pop(seen_content[base_url], None)
-                    seen_content[base_url] = clean_url
-                continue
-            
-            seen_content[base_url] = clean_url
-            cleaned_metadata[clean_url] = metadata
-            
-        return cleaned_metadata
-
-    def clean_title(self, title):
-        """Clean and format titles"""
-        if not title:
-            return ""
+        text = re.sub(r'[^\x00-\x7F]+', '', text)
         
-        title = self.normalize_text(title)
+        if is_title:
+            # Remove common suffixes and fragments for titles
+            text = re.sub(r'\s*[\|\-#:•].*', '', text)
+            text = re.sub(r'^\s*Welcome to\s+', '', text)
+            text = text.replace('docusaurus_skipToContent_fallback', '')
         
-        # Remove common suffixes and prefixes
-        patterns = [
-            r'\s*\|\s*.*
-
-    def is_valid_url(self, url, base_domain):
-        """Check if URL is valid and belongs to the same domain"""
-        try:
-            parsed = urlparse(url)
-            base_parsed = urlparse(base_domain)
-            return (parsed.netloc == base_parsed.netloc and
-                   parsed.scheme in ['http', 'https'] and
-                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
-        except:
-            return False
-
-    def extract_content(self, soup):
-        """Extract meaningful content from HTML"""
-        # Remove script and style elements
-        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
-            element.decompose()
-
-        # Get main content
-        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
-        if main_content:
-            return self.normalize_text(main_content.get_text(strip=True))
-        return self.normalize_text(soup.get_text(strip=True))
-
-    def get_page_metadata(self, soup, url):
-        """Extract metadata from the page"""
-        metadata = {
-            'title': None,
-            'description': None,
-            'importance': 0,
-            'category': 'Optional'
-        }
-
-        # Title extraction with cleaning
-        title = (
-            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
-            soup.find('title').text if soup.find('title') else
-            soup.find('h1').text if soup.find('h1') else
-            url.split('/')[-1]
-        )
-        metadata['title'] = self.clean_title(title)
-
-        # Description extraction with cleaning
-        description = (
-            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
-            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
-            ""
-        )
-        metadata['description'] = self.clean_description(description)
-
-        # Calculate importance and category
-        url_lower = url.lower()
-        if 'docs' in url_lower or 'documentation' in url_lower:
-            metadata['importance'] = 5
-            metadata['category'] = 'Docs'
-        elif 'api' in url_lower:
-            metadata['importance'] = 4
-            metadata['category'] = 'API'
-        elif 'guide' in url_lower or 'tutorial' in url_lower:
-            metadata['importance'] = 3
-            metadata['category'] = 'Guides'
-        elif 'example' in url_lower:
-            metadata['importance'] = 2
-            metadata['category'] = 'Examples'
-        elif 'blog' in url_lower:
-            metadata['importance'] = 1
-            metadata['category'] = 'Blog'
-
-        return metadata
+        return ' '.join(text.split()).strip()
 
     async def crawl_page(self, url, depth, base_domain):
         """Crawl a single page and extract information"""
@@ -439,26 +43,50 @@ if __name__ == "__main__":
             return []
 
         try:
-            response = requests.get(url, headers=self.headers, timeout=self.timeout)
+            response = requests.get(url, headers=self.headers, timeout=10)
             response.encoding = 'utf-8'
-            response.raise_for_status()
             self.visited_urls.add(url)
 
             soup = BeautifulSoup(response.text, 'html.parser')
-            content = self.extract_content(soup)
-            metadata = self.get_page_metadata(soup, url)
             
-            self.url_content[url] = content
-            self.url_metadata[url] = metadata
-
-            # Find all links
-            links = []
-            for a in soup.find_all('a', href=True):
-                next_url = urljoin(url, a['href'])
-                if self.is_valid_url(next_url, base_domain):
-                    links.append(next_url)
-
-            return links
+            # Extract metadata
+            title = (
+                soup.find('meta', property='og:title') or 
+                soup.find('title') or 
+                soup.find('h1')
+            )
+            title = self.clean_text(title.text if title else url.split('/')[-1], is_title=True)
+            
+            desc = soup.find('meta', {'name': 'description'}) or soup.find('meta', property='og:description')
+            desc = self.clean_text(desc['content'] if desc else '')
+
+            # Determine category and importance
+            url_lower = url.lower()
+            category = 'Optional'
+            importance = 0
+            
+            if 'docs' in url_lower or 'documentation' in url_lower:
+                category = 'Docs'
+                importance = 5
+            elif 'api' in url_lower:
+                category = 'API'
+                importance = 4
+            
+            # Store metadata
+            clean_url = re.sub(r'#.*', '', url).rstrip('/')
+            self.url_metadata[clean_url] = {
+                'title': title,
+                'description': desc,
+                'category': category,
+                'importance': importance
+            }
+
+            # Find links
+            return [
+                urljoin(url, a['href'])
+                for a in soup.find_all('a', href=True)
+                if not any(x in a['href'].lower() for x in ['javascript:', 'mailto:', '.pdf', '.jpg', '.png', '.gif'])
+            ]
 
         except Exception as e:
             logger.error(f"Error crawling {url}: {str(e)}")
@@ -466,56 +94,55 @@ if __name__ == "__main__":
 
     async def crawl_website(self, start_url):
         """Crawl website starting from the given URL"""
-        base_domain = start_url
+        base_domain = urlparse(start_url).netloc
         queue = [(start_url, 0)]
         seen = {start_url}
 
         while queue and len(self.visited_urls) < self.max_pages:
             current_url, depth = queue.pop(0)
-            
             if depth > self.max_depth:
                 continue
 
             links = await self.crawl_page(current_url, depth, base_domain)
-            
             for link in links:
-                if link not in seen:
+                if link not in seen and urlparse(link).netloc == base_domain:
                     seen.add(link)
                     queue.append((link, depth + 1))
 
     def generate_llms_txt(self):
-        """Generate llms.txt content from crawled data"""
-        # Sort URLs by importance
+        """Generate llms.txt content"""
+        if not self.url_metadata:
+            return "No content was found to generate llms.txt"
+
+        # Sort and filter URLs
         sorted_urls = sorted(
             self.url_metadata.items(),
             key=lambda x: (x[1]['importance'], x[0]),
             reverse=True
         )
 
-        if not sorted_urls:
-            return "No content was found to generate llms.txt"
-
-        # Group URLs by category
-        categorized_urls = defaultdict(list)
-        for url, metadata in sorted_urls:
-            categorized_urls[metadata['category']].append((url, metadata))
-
         # Generate content
         content = []
-        
-        # Add main title and description
         main_metadata = sorted_urls[0][1]
         content.append(f"# {main_metadata['title']}")
         if main_metadata['description']:
             content.append(f"\n> {main_metadata['description']}")
+
+        # Group by category
+        categories = defaultdict(list)
+        seen_titles = set()
         
-        # Add categorized sections
-        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
-        
-        for category in priority_order:
-            if category in categorized_urls:
+        for url, metadata in sorted_urls:
+            title = metadata['title']
+            if title not in seen_titles:
+                categories[metadata['category']].append((url, metadata))
+                seen_titles.add(title)
+
+        # Add sections
+        for category in ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']:
+            if category in categories:
                 content.append(f"\n## {category}")
-                for url, metadata in categorized_urls[category]:
+                for url, metadata in categories[category]:
                     if metadata['description']:
                         content.append(f"\n- [{metadata['title']}]({url}): {metadata['description']}")
                     else:
@@ -530,15 +157,12 @@ async def process_url(url, max_depth, max_pages):
         if not url.startswith(('http://', 'https://')):
             url = 'https://' + url
 
-        # Validate URL format
-        try:
-            result = urlparse(url)
-            if not all([result.scheme, result.netloc]):
-                return "", "Invalid URL format. Please enter a valid URL."
-        except:
+        # Validate URL
+        result = urlparse(url)
+        if not all([result.scheme, result.netloc]):
             return "", "Invalid URL format. Please enter a valid URL."
 
-        # Create crawler and process
+        # Process website
         crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
         await crawler.crawl_website(url)
         content = crawler.generate_llms_txt()
@@ -548,1758 +172,37 @@ async def process_url(url, max_depth, max_pages):
     except Exception as e:
         return "", f"Error: {str(e)}"
 
-# Create custom theme
-theme = gr.themes.Soft(
-    primary_hue="blue",
-    font="Open Sans"
-)
+# Create Gradio interface
+theme = gr.themes.Soft(primary_hue="blue", font="Open Sans")
 
-# Create the Gradio interface
-with gr.Blocks(theme=theme,     css="""
-    @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');
-    
-    .gradio-container {
-        font-family: 'Open Sans', sans-serif !important;
-    }
-    
-    .gr-button {
-        font-family: 'Open Sans', sans-serif !important;
-        font-weight: 600 !important;
-    }
-
-    /* Primary color customization */
-    .primary-btn {
-        background-color: #2436d4 !important;
-        color: white !important;
-    }
-
-    .primary-btn:hover {
-        background-color: #1c2aa8 !important;
-    }
-    
-    [data-testid="textbox"] {
-        font-family: 'Open Sans', sans-serif !important;
-    }
+with gr.Blocks(theme=theme, css="""
+    .primary-btn {background-color: #2436d4 !important;}
+    .primary-btn:hover {background-color: #1c2aa8 !important;}
 """) as iface:
     gr.Markdown("# llms.txt Generator")
-    gr.Markdown("Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.")
+    gr.Markdown("Generate an llms.txt file from a website following the specification.")
     
     with gr.Row():
         url_input = gr.Textbox(
             label="Website URL", 
-            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
-            info="The URL will be automatically prefixed with https:// if no protocol is specified."
+            placeholder="Enter the website URL (e.g., example.com)",
+            info="The URL will be automatically prefixed with https:// if not provided"
         )
     
     with gr.Row():
         with gr.Column():
-            depth_input = gr.Slider(
-                minimum=1, 
-                maximum=5, 
-                value=3, 
-                step=1, 
-                label="Maximum Crawl Depth",
-                info="Higher values will result in more thorough but slower crawling"
-            )
+            depth_input = gr.Slider(minimum=1, maximum=5, value=3, step=1, label="Maximum Crawl Depth")
         with gr.Column():
-            pages_input = gr.Slider(
-                minimum=10, 
-                maximum=100, 
-                value=50, 
-                step=10, 
-                label="Maximum Pages to Crawl",
-                info="Higher values will result in more comprehensive but slower results"
-            )
+            pages_input = gr.Slider(minimum=10, maximum=100, value=50, step=10, label="Maximum Pages")
     
     generate_btn = gr.Button("Generate llms.txt", variant="primary")
     
-    with gr.Row():
-        output = gr.Textbox(
-            label="Generated llms.txt Content",
-            lines=20,
-            max_lines=30,
-            show_copy_button=True,
-            container=True,
-            scale=2,
-            interactive=True
-        )
-    
-    status = gr.Textbox(label="Status")
-    
-    generate_btn.click(
-        fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)),
-        inputs=[url_input, depth_input, pages_input],
-        outputs=[output, status]
+    output = gr.Textbox(
+        label="Generated llms.txt Content",
+        lines=20,
+        show_copy_button=True,
+        container=True
     )
-
-# Launch the app
-if __name__ == "__main__":
-    iface.launch()
-,         # Remove pipe and everything after
-            r'\s*-\s*.*
-
-    def is_valid_url(self, url, base_domain):
-        """Check if URL is valid and belongs to the same domain"""
-        try:
-            parsed = urlparse(url)
-            base_parsed = urlparse(base_domain)
-            return (parsed.netloc == base_parsed.netloc and
-                   parsed.scheme in ['http', 'https'] and
-                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
-        except:
-            return False
-
-    def extract_content(self, soup):
-        """Extract meaningful content from HTML"""
-        # Remove script and style elements
-        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
-            element.decompose()
-
-        # Get main content
-        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
-        if main_content:
-            return self.normalize_text(main_content.get_text(strip=True))
-        return self.normalize_text(soup.get_text(strip=True))
-
-    def get_page_metadata(self, soup, url):
-        """Extract metadata from the page"""
-        metadata = {
-            'title': None,
-            'description': None,
-            'importance': 0,
-            'category': 'Optional'
-        }
-
-        # Title extraction with cleaning
-        title = (
-            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
-            soup.find('title').text if soup.find('title') else
-            soup.find('h1').text if soup.find('h1') else
-            url.split('/')[-1]
-        )
-        metadata['title'] = self.clean_title(title)
-
-        # Description extraction with cleaning
-        description = (
-            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
-            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
-            ""
-        )
-        metadata['description'] = self.clean_description(description)
-
-        # Calculate importance and category
-        url_lower = url.lower()
-        if 'docs' in url_lower or 'documentation' in url_lower:
-            metadata['importance'] = 5
-            metadata['category'] = 'Docs'
-        elif 'api' in url_lower:
-            metadata['importance'] = 4
-            metadata['category'] = 'API'
-        elif 'guide' in url_lower or 'tutorial' in url_lower:
-            metadata['importance'] = 3
-            metadata['category'] = 'Guides'
-        elif 'example' in url_lower:
-            metadata['importance'] = 2
-            metadata['category'] = 'Examples'
-        elif 'blog' in url_lower:
-            metadata['importance'] = 1
-            metadata['category'] = 'Blog'
-
-        return metadata
-
-    async def crawl_page(self, url, depth, base_domain):
-        """Crawl a single page and extract information"""
-        if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages:
-            return []
-
-        try:
-            response = requests.get(url, headers=self.headers, timeout=self.timeout)
-            response.encoding = 'utf-8'
-            response.raise_for_status()
-            self.visited_urls.add(url)
-
-            soup = BeautifulSoup(response.text, 'html.parser')
-            content = self.extract_content(soup)
-            metadata = self.get_page_metadata(soup, url)
-            
-            self.url_content[url] = content
-            self.url_metadata[url] = metadata
-
-            # Find all links
-            links = []
-            for a in soup.find_all('a', href=True):
-                next_url = urljoin(url, a['href'])
-                if self.is_valid_url(next_url, base_domain):
-                    links.append(next_url)
-
-            return links
-
-        except Exception as e:
-            logger.error(f"Error crawling {url}: {str(e)}")
-            return []
-
-    async def crawl_website(self, start_url):
-        """Crawl website starting from the given URL"""
-        base_domain = start_url
-        queue = [(start_url, 0)]
-        seen = {start_url}
-
-        while queue and len(self.visited_urls) < self.max_pages:
-            current_url, depth = queue.pop(0)
-            
-            if depth > self.max_depth:
-                continue
-
-            links = await self.crawl_page(current_url, depth, base_domain)
-            
-            for link in links:
-                if link not in seen:
-                    seen.add(link)
-                    queue.append((link, depth + 1))
-
-    def generate_llms_txt(self):
-        """Generate llms.txt content from crawled data"""
-        # Sort URLs by importance
-        sorted_urls = sorted(
-            self.url_metadata.items(),
-            key=lambda x: (x[1]['importance'], x[0]),
-            reverse=True
-        )
-
-        if not sorted_urls:
-            return "No content was found to generate llms.txt"
-
-        # Group URLs by category
-        categorized_urls = defaultdict(list)
-        for url, metadata in sorted_urls:
-            categorized_urls[metadata['category']].append((url, metadata))
-
-        # Generate content
-        content = []
-        
-        # Add main title and description
-        main_metadata = sorted_urls[0][1]
-        content.append(f"# {main_metadata['title']}")
-        if main_metadata['description']:
-            content.append(f"\n> {main_metadata['description']}")
-        
-        # Add categorized sections
-        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
-        
-        for category in priority_order:
-            if category in categorized_urls:
-                content.append(f"\n## {category}")
-                for url, metadata in categorized_urls[category]:
-                    if metadata['description']:
-                        content.append(f"\n- [{metadata['title']}]({url}): {metadata['description']}")
-                    else:
-                        content.append(f"\n- [{metadata['title']}]({url})")
-
-        return "\n".join(content)
-
-async def process_url(url, max_depth, max_pages):
-    """Process URL and generate llms.txt"""
-    try:
-        # Add https:// if not present
-        if not url.startswith(('http://', 'https://')):
-            url = 'https://' + url
-
-        # Validate URL format
-        try:
-            result = urlparse(url)
-            if not all([result.scheme, result.netloc]):
-                return "", "Invalid URL format. Please enter a valid URL."
-        except:
-            return "", "Invalid URL format. Please enter a valid URL."
-
-        # Create crawler and process
-        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
-        await crawler.crawl_website(url)
-        content = crawler.generate_llms_txt()
-        
-        return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
-    
-    except Exception as e:
-        return "", f"Error: {str(e)}"
-
-# Create custom theme
-theme = gr.themes.Soft(
-    primary_hue="blue",
-    font="Open Sans"
-)
-
-# Create the Gradio interface
-with gr.Blocks(theme=theme,     css="""
-    @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');
-    
-    .gradio-container {
-        font-family: 'Open Sans', sans-serif !important;
-    }
-    
-    .gr-button {
-        font-family: 'Open Sans', sans-serif !important;
-        font-weight: 600 !important;
-    }
-
-    /* Primary color customization */
-    .primary-btn {
-        background-color: #2436d4 !important;
-        color: white !important;
-    }
-
-    .primary-btn:hover {
-        background-color: #1c2aa8 !important;
-    }
-    
-    [data-testid="textbox"] {
-        font-family: 'Open Sans', sans-serif !important;
-    }
-""") as iface:
-    gr.Markdown("# llms.txt Generator")
-    gr.Markdown("Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.")
-    
-    with gr.Row():
-        url_input = gr.Textbox(
-            label="Website URL", 
-            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
-            info="The URL will be automatically prefixed with https:// if no protocol is specified."
-        )
-    
-    with gr.Row():
-        with gr.Column():
-            depth_input = gr.Slider(
-                minimum=1, 
-                maximum=5, 
-                value=3, 
-                step=1, 
-                label="Maximum Crawl Depth",
-                info="Higher values will result in more thorough but slower crawling"
-            )
-        with gr.Column():
-            pages_input = gr.Slider(
-                minimum=10, 
-                maximum=100, 
-                value=50, 
-                step=10, 
-                label="Maximum Pages to Crawl",
-                info="Higher values will result in more comprehensive but slower results"
-            )
-    
-    generate_btn = gr.Button("Generate llms.txt", variant="primary")
-    
-    with gr.Row():
-        output = gr.Textbox(
-            label="Generated llms.txt Content",
-            lines=20,
-            max_lines=30,
-            show_copy_button=True,
-            container=True,
-            scale=2,
-            interactive=True
-        )
-    
-    status = gr.Textbox(label="Status")
-    
-    generate_btn.click(
-        fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)),
-        inputs=[url_input, depth_input, pages_input],
-        outputs=[output, status]
-    )
-
-# Launch the app
-if __name__ == "__main__":
-    iface.launch()
-,          # Remove dash and everything after
-            r'\s*:\s*.*
-
-    def is_valid_url(self, url, base_domain):
-        """Check if URL is valid and belongs to the same domain"""
-        try:
-            parsed = urlparse(url)
-            base_parsed = urlparse(base_domain)
-            return (parsed.netloc == base_parsed.netloc and
-                   parsed.scheme in ['http', 'https'] and
-                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
-        except:
-            return False
-
-    def extract_content(self, soup):
-        """Extract meaningful content from HTML"""
-        # Remove script and style elements
-        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
-            element.decompose()
-
-        # Get main content
-        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
-        if main_content:
-            return self.normalize_text(main_content.get_text(strip=True))
-        return self.normalize_text(soup.get_text(strip=True))
-
-    def get_page_metadata(self, soup, url):
-        """Extract metadata from the page"""
-        metadata = {
-            'title': None,
-            'description': None,
-            'importance': 0,
-            'category': 'Optional'
-        }
-
-        # Title extraction with cleaning
-        title = (
-            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
-            soup.find('title').text if soup.find('title') else
-            soup.find('h1').text if soup.find('h1') else
-            url.split('/')[-1]
-        )
-        metadata['title'] = self.clean_title(title)
-
-        # Description extraction with cleaning
-        description = (
-            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
-            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
-            ""
-        )
-        metadata['description'] = self.clean_description(description)
-
-        # Calculate importance and category
-        url_lower = url.lower()
-        if 'docs' in url_lower or 'documentation' in url_lower:
-            metadata['importance'] = 5
-            metadata['category'] = 'Docs'
-        elif 'api' in url_lower:
-            metadata['importance'] = 4
-            metadata['category'] = 'API'
-        elif 'guide' in url_lower or 'tutorial' in url_lower:
-            metadata['importance'] = 3
-            metadata['category'] = 'Guides'
-        elif 'example' in url_lower:
-            metadata['importance'] = 2
-            metadata['category'] = 'Examples'
-        elif 'blog' in url_lower:
-            metadata['importance'] = 1
-            metadata['category'] = 'Blog'
-
-        return metadata
-
-    async def crawl_page(self, url, depth, base_domain):
-        """Crawl a single page and extract information"""
-        if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages:
-            return []
-
-        try:
-            response = requests.get(url, headers=self.headers, timeout=self.timeout)
-            response.encoding = 'utf-8'
-            response.raise_for_status()
-            self.visited_urls.add(url)
-
-            soup = BeautifulSoup(response.text, 'html.parser')
-            content = self.extract_content(soup)
-            metadata = self.get_page_metadata(soup, url)
-            
-            self.url_content[url] = content
-            self.url_metadata[url] = metadata
-
-            # Find all links
-            links = []
-            for a in soup.find_all('a', href=True):
-                next_url = urljoin(url, a['href'])
-                if self.is_valid_url(next_url, base_domain):
-                    links.append(next_url)
-
-            return links
-
-        except Exception as e:
-            logger.error(f"Error crawling {url}: {str(e)}")
-            return []
-
-    async def crawl_website(self, start_url):
-        """Crawl website starting from the given URL"""
-        base_domain = start_url
-        queue = [(start_url, 0)]
-        seen = {start_url}
-
-        while queue and len(self.visited_urls) < self.max_pages:
-            current_url, depth = queue.pop(0)
-            
-            if depth > self.max_depth:
-                continue
-
-            links = await self.crawl_page(current_url, depth, base_domain)
-            
-            for link in links:
-                if link not in seen:
-                    seen.add(link)
-                    queue.append((link, depth + 1))
-
-    def generate_llms_txt(self):
-        """Generate llms.txt content from crawled data"""
-        # Sort URLs by importance
-        sorted_urls = sorted(
-            self.url_metadata.items(),
-            key=lambda x: (x[1]['importance'], x[0]),
-            reverse=True
-        )
-
-        if not sorted_urls:
-            return "No content was found to generate llms.txt"
-
-        # Group URLs by category
-        categorized_urls = defaultdict(list)
-        for url, metadata in sorted_urls:
-            categorized_urls[metadata['category']].append((url, metadata))
-
-        # Generate content
-        content = []
-        
-        # Add main title and description
-        main_metadata = sorted_urls[0][1]
-        content.append(f"# {main_metadata['title']}")
-        if main_metadata['description']:
-            content.append(f"\n> {main_metadata['description']}")
-        
-        # Add categorized sections
-        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
-        
-        for category in priority_order:
-            if category in categorized_urls:
-                content.append(f"\n## {category}")
-                for url, metadata in categorized_urls[category]:
-                    if metadata['description']:
-                        content.append(f"\n- [{metadata['title']}]({url}): {metadata['description']}")
-                    else:
-                        content.append(f"\n- [{metadata['title']}]({url})")
-
-        return "\n".join(content)
-
-async def process_url(url, max_depth, max_pages):
-    """Process URL and generate llms.txt"""
-    try:
-        # Add https:// if not present
-        if not url.startswith(('http://', 'https://')):
-            url = 'https://' + url
-
-        # Validate URL format
-        try:
-            result = urlparse(url)
-            if not all([result.scheme, result.netloc]):
-                return "", "Invalid URL format. Please enter a valid URL."
-        except:
-            return "", "Invalid URL format. Please enter a valid URL."
-
-        # Create crawler and process
-        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
-        await crawler.crawl_website(url)
-        content = crawler.generate_llms_txt()
-        
-        return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
-    
-    except Exception as e:
-        return "", f"Error: {str(e)}"
-
-# Create custom theme
-theme = gr.themes.Soft(
-    primary_hue="blue",
-    font="Open Sans"
-)
-
-# Create the Gradio interface
-with gr.Blocks(theme=theme,     css="""
-    @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');
-    
-    .gradio-container {
-        font-family: 'Open Sans', sans-serif !important;
-    }
-    
-    .gr-button {
-        font-family: 'Open Sans', sans-serif !important;
-        font-weight: 600 !important;
-    }
-
-    /* Primary color customization */
-    .primary-btn {
-        background-color: #2436d4 !important;
-        color: white !important;
-    }
-
-    .primary-btn:hover {
-        background-color: #1c2aa8 !important;
-    }
-    
-    [data-testid="textbox"] {
-        font-family: 'Open Sans', sans-serif !important;
-    }
-""") as iface:
-    gr.Markdown("# llms.txt Generator")
-    gr.Markdown("Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.")
-    
-    with gr.Row():
-        url_input = gr.Textbox(
-            label="Website URL", 
-            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
-            info="The URL will be automatically prefixed with https:// if no protocol is specified."
-        )
-    
-    with gr.Row():
-        with gr.Column():
-            depth_input = gr.Slider(
-                minimum=1, 
-                maximum=5, 
-                value=3, 
-                step=1, 
-                label="Maximum Crawl Depth",
-                info="Higher values will result in more thorough but slower crawling"
-            )
-        with gr.Column():
-            pages_input = gr.Slider(
-                minimum=10, 
-                maximum=100, 
-                value=50, 
-                step=10, 
-                label="Maximum Pages to Crawl",
-                info="Higher values will result in more comprehensive but slower results"
-            )
-    
-    generate_btn = gr.Button("Generate llms.txt", variant="primary")
-    
-    with gr.Row():
-        output = gr.Textbox(
-            label="Generated llms.txt Content",
-            lines=20,
-            max_lines=30,
-            show_copy_button=True,
-            container=True,
-            scale=2,
-            interactive=True
-        )
-    
-    status = gr.Textbox(label="Status")
-    
-    generate_btn.click(
-        fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)),
-        inputs=[url_input, depth_input, pages_input],
-        outputs=[output, status]
-    )
-
-# Launch the app
-if __name__ == "__main__":
-    iface.launch()
-,          # Remove colon and everything after
-            r'#.*
-
-    def is_valid_url(self, url, base_domain):
-        """Check if URL is valid and belongs to the same domain"""
-        try:
-            parsed = urlparse(url)
-            base_parsed = urlparse(base_domain)
-            return (parsed.netloc == base_parsed.netloc and
-                   parsed.scheme in ['http', 'https'] and
-                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
-        except:
-            return False
-
-    def extract_content(self, soup):
-        """Extract meaningful content from HTML"""
-        # Remove script and style elements
-        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
-            element.decompose()
-
-        # Get main content
-        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
-        if main_content:
-            return self.normalize_text(main_content.get_text(strip=True))
-        return self.normalize_text(soup.get_text(strip=True))
-
-    def get_page_metadata(self, soup, url):
-        """Extract metadata from the page"""
-        metadata = {
-            'title': None,
-            'description': None,
-            'importance': 0,
-            'category': 'Optional'
-        }
-
-        # Title extraction with cleaning
-        title = (
-            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
-            soup.find('title').text if soup.find('title') else
-            soup.find('h1').text if soup.find('h1') else
-            url.split('/')[-1]
-        )
-        metadata['title'] = self.clean_title(title)
-
-        # Description extraction with cleaning
-        description = (
-            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
-            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
-            ""
-        )
-        metadata['description'] = self.clean_description(description)
-
-        # Calculate importance and category
-        url_lower = url.lower()
-        if 'docs' in url_lower or 'documentation' in url_lower:
-            metadata['importance'] = 5
-            metadata['category'] = 'Docs'
-        elif 'api' in url_lower:
-            metadata['importance'] = 4
-            metadata['category'] = 'API'
-        elif 'guide' in url_lower or 'tutorial' in url_lower:
-            metadata['importance'] = 3
-            metadata['category'] = 'Guides'
-        elif 'example' in url_lower:
-            metadata['importance'] = 2
-            metadata['category'] = 'Examples'
-        elif 'blog' in url_lower:
-            metadata['importance'] = 1
-            metadata['category'] = 'Blog'
-
-        return metadata
-
-    async def crawl_page(self, url, depth, base_domain):
-        """Crawl a single page and extract information"""
-        if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages:
-            return []
-
-        try:
-            response = requests.get(url, headers=self.headers, timeout=self.timeout)
-            response.encoding = 'utf-8'
-            response.raise_for_status()
-            self.visited_urls.add(url)
-
-            soup = BeautifulSoup(response.text, 'html.parser')
-            content = self.extract_content(soup)
-            metadata = self.get_page_metadata(soup, url)
-            
-            self.url_content[url] = content
-            self.url_metadata[url] = metadata
-
-            # Find all links
-            links = []
-            for a in soup.find_all('a', href=True):
-                next_url = urljoin(url, a['href'])
-                if self.is_valid_url(next_url, base_domain):
-                    links.append(next_url)
-
-            return links
-
-        except Exception as e:
-            logger.error(f"Error crawling {url}: {str(e)}")
-            return []
-
-    async def crawl_website(self, start_url):
-        """Crawl website starting from the given URL"""
-        base_domain = start_url
-        queue = [(start_url, 0)]
-        seen = {start_url}
-
-        while queue and len(self.visited_urls) < self.max_pages:
-            current_url, depth = queue.pop(0)
-            
-            if depth > self.max_depth:
-                continue
-
-            links = await self.crawl_page(current_url, depth, base_domain)
-            
-            for link in links:
-                if link not in seen:
-                    seen.add(link)
-                    queue.append((link, depth + 1))
-
-    def generate_llms_txt(self):
-        """Generate llms.txt content from crawled data"""
-        # Sort URLs by importance
-        sorted_urls = sorted(
-            self.url_metadata.items(),
-            key=lambda x: (x[1]['importance'], x[0]),
-            reverse=True
-        )
-
-        if not sorted_urls:
-            return "No content was found to generate llms.txt"
-
-        # Group URLs by category
-        categorized_urls = defaultdict(list)
-        for url, metadata in sorted_urls:
-            categorized_urls[metadata['category']].append((url, metadata))
-
-        # Generate content
-        content = []
-        
-        # Add main title and description
-        main_metadata = sorted_urls[0][1]
-        content.append(f"# {main_metadata['title']}")
-        if main_metadata['description']:
-            content.append(f"\n> {main_metadata['description']}")
-        
-        # Add categorized sections
-        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
-        
-        for category in priority_order:
-            if category in categorized_urls:
-                content.append(f"\n## {category}")
-                for url, metadata in categorized_urls[category]:
-                    if metadata['description']:
-                        content.append(f"\n- [{metadata['title']}]({url}): {metadata['description']}")
-                    else:
-                        content.append(f"\n- [{metadata['title']}]({url})")
-
-        return "\n".join(content)
-
-async def process_url(url, max_depth, max_pages):
-    """Process URL and generate llms.txt"""
-    try:
-        # Add https:// if not present
-        if not url.startswith(('http://', 'https://')):
-            url = 'https://' + url
-
-        # Validate URL format
-        try:
-            result = urlparse(url)
-            if not all([result.scheme, result.netloc]):
-                return "", "Invalid URL format. Please enter a valid URL."
-        except:
-            return "", "Invalid URL format. Please enter a valid URL."
-
-        # Create crawler and process
-        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
-        await crawler.crawl_website(url)
-        content = crawler.generate_llms_txt()
-        
-        return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
-    
-    except Exception as e:
-        return "", f"Error: {str(e)}"
-
-# Create custom theme
-theme = gr.themes.Soft(
-    primary_hue="blue",
-    font="Open Sans"
-)
-
-# Create the Gradio interface
-with gr.Blocks(theme=theme,     css="""
-    @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');
-    
-    .gradio-container {
-        font-family: 'Open Sans', sans-serif !important;
-    }
-    
-    .gr-button {
-        font-family: 'Open Sans', sans-serif !important;
-        font-weight: 600 !important;
-    }
-
-    /* Primary color customization */
-    .primary-btn {
-        background-color: #2436d4 !important;
-        color: white !important;
-    }
-
-    .primary-btn:hover {
-        background-color: #1c2aa8 !important;
-    }
-    
-    [data-testid="textbox"] {
-        font-family: 'Open Sans', sans-serif !important;
-    }
-""") as iface:
-    gr.Markdown("# llms.txt Generator")
-    gr.Markdown("Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.")
-    
-    with gr.Row():
-        url_input = gr.Textbox(
-            label="Website URL", 
-            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
-            info="The URL will be automatically prefixed with https:// if no protocol is specified."
-        )
-    
-    with gr.Row():
-        with gr.Column():
-            depth_input = gr.Slider(
-                minimum=1, 
-                maximum=5, 
-                value=3, 
-                step=1, 
-                label="Maximum Crawl Depth",
-                info="Higher values will result in more thorough but slower crawling"
-            )
-        with gr.Column():
-            pages_input = gr.Slider(
-                minimum=10, 
-                maximum=100, 
-                value=50, 
-                step=10, 
-                label="Maximum Pages to Crawl",
-                info="Higher values will result in more comprehensive but slower results"
-            )
-    
-    generate_btn = gr.Button("Generate llms.txt", variant="primary")
-    
-    with gr.Row():
-        output = gr.Textbox(
-            label="Generated llms.txt Content",
-            lines=20,
-            max_lines=30,
-            show_copy_button=True,
-            container=True,
-            scale=2,
-            interactive=True
-        )
-    
-    status = gr.Textbox(label="Status")
-    
-    generate_btn.click(
-        fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)),
-        inputs=[url_input, depth_input, pages_input],
-        outputs=[output, status]
-    )
-
-# Launch the app
-if __name__ == "__main__":
-    iface.launch()
-,                # Remove hash and everything after
-            r'\s*\|.*
-
-    def is_valid_url(self, url, base_domain):
-        """Check if URL is valid and belongs to the same domain"""
-        try:
-            parsed = urlparse(url)
-            base_parsed = urlparse(base_domain)
-            return (parsed.netloc == base_parsed.netloc and
-                   parsed.scheme in ['http', 'https'] and
-                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
-        except:
-            return False
-
-    def extract_content(self, soup):
-        """Extract meaningful content from HTML"""
-        # Remove script and style elements
-        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
-            element.decompose()
-
-        # Get main content
-        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
-        if main_content:
-            return self.normalize_text(main_content.get_text(strip=True))
-        return self.normalize_text(soup.get_text(strip=True))
-
-    def get_page_metadata(self, soup, url):
-        """Extract metadata from the page"""
-        metadata = {
-            'title': None,
-            'description': None,
-            'importance': 0,
-            'category': 'Optional'
-        }
-
-        # Title extraction with cleaning
-        title = (
-            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
-            soup.find('title').text if soup.find('title') else
-            soup.find('h1').text if soup.find('h1') else
-            url.split('/')[-1]
-        )
-        metadata['title'] = self.clean_title(title)
-
-        # Description extraction with cleaning
-        description = (
-            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
-            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
-            ""
-        )
-        metadata['description'] = self.clean_description(description)
-
-        # Calculate importance and category
-        url_lower = url.lower()
-        if 'docs' in url_lower or 'documentation' in url_lower:
-            metadata['importance'] = 5
-            metadata['category'] = 'Docs'
-        elif 'api' in url_lower:
-            metadata['importance'] = 4
-            metadata['category'] = 'API'
-        elif 'guide' in url_lower or 'tutorial' in url_lower:
-            metadata['importance'] = 3
-            metadata['category'] = 'Guides'
-        elif 'example' in url_lower:
-            metadata['importance'] = 2
-            metadata['category'] = 'Examples'
-        elif 'blog' in url_lower:
-            metadata['importance'] = 1
-            metadata['category'] = 'Blog'
-
-        return metadata
-
-    async def crawl_page(self, url, depth, base_domain):
-        """Crawl a single page and extract information"""
-        if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages:
-            return []
-
-        try:
-            response = requests.get(url, headers=self.headers, timeout=self.timeout)
-            response.encoding = 'utf-8'
-            response.raise_for_status()
-            self.visited_urls.add(url)
-
-            soup = BeautifulSoup(response.text, 'html.parser')
-            content = self.extract_content(soup)
-            metadata = self.get_page_metadata(soup, url)
-            
-            self.url_content[url] = content
-            self.url_metadata[url] = metadata
-
-            # Find all links
-            links = []
-            for a in soup.find_all('a', href=True):
-                next_url = urljoin(url, a['href'])
-                if self.is_valid_url(next_url, base_domain):
-                    links.append(next_url)
-
-            return links
-
-        except Exception as e:
-            logger.error(f"Error crawling {url}: {str(e)}")
-            return []
-
-    async def crawl_website(self, start_url):
-        """Crawl website starting from the given URL"""
-        base_domain = start_url
-        queue = [(start_url, 0)]
-        seen = {start_url}
-
-        while queue and len(self.visited_urls) < self.max_pages:
-            current_url, depth = queue.pop(0)
-            
-            if depth > self.max_depth:
-                continue
-
-            links = await self.crawl_page(current_url, depth, base_domain)
-            
-            for link in links:
-                if link not in seen:
-                    seen.add(link)
-                    queue.append((link, depth + 1))
-
-    def generate_llms_txt(self):
-        """Generate llms.txt content from crawled data"""
-        # Sort URLs by importance
-        sorted_urls = sorted(
-            self.url_metadata.items(),
-            key=lambda x: (x[1]['importance'], x[0]),
-            reverse=True
-        )
-
-        if not sorted_urls:
-            return "No content was found to generate llms.txt"
-
-        # Group URLs by category
-        categorized_urls = defaultdict(list)
-        for url, metadata in sorted_urls:
-            categorized_urls[metadata['category']].append((url, metadata))
-
-        # Generate content
-        content = []
-        
-        # Add main title and description
-        main_metadata = sorted_urls[0][1]
-        content.append(f"# {main_metadata['title']}")
-        if main_metadata['description']:
-            content.append(f"\n> {main_metadata['description']}")
-        
-        # Add categorized sections
-        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
-        
-        for category in priority_order:
-            if category in categorized_urls:
-                content.append(f"\n## {category}")
-                for url, metadata in categorized_urls[category]:
-                    if metadata['description']:
-                        content.append(f"\n- [{metadata['title']}]({url}): {metadata['description']}")
-                    else:
-                        content.append(f"\n- [{metadata['title']}]({url})")
-
-        return "\n".join(content)
-
-async def process_url(url, max_depth, max_pages):
-    """Process URL and generate llms.txt"""
-    try:
-        # Add https:// if not present
-        if not url.startswith(('http://', 'https://')):
-            url = 'https://' + url
-
-        # Validate URL format
-        try:
-            result = urlparse(url)
-            if not all([result.scheme, result.netloc]):
-                return "", "Invalid URL format. Please enter a valid URL."
-        except:
-            return "", "Invalid URL format. Please enter a valid URL."
-
-        # Create crawler and process
-        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
-        await crawler.crawl_website(url)
-        content = crawler.generate_llms_txt()
-        
-        return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
-    
-    except Exception as e:
-        return "", f"Error: {str(e)}"
-
-# Create custom theme
-theme = gr.themes.Soft(
-    primary_hue="blue",
-    font="Open Sans"
-)
-
-# Create the Gradio interface
-with gr.Blocks(theme=theme,     css="""
-    @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');
-    
-    .gradio-container {
-        font-family: 'Open Sans', sans-serif !important;
-    }
-    
-    .gr-button {
-        font-family: 'Open Sans', sans-serif !important;
-        font-weight: 600 !important;
-    }
-
-    /* Primary color customization */
-    .primary-btn {
-        background-color: #2436d4 !important;
-        color: white !important;
-    }
-
-    .primary-btn:hover {
-        background-color: #1c2aa8 !important;
-    }
-    
-    [data-testid="textbox"] {
-        font-family: 'Open Sans', sans-serif !important;
-    }
-""") as iface:
-    gr.Markdown("# llms.txt Generator")
-    gr.Markdown("Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.")
-    
-    with gr.Row():
-        url_input = gr.Textbox(
-            label="Website URL", 
-            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
-            info="The URL will be automatically prefixed with https:// if no protocol is specified."
-        )
-    
-    with gr.Row():
-        with gr.Column():
-            depth_input = gr.Slider(
-                minimum=1, 
-                maximum=5, 
-                value=3, 
-                step=1, 
-                label="Maximum Crawl Depth",
-                info="Higher values will result in more thorough but slower crawling"
-            )
-        with gr.Column():
-            pages_input = gr.Slider(
-                minimum=10, 
-                maximum=100, 
-                value=50, 
-                step=10, 
-                label="Maximum Pages to Crawl",
-                info="Higher values will result in more comprehensive but slower results"
-            )
-    
-    generate_btn = gr.Button("Generate llms.txt", variant="primary")
-    
-    with gr.Row():
-        output = gr.Textbox(
-            label="Generated llms.txt Content",
-            lines=20,
-            max_lines=30,
-            show_copy_button=True,
-            container=True,
-            scale=2,
-            interactive=True
-        )
-    
-    status = gr.Textbox(label="Status")
-    
-    generate_btn.click(
-        fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)),
-        inputs=[url_input, depth_input, pages_input],
-        outputs=[output, status]
-    )
-
-# Launch the app
-if __name__ == "__main__":
-    iface.launch()
-,            # Remove pipe and everything after
-            r'\s*•.*
-
-    def is_valid_url(self, url, base_domain):
-        """Check if URL is valid and belongs to the same domain"""
-        try:
-            parsed = urlparse(url)
-            base_parsed = urlparse(base_domain)
-            return (parsed.netloc == base_parsed.netloc and
-                   parsed.scheme in ['http', 'https'] and
-                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
-        except:
-            return False
-
-    def extract_content(self, soup):
-        """Extract meaningful content from HTML"""
-        # Remove script and style elements
-        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
-            element.decompose()
-
-        # Get main content
-        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
-        if main_content:
-            return self.normalize_text(main_content.get_text(strip=True))
-        return self.normalize_text(soup.get_text(strip=True))
-
-    def get_page_metadata(self, soup, url):
-        """Extract metadata from the page"""
-        metadata = {
-            'title': None,
-            'description': None,
-            'importance': 0,
-            'category': 'Optional'
-        }
-
-        # Title extraction with cleaning
-        title = (
-            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
-            soup.find('title').text if soup.find('title') else
-            soup.find('h1').text if soup.find('h1') else
-            url.split('/')[-1]
-        )
-        metadata['title'] = self.clean_title(title)
-
-        # Description extraction with cleaning
-        description = (
-            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
-            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
-            ""
-        )
-        metadata['description'] = self.clean_description(description)
-
-        # Calculate importance and category
-        url_lower = url.lower()
-        if 'docs' in url_lower or 'documentation' in url_lower:
-            metadata['importance'] = 5
-            metadata['category'] = 'Docs'
-        elif 'api' in url_lower:
-            metadata['importance'] = 4
-            metadata['category'] = 'API'
-        elif 'guide' in url_lower or 'tutorial' in url_lower:
-            metadata['importance'] = 3
-            metadata['category'] = 'Guides'
-        elif 'example' in url_lower:
-            metadata['importance'] = 2
-            metadata['category'] = 'Examples'
-        elif 'blog' in url_lower:
-            metadata['importance'] = 1
-            metadata['category'] = 'Blog'
-
-        return metadata
-
-    async def crawl_page(self, url, depth, base_domain):
-        """Crawl a single page and extract information"""
-        if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages:
-            return []
-
-        try:
-            response = requests.get(url, headers=self.headers, timeout=self.timeout)
-            response.encoding = 'utf-8'
-            response.raise_for_status()
-            self.visited_urls.add(url)
-
-            soup = BeautifulSoup(response.text, 'html.parser')
-            content = self.extract_content(soup)
-            metadata = self.get_page_metadata(soup, url)
-            
-            self.url_content[url] = content
-            self.url_metadata[url] = metadata
-
-            # Find all links
-            links = []
-            for a in soup.find_all('a', href=True):
-                next_url = urljoin(url, a['href'])
-                if self.is_valid_url(next_url, base_domain):
-                    links.append(next_url)
-
-            return links
-
-        except Exception as e:
-            logger.error(f"Error crawling {url}: {str(e)}")
-            return []
-
-    async def crawl_website(self, start_url):
-        """Crawl website starting from the given URL"""
-        base_domain = start_url
-        queue = [(start_url, 0)]
-        seen = {start_url}
-
-        while queue and len(self.visited_urls) < self.max_pages:
-            current_url, depth = queue.pop(0)
-            
-            if depth > self.max_depth:
-                continue
-
-            links = await self.crawl_page(current_url, depth, base_domain)
-            
-            for link in links:
-                if link not in seen:
-                    seen.add(link)
-                    queue.append((link, depth + 1))
-
-    def generate_llms_txt(self):
-        """Generate llms.txt content from crawled data"""
-        # Sort URLs by importance
-        sorted_urls = sorted(
-            self.url_metadata.items(),
-            key=lambda x: (x[1]['importance'], x[0]),
-            reverse=True
-        )
-
-        if not sorted_urls:
-            return "No content was found to generate llms.txt"
-
-        # Group URLs by category
-        categorized_urls = defaultdict(list)
-        for url, metadata in sorted_urls:
-            categorized_urls[metadata['category']].append((url, metadata))
-
-        # Generate content
-        content = []
-        
-        # Add main title and description
-        main_metadata = sorted_urls[0][1]
-        content.append(f"# {main_metadata['title']}")
-        if main_metadata['description']:
-            content.append(f"\n> {main_metadata['description']}")
-        
-        # Add categorized sections
-        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
-        
-        for category in priority_order:
-            if category in categorized_urls:
-                content.append(f"\n## {category}")
-                for url, metadata in categorized_urls[category]:
-                    if metadata['description']:
-                        content.append(f"\n- [{metadata['title']}]({url}): {metadata['description']}")
-                    else:
-                        content.append(f"\n- [{metadata['title']}]({url})")
-
-        return "\n".join(content)
-
-async def process_url(url, max_depth, max_pages):
-    """Process URL and generate llms.txt"""
-    try:
-        # Add https:// if not present
-        if not url.startswith(('http://', 'https://')):
-            url = 'https://' + url
-
-        # Validate URL format
-        try:
-            result = urlparse(url)
-            if not all([result.scheme, result.netloc]):
-                return "", "Invalid URL format. Please enter a valid URL."
-        except:
-            return "", "Invalid URL format. Please enter a valid URL."
-
-        # Create crawler and process
-        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
-        await crawler.crawl_website(url)
-        content = crawler.generate_llms_txt()
-        
-        return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
-    
-    except Exception as e:
-        return "", f"Error: {str(e)}"
-
-# Create custom theme
-theme = gr.themes.Soft(
-    primary_hue="blue",
-    font="Open Sans"
-)
-
-# Create the Gradio interface
-with gr.Blocks(theme=theme,     css="""
-    @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');
-    
-    .gradio-container {
-        font-family: 'Open Sans', sans-serif !important;
-    }
-    
-    .gr-button {
-        font-family: 'Open Sans', sans-serif !important;
-        font-weight: 600 !important;
-    }
-
-    /* Primary color customization */
-    .primary-btn {
-        background-color: #2436d4 !important;
-        color: white !important;
-    }
-
-    .primary-btn:hover {
-        background-color: #1c2aa8 !important;
-    }
-    
-    [data-testid="textbox"] {
-        font-family: 'Open Sans', sans-serif !important;
-    }
-""") as iface:
-    gr.Markdown("# llms.txt Generator")
-    gr.Markdown("Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.")
-    
-    with gr.Row():
-        url_input = gr.Textbox(
-            label="Website URL", 
-            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
-            info="The URL will be automatically prefixed with https:// if no protocol is specified."
-        )
-    
-    with gr.Row():
-        with gr.Column():
-            depth_input = gr.Slider(
-                minimum=1, 
-                maximum=5, 
-                value=3, 
-                step=1, 
-                label="Maximum Crawl Depth",
-                info="Higher values will result in more thorough but slower crawling"
-            )
-        with gr.Column():
-            pages_input = gr.Slider(
-                minimum=10, 
-                maximum=100, 
-                value=50, 
-                step=10, 
-                label="Maximum Pages to Crawl",
-                info="Higher values will result in more comprehensive but slower results"
-            )
-    
-    generate_btn = gr.Button("Generate llms.txt", variant="primary")
-    
-    with gr.Row():
-        output = gr.Textbox(
-            label="Generated llms.txt Content",
-            lines=20,
-            max_lines=30,
-            show_copy_button=True,
-            container=True,
-            scale=2,
-            interactive=True
-        )
-    
-    status = gr.Textbox(label="Status")
-    
-    generate_btn.click(
-        fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)),
-        inputs=[url_input, depth_input, pages_input],
-        outputs=[output, status]
-    )
-
-# Launch the app
-if __name__ == "__main__":
-    iface.launch()
-,             # Remove bullet and everything after
-            r'^\s*Welcome to\s+',   # Remove "Welcome to" at start
-            r'docusaurus_skipToContent_fallback',  # Remove docusaurus fragments
-        ]
-        
-        for pattern in patterns:
-            title = re.sub(pattern, '', title)
-        
-        # Clean up whitespace
-        title = ' '.join(title.split())
-        return title.strip()
-
-    def clean_description(self, desc):
-        """Clean and format descriptions"""
-        if not desc:
-            return ""
-            
-        desc = self.normalize_text(desc)
-        
-        # Remove duplicate sentences
-        sentences = re.split(r'(?<=[.!?])\s+', desc)
-        unique_sentences = []
-        seen_sentences = set()
-        
-        for sentence in sentences:
-            sentence = sentence.strip()
-            sentence_lower = sentence.lower()
-            if sentence_lower not in seen_sentences and sentence:
-                if not sentence[-1] in '.!?':
-                    sentence += '.'
-                unique_sentences.append(sentence)
-                seen_sentences.add(sentence_lower)
-        
-        cleaned_desc = ' '.join(unique_sentences)
-        return cleaned_desc
-
-    def is_valid_url(self, url, base_domain):
-        """Check if URL is valid and belongs to the same domain"""
-        try:
-            parsed = urlparse(url)
-            base_parsed = urlparse(base_domain)
-            return (parsed.netloc == base_parsed.netloc and
-                   parsed.scheme in ['http', 'https'] and
-                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
-        except:
-            return False
-
-    def extract_content(self, soup):
-        """Extract meaningful content from HTML"""
-        # Remove script and style elements
-        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
-            element.decompose()
-
-        # Get main content
-        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
-        if main_content:
-            return self.normalize_text(main_content.get_text(strip=True))
-        return self.normalize_text(soup.get_text(strip=True))
-
-    def get_page_metadata(self, soup, url):
-        """Extract metadata from the page"""
-        metadata = {
-            'title': None,
-            'description': None,
-            'importance': 0,
-            'category': 'Optional'
-        }
-
-        # Title extraction with cleaning
-        title = (
-            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
-            soup.find('title').text if soup.find('title') else
-            soup.find('h1').text if soup.find('h1') else
-            url.split('/')[-1]
-        )
-        metadata['title'] = self.clean_title(title)
-
-        # Description extraction with cleaning
-        description = (
-            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
-            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
-            ""
-        )
-        metadata['description'] = self.clean_description(description)
-
-        # Calculate importance and category
-        url_lower = url.lower()
-        if 'docs' in url_lower or 'documentation' in url_lower:
-            metadata['importance'] = 5
-            metadata['category'] = 'Docs'
-        elif 'api' in url_lower:
-            metadata['importance'] = 4
-            metadata['category'] = 'API'
-        elif 'guide' in url_lower or 'tutorial' in url_lower:
-            metadata['importance'] = 3
-            metadata['category'] = 'Guides'
-        elif 'example' in url_lower:
-            metadata['importance'] = 2
-            metadata['category'] = 'Examples'
-        elif 'blog' in url_lower:
-            metadata['importance'] = 1
-            metadata['category'] = 'Blog'
-
-        return metadata
-
-    async def crawl_page(self, url, depth, base_domain):
-        """Crawl a single page and extract information"""
-        if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages:
-            return []
-
-        try:
-            response = requests.get(url, headers=self.headers, timeout=self.timeout)
-            response.encoding = 'utf-8'
-            response.raise_for_status()
-            self.visited_urls.add(url)
-
-            soup = BeautifulSoup(response.text, 'html.parser')
-            content = self.extract_content(soup)
-            metadata = self.get_page_metadata(soup, url)
-            
-            self.url_content[url] = content
-            self.url_metadata[url] = metadata
-
-            # Find all links
-            links = []
-            for a in soup.find_all('a', href=True):
-                next_url = urljoin(url, a['href'])
-                if self.is_valid_url(next_url, base_domain):
-                    links.append(next_url)
-
-            return links
-
-        except Exception as e:
-            logger.error(f"Error crawling {url}: {str(e)}")
-            return []
-
-    async def crawl_website(self, start_url):
-        """Crawl website starting from the given URL"""
-        base_domain = start_url
-        queue = [(start_url, 0)]
-        seen = {start_url}
-
-        while queue and len(self.visited_urls) < self.max_pages:
-            current_url, depth = queue.pop(0)
-            
-            if depth > self.max_depth:
-                continue
-
-            links = await self.crawl_page(current_url, depth, base_domain)
-            
-            for link in links:
-                if link not in seen:
-                    seen.add(link)
-                    queue.append((link, depth + 1))
-
-    def generate_llms_txt(self):
-        """Generate llms.txt content from crawled data"""
-        # Sort URLs by importance
-        sorted_urls = sorted(
-            self.url_metadata.items(),
-            key=lambda x: (x[1]['importance'], x[0]),
-            reverse=True
-        )
-
-        if not sorted_urls:
-            return "No content was found to generate llms.txt"
-
-        # Group URLs by category
-        categorized_urls = defaultdict(list)
-        for url, metadata in sorted_urls:
-            categorized_urls[metadata['category']].append((url, metadata))
-
-        # Generate content
-        content = []
-        
-        # Add main title and description
-        main_metadata = sorted_urls[0][1]
-        content.append(f"# {main_metadata['title']}")
-        if main_metadata['description']:
-            content.append(f"\n> {main_metadata['description']}")
-        
-        # Add categorized sections
-        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
-        
-        for category in priority_order:
-            if category in categorized_urls:
-                content.append(f"\n## {category}")
-                for url, metadata in categorized_urls[category]:
-                    if metadata['description']:
-                        content.append(f"\n- [{metadata['title']}]({url}): {metadata['description']}")
-                    else:
-                        content.append(f"\n- [{metadata['title']}]({url})")
-
-        return "\n".join(content)
-
-async def process_url(url, max_depth, max_pages):
-    """Process URL and generate llms.txt"""
-    try:
-        # Add https:// if not present
-        if not url.startswith(('http://', 'https://')):
-            url = 'https://' + url
-
-        # Validate URL format
-        try:
-            result = urlparse(url)
-            if not all([result.scheme, result.netloc]):
-                return "", "Invalid URL format. Please enter a valid URL."
-        except:
-            return "", "Invalid URL format. Please enter a valid URL."
-
-        # Create crawler and process
-        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
-        await crawler.crawl_website(url)
-        content = crawler.generate_llms_txt()
-        
-        return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
-    
-    except Exception as e:
-        return "", f"Error: {str(e)}"
-
-# Create custom theme
-theme = gr.themes.Soft(
-    primary_hue="blue",
-    font="Open Sans"
-)
-
-# Create the Gradio interface
-with gr.Blocks(theme=theme,     css="""
-    @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');
-    
-    .gradio-container {
-        font-family: 'Open Sans', sans-serif !important;
-    }
-    
-    .gr-button {
-        font-family: 'Open Sans', sans-serif !important;
-        font-weight: 600 !important;
-    }
-
-    /* Primary color customization */
-    .primary-btn {
-        background-color: #2436d4 !important;
-        color: white !important;
-    }
-
-    .primary-btn:hover {
-        background-color: #1c2aa8 !important;
-    }
-    
-    [data-testid="textbox"] {
-        font-family: 'Open Sans', sans-serif !important;
-    }
-""") as iface:
-    gr.Markdown("# llms.txt Generator")
-    gr.Markdown("Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.")
-    
-    with gr.Row():
-        url_input = gr.Textbox(
-            label="Website URL", 
-            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
-            info="The URL will be automatically prefixed with https:// if no protocol is specified."
-        )
-    
-    with gr.Row():
-        with gr.Column():
-            depth_input = gr.Slider(
-                minimum=1, 
-                maximum=5, 
-                value=3, 
-                step=1, 
-                label="Maximum Crawl Depth",
-                info="Higher values will result in more thorough but slower crawling"
-            )
-        with gr.Column():
-            pages_input = gr.Slider(
-                minimum=10, 
-                maximum=100, 
-                value=50, 
-                step=10, 
-                label="Maximum Pages to Crawl",
-                info="Higher values will result in more comprehensive but slower results"
-            )
-    
-    generate_btn = gr.Button("Generate llms.txt", variant="primary")
-    
-    with gr.Row():
-        output = gr.Textbox(
-            label="Generated llms.txt Content",
-            lines=20,
-            max_lines=30,
-            show_copy_button=True,
-            container=True,
-            scale=2,
-            interactive=True
-        )
     
     status = gr.Textbox(label="Status")
     
@@ -2309,6 +212,5 @@ with gr.Blocks(theme=theme,     css="""
         outputs=[output, status]
     )
 
-# Launch the app
 if __name__ == "__main__":
     iface.launch()