Update app.py

app.py

@@ -9,23 +9,36 @@ import os
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-def explode_link_df(crawl_df, col_group):
-    """Process links from a specific column group in the crawl dataframe"""
+def safe_crawl(url, output_file):
+    """Safely perform web crawl with timeout"""
     try:
-        link = crawl_df[f'{col_group}_links_url'].str.split('@@').explode()
-        text = crawl_df[f'{col_group}_links_text'].str.split('@@').explode()
-        all_links = []
+        adv.crawl(url, output_file, 
+                 follow_links=False,  # Only crawl the main page
+                 custom_settings={'CLOSESPIDER_TIMEOUT': 30})  # 30 second timeout
+        return True
+    except Exception as e:
+        logger.error(f"Crawl error: {str(e)}")
+        return False
+
+def process_links(df, link_type=None):
+    """Process links based on type"""
+    try:
+        if link_type:
+            mask = df['source'].str.contains(f'<{link_type}', case=False, na=False)
+            df = df[mask]
         
-        for link, text in zip(link.dropna(), text.dropna()):
-            if text and text.strip():
-                text = re.sub(r"\n+", " ", text.strip(), re.DOTALL)
-                text = re.sub(r"\s{3,}", " ", text)
-                all_links.append("\n".join(['## ' + text, f"[{text}]({link})"]))
+        all_links = []
+        for _, row in df.iterrows():
+            if row['text'] and str(row['text']).strip():
+                text = str(row['text']).strip()
+                text = re.sub(r'\s+', ' ', text)
+                link = str(row['link']).strip()
+                all_links.append(f"## {text}\n[{text}]({link})")
         
-        return "\n\n".join(all_links)
+        return all_links
     except Exception as e:
-        logger.error(f"Error processing {col_group} links: {str(e)}")
-        return ""
+        logger.error(f"Link processing error: {str(e)}")
+        return []
 
 def process_url(url, link_types):
     """Process URL and generate llms.txt content"""
@@ -33,131 +46,87 @@ def process_url(url, link_types):
         return "", "Please enter a URL"
     
     try:
-        if not url.startswith(("http://", "https://")):
-            url = "https://" + url
-
-        # Generate unique filename for this crawl
-        output_file = token_hex(6)
-        jsonl_path = f"{output_file}.jsonl"
-
+        # Ensure URL has protocol
+        if not url.startswith(('http://', 'https://')):
+            url = 'https://' + url
+        
+        # Create temporary file
+        output_file = f"{token_hex(4)}.jsonl"
+        
         try:
-            # Perform the crawl using advertools
-            adv.crawl(url, jsonl_path)
+            # Perform crawl
+            if not safe_crawl(url, output_file):
+                return "", "Crawl failed or timed out"
             
-            # Read the crawl results
-            crawl_df = pd.read_json(jsonl_path, lines=True)
+            # Read results
+            df = pd.read_json(output_file, lines=True)
             
-            # Extract title and meta description
-            title = crawl_df['title'].values[0]
-            meta_desc = crawl_df['meta_desc'].values[0]
+            # Get basic info
+            title = df['title'].iloc[0] if not pd.isna(df['title'].iloc[0]) else "Untitled"
+            meta_desc = df['meta_desc'].iloc[0] if not pd.isna(df['meta_desc'].iloc[0]) else ""
             
+            # Process links
+            link_df = adv.crawlytics.links(df)
             all_links = []
             
-            # Process links based on selected types
             if link_types and "All links" not in link_types:
                 for link_type in link_types:
-                    type_match = re.findall(r"header|footer|nav", link_type)
-                    if type_match:
-                        link_content = explode_link_df(crawl_df, type_match[0])
-                        if link_content:
-                            all_links.append(link_content)
-                            all_links.append('\n\n')
+                    type_name = re.search(r'<(\w+)>', link_type)
+                    if type_name:
+                        links = process_links(link_df, type_name.group(1))
+                        all_links.extend(links)
             else:
-                # Process all links using advertools
-                link_df = adv.crawlytics.links(crawl_df)
-                for link, text in link_df[['link', 'text']].values:
-                    if text and text.strip():
-                        text = re.sub(r"\n+", " ", text.strip(), re.DOTALL)
-                        text = re.sub(r"\s{3,}", " ", text)
-                        all_links.append("\n".join(['## ' + text, f"[{text}]({link})"]))
-
-            # Generate final content with proper spacing
-            links_text = "\n\n".join(all_links)
-            final_content = f"# {title}\n\n> {meta_desc}\n\n{links_text}"
-
+                all_links = process_links(link_df)
+            
+            # Create content
+            content_parts = [
+                f"# {title}",
+                f"> {meta_desc}",
+                "\n\n".join(all_links)
+            ]
+            final_content = "\n\n".join(content_parts)
+            
+            return final_content, f"Found {len(all_links)} links"
+            
         finally:
-            # Cleanup temporary file
-            if os.path.exists(jsonl_path):
-                os.remove(jsonl_path)
-
-        return final_content, f"Successfully crawled website. Found {len(all_links)} sections."
-    
+            # Cleanup
+            if os.path.exists(output_file):
+                os.remove(output_file)
+                
     except Exception as e:
-        logger.error(f"Error processing URL {url}: {str(e)}")
+        logger.error(f"Error processing {url}: {str(e)}")
         return "", f"Error: {str(e)}"
 
-# Custom CSS for Open Sans font and color theme
-css = """
-@import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@300;400;600;700&display=swap');
-
-body {
-    font-family: 'Open Sans', sans-serif !important;
-}
-
-.primary-btn {
-    background-color: #3452db !important;
-}
-
-.primary-btn:hover {
-    background-color: #2a41af !important;
-}
-"""
-
-# Create custom theme with specific color
-theme = gr.themes.Soft(
-    primary_hue=gr.themes.colors.Color(
-        name="blue",
-        c50="#eef1ff",
-        c100="#e0e5ff",
-        c200="#c3cbff",
-        c300="#a5b2ff",
-        c400="#8798ff",
-        c500="#6a7eff",
-        c600="#3452db",  # Main color
-        c700="#2a41af",
-        c800="#1f3183",
-        c900="#152156",
-        c950="#0a102b",
-    )
-)
-
-with gr.Blocks(theme=theme, css=css) as iface:
-    with gr.Row():
-        gr.Markdown("# Generate an `llms.txt` file")
-    
-    with gr.Row():
-        url_input = gr.Textbox(
-            label="Enter the home page of a website:",
-            placeholder="example: https://example.com",
-            lines=1,
-        )
-
-    with gr.Row():
-        link_types = gr.Dropdown(
-            label="Select types of links to extract (leave empty to get all links)",
+# Create interface
+iface = gr.Interface(
+    fn=process_url,
+    inputs=[
+        gr.Textbox(
+            label="Enter website URL",
+            placeholder="example: example.com"
+        ),
+        gr.Dropdown(
             choices=["<header> links", "<nav> links", "<footer> links", "All links"],
+            label="Select link types",
             multiselect=True,
             value=["All links"]
         )
-
-    with gr.Row():
-        generate_btn = gr.Button("Submit", variant="primary", elem_classes=["primary-btn"])
-
-    with gr.Row():
-        output = gr.Textbox(
-            label="Generated llms.txt Content",
+    ],
+    outputs=[
+        gr.Textbox(
+            label="Generated llms.txt",
             lines=20,
-            show_copy_button=True,
-            container=True,
+            show_copy_button=True
+        ),
+        gr.Textbox(
+            label="Status"
         )
-        status = gr.Textbox(label="Status", interactive=False)
-    
-    # Set up the click event
-    generate_btn.click(
-        fn=process_url,
-        inputs=[url_input, link_types],
-        outputs=[output, status],
-    )
+    ],
+    title="LLMs.txt Generator",
+    description="Generate an llms.txt file from a website",
+    theme=gr.themes.Soft(),
+    allow_flagging="never"
+)
 
 if __name__ == "__main__":
     iface.launch()