Update app.py

app.py

@@ -20,25 +20,23 @@ def safe_crawl(url, output_file):
         logger.error(f"Crawl error: {str(e)}")
         return False
 
-def process_links(df, link_type=None):
-    """Process links based on type"""
+def explode_link_df(crawl_df, col_group):
+    """Process links from a specific column group in the crawl dataframe"""
     try:
-        if link_type:
-            mask = df['source'].str.contains(f'<{link_type}', case=False, na=False)
-            df = df[mask]
-        
+        link = crawl_df[f'{col_group}_links_url'].str.split('@@').explode()
+        text = crawl_df[f'{col_group}_links_text'].str.split('@@').explode()
         all_links = []
-        for _, row in df.iterrows():
-            if row['text'] and str(row['text']).strip():
-                text = str(row['text']).strip()
-                text = re.sub(r'\s+', ' ', text)
-                link = str(row['link']).strip()
-                all_links.append(f"## {text}\n[{text}]({link})")
         
-        return all_links
+        for link, text in zip(link.dropna(), text.dropna()):
+            if text and text.strip():
+                text = re.sub(r"\n+", " ", text.strip(), re.DOTALL)
+                text = re.sub(r"\s{3,}", " ", text)
+                all_links.append("\n".join(['## ' + text, f"[{text}]({link})"]))
+        
+        return "\n\n".join(all_links)
     except Exception as e:
-        logger.error(f"Link processing error: {str(e)}")
-        return []
+        logger.error(f"Error processing {col_group} links: {str(e)}")
+        return ""
 
 def process_url(url, link_types):
     """Process URL and generate llms.txt content"""
@@ -46,87 +44,132 @@ def process_url(url, link_types):
         return "", "Please enter a URL"
     
     try:
-        # Ensure URL has protocol
-        if not url.startswith(('http://', 'https://')):
-            url = 'https://' + url
-        
-        # Create temporary file
-        output_file = f"{token_hex(4)}.jsonl"
-        
+        if not url.startswith(("http://", "https://")):
+            url = "https://" + url
+
+        # Generate unique filename for this crawl
+        output_file = token_hex(6)
+        jsonl_path = f"{output_file}.jsonl"
+
         try:
-            # Perform crawl
-            if not safe_crawl(url, output_file):
+            # Perform the crawl using advertools
+            if not safe_crawl(url, jsonl_path):
                 return "", "Crawl failed or timed out"
             
-            # Read results
-            df = pd.read_json(output_file, lines=True)
+            # Read the crawl results
+            crawl_df = pd.read_json(jsonl_path, lines=True)
             
-            # Get basic info
-            title = df['title'].iloc[0] if not pd.isna(df['title'].iloc[0]) else "Untitled"
-            meta_desc = df['meta_desc'].iloc[0] if not pd.isna(df['meta_desc'].iloc[0]) else ""
+            # Extract title and meta description
+            title = crawl_df['title'].iloc[0] if not pd.isna(crawl_df['title'].iloc[0]) else "Untitled"
+            meta_desc = crawl_df['meta_desc'].iloc[0] if not pd.isna(crawl_df['meta_desc'].iloc[0]) else ""
             
-            # Process links
-            link_df = adv.crawlytics.links(df)
             all_links = []
             
+            # Process links based on selected types
             if link_types and "All links" not in link_types:
                 for link_type in link_types:
-                    type_name = re.search(r'<(\w+)>', link_type)
-                    if type_name:
-                        links = process_links(link_df, type_name.group(1))
-                        all_links.extend(links)
+                    type_match = re.findall(r"header|footer|nav", link_type)
+                    if type_match:
+                        link_content = explode_link_df(crawl_df, type_match[0])
+                        if link_content:
+                            all_links.append(link_content)
+                            all_links.append('\n\n')
             else:
-                all_links = process_links(link_df)
-            
-            # Create content
-            content_parts = [
-                f"# {title}",
-                f"> {meta_desc}",
-                "\n\n".join(all_links)
-            ]
-            final_content = "\n\n".join(content_parts)
-            
-            return final_content, f"Found {len(all_links)} links"
-            
+                # Process all links using advertools
+                link_df = adv.crawlytics.links(crawl_df)
+                for link, text in link_df[['link', 'text']].values:
+                    if text and text.strip():
+                        text = re.sub(r"\n+", " ", text.strip(), re.DOTALL)
+                        text = re.sub(r"\s{3,}", " ", text)
+                        all_links.append("\n".join(['## ' + text, f"[{text}]({link})"]))
+
+            # Generate final content
+            links_text = "\n\n".join(all_links)
+            final_content = f"# {title}\n\n> {meta_desc}\n\n{links_text}"
+
         finally:
-            # Cleanup
-            if os.path.exists(output_file):
-                os.remove(output_file)
-                
+            # Cleanup temporary file
+            if os.path.exists(jsonl_path):
+                os.remove(jsonl_path)
+
+        return final_content, f"Successfully crawled website. Found {len(all_links)} sections."
+    
     except Exception as e:
-        logger.error(f"Error processing {url}: {str(e)}")
+        logger.error(f"Error processing URL {url}: {str(e)}")
         return "", f"Error: {str(e)}"
 
-# Create interface
-iface = gr.Interface(
-    fn=process_url,
-    inputs=[
-        gr.Textbox(
-            label="Enter website URL",
-            placeholder="example: example.com"
-        ),
-        gr.Dropdown(
+# Custom CSS for Open Sans font and color theme
+css = """
+@import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@300;400;600;700&display=swap');
+
+body {
+    font-family: 'Open Sans', sans-serif !important;
+}
+
+.primary-btn {
+    background-color: #3452db !important;
+}
+
+.primary-btn:hover {
+    background-color: #2a41af !important;
+}
+"""
+
+# Create custom theme with specific color
+theme = gr.themes.Soft(
+    primary_hue=gr.themes.colors.Color(
+        name="blue",
+        c50="#eef1ff",
+        c100="#e0e5ff",
+        c200="#c3cbff",
+        c300="#a5b2ff",
+        c400="#8798ff",
+        c500="#6a7eff",
+        c600="#3452db",  # Main color
+        c700="#2a41af",
+        c800="#1f3183",
+        c900="#152156",
+        c950="#0a102b",
+    )
+)
+
+with gr.Blocks(theme=theme, css=css) as iface:
+    with gr.Row():
+        gr.Markdown("# Generate an `llms.txt` file")
+    
+    with gr.Row():
+        url_input = gr.Textbox(
+            label="Enter the home page of a website:",
+            placeholder="example: https://example.com",
+            lines=1,
+        )
+
+    with gr.Row():
+        link_types = gr.Dropdown(
+            label="Select types of links to extract (leave empty to get all links)",
             choices=["<header> links", "<nav> links", "<footer> links", "All links"],
-            label="Select link types",
             multiselect=True,
             value=["All links"]
         )
-    ],
-    outputs=[
-        gr.Textbox(
-            label="Generated llms.txt",
+
+    with gr.Row():
+        generate_btn = gr.Button("Submit", variant="primary", elem_classes=["primary-btn"])
+
+    with gr.Row():
+        output = gr.Textbox(
+            label="Generated llms.txt Content",
             lines=20,
-            show_copy_button=True
-        ),
-        gr.Textbox(
-            label="Status"
+            show_copy_button=True,
+            container=True,
         )
-    ],
-    title="LLMs.txt Generator",
-    description="Generate an llms.txt file from a website",
-    theme=gr.themes.Soft(),
-    allow_flagging="never"
-)
+        status = gr.Textbox(label="Status", interactive=False)
+    
+    # Set up the click event
+    generate_btn.click(
+        fn=process_url,
+        inputs=[url_input, link_types],
+        outputs=[output, status],
+    )
 
 if __name__ == "__main__":
     iface.launch()