modified app and scraper file

app.py

@@ -6,6 +6,7 @@ import os
 import time
 import sys
 import requests
+import asyncio
 
 # Import our custom classes
 from secure_scraper import SecureScraper
@@ -94,42 +95,13 @@ def main():
         st.subheader("Privacy Settings")
         use_proxy = st.checkbox("Use Proxy Rotation", value=False)
         use_user_agent = st.checkbox("Use User-Agent Rotation", value=True)
-    
-    test_mode = st.sidebar.checkbox("Enable Test Mode", value=False)
-
-    # If in test mode, show a simplified test interface
-    if test_mode:
-        st.header("🔍 Test Mode")
-        st.info("This mode lets you test basic web connectivity without the full pipeline")
         
-        test_url = st.text_input("Test URL", "https://www.example.com")
-        
-        if st.button("Test Connection"):
-            try:
-                with st.spinner("Testing connection..."):
-                    # First try with requests for basic connectivity
-                    basic_response = requests.get(test_url, timeout=10)
-                    st.success(f"Basic HTTP connection successful: Status {basic_response.status_code}")
-                    
-                    # Then try with our crawler
-                    st.info("Now testing with crawl4ai integration...")
-                    test_scraper = SecureScraper()
-                    result = test_scraper.scrape_url(test_url)
-                    
-                    if result['status'] == 'success':
-                        st.success(f"crawl4ai connection successful")
-                        st.write("Privacy settings used:")
-                        st.json(result['privacy'])
-                        
-                        with st.expander("Response Preview"):
-                            st.write(result['data']['title'])
-                            st.write(result['data']['text'][:1000] + "..." if len(result['data']['text']) > 1000 else result['data']['text'])
-                    else:
-                        st.error(f"crawl4ai connection failed: {result['message']}")
-                        
-            except Exception as e:
-                st.error(f"Connection failed: {str(e)}")
-                st.code(traceback.format_exc())
+        # Add AsyncWebCrawler specific settings
+        st.subheader("Crawler Settings")
+        max_connections = st.slider("Max Connections", min_value=1, max_value=20, value=10)
+        timeout_seconds = st.slider("Request Timeout (seconds)", min_value=5, max_value=60, value=30)
+        max_retries = st.slider("Max Retries", min_value=1, max_value=10, value=5)
+    
     
     # Input section
     st.header("Scraping Target")
@@ -173,9 +145,14 @@ def main():
                     "http://example-proxy2.com:8080"
                 ]
             
-            # Initialize the scraper
+            # Initialize the scraper with updated parameters
             scraper = SecureScraper(proxy_list=proxy_list)
             
+            # Update AsyncWebCrawler settings based on user input
+            scraper.crawler.max_connections = max_connections
+            scraper.crawler.timeout = timeout_seconds
+            scraper.crawler.random_user_agent = use_user_agent
+            
         error_placeholder = st.empty()
         
         # Perform scraping
@@ -231,6 +208,14 @@ def main():
                 logging.error(f"LLM processing error: {str(e)}")
                 logging.error(traceback.format_exc())
 
+# Create a utility for running async code in Streamlit
+def run_async_code(coro):
+    """Run an async coroutine in a Streamlit app."""
+    try:
+        loop = asyncio.new_event_loop()
+        return loop.run_until_complete(coro)
+    finally:
+        loop.close()
 
 if __name__ == "__main__":
     main()

secure_scraper.py

@@ -2,26 +2,29 @@ import random
 import logging
 import time
 import json
-import crawl4ai
+import asyncio
+from crawl4ai import AsyncWebCrawler
 
 class SecureScraper:
     def __init__(self, proxy_list=None):
-        # Initialize with crawl4ai - adjust parameters according to the actual API
+        # Initialize with AsyncWebCrawler from crawl4ai
         self.use_proxies = bool(proxy_list)
         self.proxy_list = proxy_list
         
-        # Initialize crawler based on actual crawl4ai structure
-        self.crawler = crawl4ai.Crawler() if hasattr(crawl4ai, 'Crawler') else None
-        
-        # If the direct Crawler class doesn't exist, try to use the package's main functionality
-        if self.crawler is None:
-            self.crawler = crawl4ai
+        # Initialize async crawler
+        self.crawler = AsyncWebCrawler(
+            max_connections=10,
+            timeout=30,
+            proxies=self.proxy_list if self.use_proxies and self.proxy_list else None,
+            follow_redirects=True,
+            random_user_agent=True  # Enable random user agent rotation
+        )
             
         logging.basicConfig(level=logging.INFO)
     
-    def scrape_url(self, url, css_selectors=None):
+    async def async_scrape_url(self, url, css_selectors=None):
         """
-        Scrape a URL with privacy protection measures
+        Asynchronously scrape a URL with privacy protection measures
         
         Args:
             url: URL to scrape
@@ -36,89 +39,49 @@ class SecureScraper:
                 proxy_status = "using proxy" if self.use_proxies else "without proxy"
                 logging.info(f"Scraping {url} (Attempt {current_retry+1}/{max_retries}) {proxy_status}")
                 
-                # Use the crawl4ai functionality to scrape the URL
-                # Adapt this based on the actual crawl4ai API
-                if hasattr(self.crawler, 'fetch'):
-                    page_data = self.crawler.fetch(url, proxy=self.proxy_list[0] if self.use_proxies and self.proxy_list else None)
-                elif hasattr(self.crawler, 'scrape'):
-                    page_data = self.crawler.scrape(url, proxy=self.proxy_list[0] if self.use_proxies and self.proxy_list else None)
-                elif hasattr(self.crawler, 'get'):
-                    page_data = self.crawler.get(url, proxy=self.proxy_list[0] if self.use_proxies and self.proxy_list else None)
-                else:
-                    # Fallback to crawl4ai's default call pattern
-                    page_data = self.crawler(url, proxy=self.proxy_list[0] if self.use_proxies and self.proxy_list else None)
-                
-                # Extract content based on the returned data structure
-                # This will need to be adjusted based on what crawl4ai actually returns
+                # Use AsyncWebCrawler to fetch the page
+                response = await self.crawler.arun(url)
                 
-                # Create a basic result structure
-                if not css_selectors:
-                    # Default extraction if no selectors provided
-                    if hasattr(page_data, 'title'):
-                        title = page_data.title
-                    elif hasattr(page_data, 'get_title'):
-                        title = page_data.get_title()
-                    else:
-                        title = "Title extraction not supported"
-                        
-                    if hasattr(page_data, 'text'):
-                        text = page_data.text[:10000]  # Limit text size
-                    elif hasattr(page_data, 'get_text'):
-                        text = page_data.get_text()[:10000]
-                    else:
-                        text = "Text extraction not supported"
-                        
-                    if hasattr(page_data, 'links'):
-                        links = page_data.links[:20]  # Limit links
-                    elif hasattr(page_data, 'get_links'):
-                        links = page_data.get_links()[:20]
+                # Process the response based on content type
+                if response.is_html:
+                    page_data = await response.parse_html()
+                    
+                    # Create a basic result structure
+                    if not css_selectors:
+                        # Default extraction if no selectors provided
+                        title = page_data.title or "Title extraction not supported"
+                        text = page_data.text[:10000] if hasattr(page_data, 'text') else "Text extraction not supported"
+                        links = page_data.links[:20] if hasattr(page_data, 'links') else []
+                            
+                        result = {
+                            'title': title,
+                            'text': text,
+                            'links': links
+                        }
                     else:
-                        links = []
-                        
-                    result = {
-                        'title': title,
-                        'text': text,
-                        'links': links
-                    }
-                else:
-                    # Extract requested elements using CSS selectors
-                    result = {}
-                    for key, selector in css_selectors.items():
-                        if hasattr(page_data, 'select'):
+                        # Extract requested elements using CSS selectors
+                        result = {}
+                        for key, selector in css_selectors.items():
                             elements = page_data.select(selector)
-                        elif hasattr(page_data, 'query'):
-                            elements = page_data.query(selector)
-                        else:
-                            result[key] = f"Selector functionality not supported: {selector}"
-                            continue
                             
-                        if elements:
-                            # If multiple elements match, create a list
-                            if isinstance(elements, list) and len(elements) > 1:
-                                if hasattr(elements[0], 'text'):
+                            if elements:
+                                # If multiple elements match, create a list
+                                if len(elements) > 1:
                                     result[key] = [elem.text for elem in elements]
-                                elif hasattr(elements[0], 'get_text'):
-                                    result[key] = [elem.get_text() for elem in elements]
                                 else:
-                                    result[key] = elements
-                            else:
-                                if hasattr(elements[0], 'text'):
                                     result[key] = elements[0].text
-                                elif hasattr(elements[0], 'get_text'):
-                                    result[key] = elements[0].get_text()
-                                else:
-                                    result[key] = str(elements[0])
-                        else:
-                            result[key] = f"No match for selector: {selector}"
+                            else:
+                                result[key] = f"No match for selector: {selector}"
+                else:
+                    # Handle non-HTML responses
+                    result = {
+                        'content_type': response.content_type,
+                        'content_length': len(response.content),
+                        'summary': 'Non-HTML content'
+                    }
                 
-                # Get user agent info if available
-                user_agent = "Unknown"
-                if hasattr(self.crawler, 'current_user_agent'):
-                    user_agent = self.crawler.current_user_agent
-                elif hasattr(self.crawler, 'user_agent'):
-                    user_agent = self.crawler.user_agent
-                elif hasattr(page_data, 'user_agent'):
-                    user_agent = page_data.user_agent
+                # Get user agent info
+                user_agent = self.crawler.current_user_agent or "Unknown"
                 
                 # Truncate for privacy
                 user_agent = user_agent[:30] + '...' if len(str(user_agent)) > 30 else user_agent
@@ -135,11 +98,23 @@ class SecureScraper:
             except Exception as e:
                 logging.error(f"Request failed: {str(e)}")
                 current_retry += 1
-                time.sleep(random.uniform(2, 5))  # Incremental backoff
+                await asyncio.sleep(random.uniform(2, 5))  # Async sleep for backoff
                 
                 # Try to rotate proxy if available
                 if self.use_proxies and self.proxy_list and len(self.proxy_list) > 1:
                     self.proxy_list = self.proxy_list[1:] + [self.proxy_list[0]]  # Rotate proxies
+                    # Update crawler's proxies
+                    await self.crawler.update_proxies(self.proxy_list)
         
         # If we've exhausted retries
-        return {'status': 'error', 'message': f"Failed after {max_retries} attempts"}
+        return {'status': 'error', 'message': f"Failed after {max_retries} attempts"}
+    
+    def scrape_url(self, url, css_selectors=None):
+        """
+        Synchronous wrapper for async_scrape_url
+        
+        Args:
+            url: URL to scrape
+            css_selectors: Dict of elements to extract
+        """
+        return asyncio.run(self.async_scrape_url(url, css_selectors))