Create scraperr.py

scraperr.py

@@ -0,0 +1,109 @@
+import asyncio
+import aiohttp
+import random
+from bs4 import BeautifulSoup
+from googlesearch import search
+from urllib.parse import urlparse
+
+async def fetch(url, session, retries=3):
+    for attempt in range(retries):
+        try:
+            async with session.get(url) as response:
+                return await response.text()
+        except aiohttp.ClientResponseError as e:
+            if e.status == 429:  # HTTP 429: Too Many Requests
+                delay = 2 ** attempt  # Exponential backoff
+                print(f"Too many requests. Retrying in {delay} seconds...")
+                await asyncio.sleep(delay)
+            else:
+                print(f"Error fetching URL: {e}")
+                return None
+    print("Maximum retries exceeded. Failed to fetch URL.")
+    return None
+
+
+
+async def scrape_google(topic, num_results=20) -> str:
+    # Initialize a set to store unique website URLs
+    unique_websites = set()
+    # Initialize an empty string to accumulate outlines
+    outlines_str = ""
+
+    # Asynchronous HTTP session
+    async with aiohttp.ClientSession() as session:
+        # Perform Google search
+        search_results = search(topic, num=num_results, stop=num_results)
+        # Limit search results to 15 if num_results exceeds 15
+        search_results = random.sample(list(search_results), min(num_results, 15))
+
+        # Iterate through search results
+        for url in search_results:
+            # Exit loop if desired number of websites are found
+            if len(unique_websites) >= num_results:
+                break
+            # Scrape outlines from URL and accumulate them
+            outlines_str += await scrape_url(url, session, unique_websites)
+
+    # Return accumulated outlines string
+    return outlines_str
+    
+async def scrape_url(url, session, unique_websites) -> str:
+   try:
+        # Extract main domain from the URL
+        domain = urlparse(url).netloc.split('www.')[-1].split('/')[0]
+
+        # Check if the domain has already been processed
+        if domain in unique_websites:
+            return ""
+
+        # Fetch HTML content asynchronously
+        html_content = await fetch(url, session)
+        # Parse HTML content with BeautifulSoup
+        soup = BeautifulSoup(html_content, 'html.parser')
+        # Extract outlines from parsed HTML
+        outlines = extract_outlines(soup)
+
+        # Skip URLs with less than three outlines
+        if len(outlines) < 3:
+            return ""
+
+        # Filter irrelevant outlines
+        outlines = filter_outlines(outlines)
+        outlines_str = ""
+
+        # If outlines exist, accumulate them with website info
+        if outlines:
+            website_name = urlparse(url).netloc  # Extract website name from URL
+            outlines_str += f"Website: {website_name}\n"
+            outlines_str += f"URL: {url}\n"
+            outlines_str += "Outlines:\n"
+            for outline in outlines:
+                outlines_str += ". "+ outline + "\n"
+            outlines_str += "----------------------------------------------------------------------------------------------\n"
+            # Add the domain to the set of unique websites
+            unique_websites.add(domain)
+
+        return outlines_str
+   except Exception as e:
+      # Handle exceptions and return empty string
+      print("Error processing URL:", url)
+      print(e)
+      return ""
+
+
+def is_irrelevant_outline(outline):
+    # Minimum length threshold for relevant outlines
+    min_length_threshold = 20
+    return len(outline) < min_length_threshold
+
+def filter_outlines(outlines):
+    filtered_outlines = [outline for outline in outlines if not is_irrelevant_outline(outline)]
+    filtered_outlines = list(set(filtered_outlines))
+    return filtered_outlines
+
+def extract_outlines(soup):
+    outlines = []
+    headings = soup.find_all(['h1', 'h2', 'h3', 'h4'])
+    for heading in headings:
+        outlines.append(heading.text.strip())
+    return outlines