import asyncio
import aiohttp
from aiohttp import ClientSession
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import sqlite3
import time
import random
import nltk
from collections import defaultdict
from sklearn.feature_extraction.text import TfidfVectorizer
import urllib.robotparser
import threading

# Load nltk data for TF-IDF
nltk.download('stopwords')
from nltk.corpus import stopwords

# Set up stop words for TF-IDF
stop_words = set(stopwords.words('english'))

# Custom User-Agent for polite crawling
USER_AGENT = 'MyWebCrawler/1.0 (+http://yourdomain.com/crawlerinfo)'

# Domain access times for throttling requests
domain_access_times = defaultdict(float)
DOMAIN_DELAY = 1  # Delay in seconds between requests to the same domain

# Robots.txt parsers cache
robots_parsers = {}
robots_lock = asyncio.Lock()

# Initialize database connection
conn = sqlite3.connect('search_engine.db', check_same_thread=False)
cursor = conn.cursor()
db_lock = threading.Lock()

# Normalize and canonicalize URLs
def normalize_url(url):
    parsed = urlparse(url)
    if parsed.scheme not in ('http', 'https'):
        return None
    # Remove fragment identifiers and query parameters (optional)
    normalized = parsed._replace(fragment='', query='').geturl()
    return normalized

# Function to parse and respect robots.txt
async def can_fetch(session, url):
    parsed_url = urlparse(url)
    base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"

    async with robots_lock:
        rp = robots_parsers.get(base_url)
        if rp is None:
            rp = urllib.robotparser.RobotFileParser()
            robots_url = urljoin(base_url, '/robots.txt')
            try:
                async with session.get(robots_url, timeout=10) as response:
                    if response.status == 200:
                        robots_content = await response.text()
                        rp.parse(robots_content.splitlines())
                    else:
                        # If robots.txt cannot be fetched, assume allowed
                        rp = None
                robots_parsers[base_url] = rp
            except Exception as e:
                print(f"Failed to read robots.txt at {base_url}: {e}")
                # If robots.txt cannot be fetched, assume allowed
                rp = None
                robots_parsers[base_url] = rp
        else:
            rp = robots_parsers[base_url]

    if rp is not None:
        return rp.can_fetch(USER_AGENT, url)
    else:
        # If no robots.txt, assume allowed
        return True

# Compute a rank score for a page based on TF-IDF of the content
def compute_rank(content):
    vectorizer = TfidfVectorizer()
    vectors = vectorizer.fit_transform([content])
    rank = sum(vectors.data)  # Simplified ranking as the sum of TF-IDF values
    return rank

# Store page content in SQLite database
def store_page(url, content, rank):
    with db_lock:
        cursor.execute("INSERT OR IGNORE INTO pages (url, content, rank) VALUES (?, ?, ?)", (url, content, rank))
        conn.commit()

# Crawl a single URL asynchronously
async def crawl(session, url):
    if not await can_fetch(session, url):
        print(f"Disallowed by robots.txt: {url}")
        return [], ""

    # Throttle requests to respect domain's crawling policy
    parsed_url = urlparse(url)
    domain = parsed_url.netloc
    elapsed = time.time() - domain_access_times[domain]
    if elapsed < DOMAIN_DELAY:
        await asyncio.sleep(DOMAIN_DELAY - elapsed)

    headers = {'User-Agent': USER_AGENT}
    try:
        print(f"Crawling: {url}")
        async with session.get(url, headers=headers, timeout=10) as response:
            domain_access_times[domain] = time.time()
            if response.status != 200:
                print(f"Failed to retrieve {url}: Status code {response.status}")
                return [], ""

            content = await response.text()

        soup = BeautifulSoup(content, 'html.parser')

        # Remove script and style elements
        for script_or_style in soup(["script", "style"]):
            script_or_style.decompose()

        text = soup.get_text(separator=' ')
        text = ' '.join([word for word in text.split() if word.lower() not in stop_words])

        # Get all links from the page and normalize them
        links = []
        for a in soup.find_all('a', href=True):
            link = urljoin(url, a.get('href'))
            norm_link = normalize_url(link)
            if norm_link:
                links.append(norm_link)
        return links, text

    except Exception as e:
        print(f"Error crawling {url}: {e}")
        return [], ""

# Function to handle crawling and storing pages
async def crawl_and_store(url, visited, session):
    if url in visited:
        return []

    visited.add(url)
    links, content = await crawl(session, url)
    if content:  # Only store the page if there's content
        rank = compute_rank(content)  # Compute the rank based on content
        store_page(url, content, rank)
    return links

# Initialize database if not already set up
def create_index():
    cursor.execute('''CREATE TABLE IF NOT EXISTS pages (url TEXT PRIMARY KEY, content TEXT, rank REAL)''')
    conn.commit()

# Crawl random websites to diversify crawling
def get_random_url():
    random_urls = [
        "https://www.example.com",
        "https://www.wikipedia.org",
        "https://www.python.org",
        "https://www.reddit.com",
        "https://www.bbc.com",
        "https://www.cnn.com"
    ]
    return random.choice(random_urls)

# Main asynchronous function to start crawling
async def start_crawling(seed_urls, max_depth=2):
    create_index()
    visited = set()  # Track visited URLs
    urls_to_crawl = set(seed_urls)

    async with aiohttp.ClientSession(headers={'User-Agent': USER_AGENT}) as session:
        for depth in range(max_depth):
            print(f"Depth {depth+1}/{max_depth}")
            tasks = []
            current_level_urls = urls_to_crawl.copy()
            urls_to_crawl.clear()

            for url in current_level_urls:
                if url not in visited:
                    tasks.append(crawl_and_store(url, visited, session))

            # Gather results concurrently
            results = await asyncio.gather(*tasks)
            for links in results:
                new_links = set(links) - visited
                # Limit to 100 new URLs per page
                sampled_links = random.sample(list(new_links), min(len(new_links), 100))
                urls_to_crawl.update(sampled_links)

            # Randomly get new websites to crawl and add them to the list
            random_url = get_random_url()
            if random_url not in visited:
                urls_to_crawl.add(random_url)

if __name__ == "__main__":
    seed_urls = [
        "https://www.wikipedia.org/",
        "https://commons.wikimedia.org/",
        "https://www.britannica.com/",
        "https://www.bbc.com/news",
        "https://www.cnn.com/",
        "https://www.nytimes.com/",
        "https://www.reuters.com/",
        "https://www.theguardian.com/",
        "https://ocw.mit.edu/",
        "https://www.stanford.edu/",
        "https://www.harvard.edu/",
        "https://www.usa.gov/",
        "https://www.un.org/",
        "https://europa.eu/",
        "https://github.com/",
        "https://stackoverflow.com/",
        "https://developer.mozilla.org/",
        "https://arxiv.org/",
        "https://pubmed.ncbi.nlm.nih.gov/",
        "https://www.nasa.gov/",
        "https://curlie.org/",
        "http://vlib.org/",
        "https://dmoztools.net/",
        "https://www.reddit.com/",
        "https://www.quora.com/",
        "https://www.linkedin.com/",
        "https://medium.com/",
        "https://www.data.gov/",
        "https://data.europa.eu/euodp/en/home",
        "https://www.gutenberg.org/",
        "https://www.amazon.com/",
        "https://www.ebay.com/",
        "https://www.youtube.com/",
        "https://www.imdb.com/",
        "https://soundcloud.com/",
        "https://www.tripadvisor.com/",
        "https://www.lonelyplanet.com/",
        "https://www.nationalgeographic.com/",
        "https://www.openstreetmap.org/",
        "https://doaj.org/",
        "https://www.scirp.org/",
        "https://artsandculture.google.com/",
        "https://www.metmuseum.org/",
        "https://www.britishmuseum.org/",
        "https://www.indeed.com/",
        "https://www.glassdoor.com/",
        "https://www.webmd.com/",
        "https://www.who.int/",
        "https://www.bloomberg.com/",
        "https://data.worldbank.org/",
        "https://www.ted.com/",
        "https://www.khanacademy.org/",
        "https://www.coursera.org/",
        "https://www.scribd.com/",
        "https://www.slideshare.net/",
        "https://techcrunch.com/",
        "https://mashable.com/",
        "https://www.theverge.com/",
        "https://stackexchange.com/",
        "https://disqus.com/",
        "https://wikimediafoundation.org/",
        "https://www.amnesty.org/",
        "https://www.habitat.org/",
        "https://archive.org/",
        "https://web.archive.org/"
    ]
    asyncio.run(start_crawling(seed_urls, max_depth=3))

    # Close the database connection
    conn.close()

Can You Make This Faster And Make It Use Random Urls + The Preset Urls And Make It Have As Many Workers As A  AMD Ryzen 5 5600 6-Core Processor