Ok So My Code import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
from concurrent.futures import ThreadPoolExecutor
import sqlite3
import threading
import time
import random
import nltk
from collections import Counter, defaultdict
from sklearn.feature_extraction.text import TfidfVectorizer
import urllib.robotparser

# Load nltk data for TF-IDF
nltk.download('stopwords')
from nltk.corpus import stopwords

# Lock to prevent race conditions during DB write
db_lock = threading.Lock()

# Set up stop words for TF-IDF
stop_words = set(stopwords.words('english'))

# Custom User-Agent for polite crawling
USER_AGENT = 'MyWebCrawler/1.0 (+http://yourdomain.com/crawlerinfo)'

# Domain access times for throttling requests
domain_access_times = defaultdict(float)
DOMAIN_DELAY = 1  # Delay in seconds between requests to the same domain

# Function to parse and respect robots.txt
robots_parsers = {}
robots_lock = threading.Lock()

def can_fetch(url):
    parsed_url = urlparse(url)
    base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
    with robots_lock:
        rp = robots_parsers.get(base_url)
        if rp is None:
            rp = urllib.robotparser.RobotFileParser()
            rp.set_url(urljoin(base_url, '/robots.txt'))
            try:
                rp.read()
                robots_parsers[base_url] = rp
            except Exception as e:
                print(f"Failed to read robots.txt at {base_url}: {e}")
                # If robots.txt cannot be fetched, assume disallowed
                return False
    return rp.can_fetch(USER_AGENT, url)

# Function to crawl a single URL
def crawl(url):
    if not can_fetch(url):
        print(f"Disallowed by robots.txt: {url}")
        return [], ""

    # Throttle requests to respect domain's crawling policy
    parsed_url = urlparse(url)
    domain = parsed_url.netloc
    elapsed = time.time() - domain_access_times[domain]
    if elapsed < DOMAIN_DELAY:
        time.sleep(DOMAIN_DELAY - elapsed)

    headers = {'User-Agent': USER_AGENT}
    try:
        print(f"Crawling: {url}")
        response = requests.get(url, headers=headers, timeout=10)
        domain_access_times[domain] = time.time()
        if response.status_code != 200:
            print(f"Failed to retrieve {url}: Status code {response.status_code}")
            return [], ""

        soup = BeautifulSoup(response.content, 'html.parser')

        # Remove script and style elements
        for script_or_style in soup(["script", "style"]):
            script_or_style.decompose()

        text = soup.get_text(separator=' ')
        text = ' '.join([word for word in text.split() if word.lower() not in stop_words])

        # Get all links from the page and normalize them
        links = []
        for a in soup.find_all('a', href=True):
            link = urljoin(url, a.get('href'))
            norm_link = normalize_url(link)
            if norm_link:
                links.append(norm_link)
        return links, text

    except Exception as e:
        print(f"Error crawling {url}: {e}")
        return [], ""

# Normalize and canonicalize URLs
def normalize_url(url):
    parsed = urlparse(url)
    if parsed.scheme not in ('http', 'https'):
        return None
    # Remove fragment identifiers and query parameters (optional)
    normalized = parsed._replace(fragment='', query='').geturl()
    return normalized

# Store page content in SQLite database
def store_page(url, content, rank):
    with db_lock:
        conn = sqlite3.connect('search_engine.db')
        cursor = conn.cursor()
        cursor.execute("INSERT OR IGNORE INTO pages (url, content, rank) VALUES (?, ?, ?)", (url, content, rank))
        conn.commit()
        conn.close()

# Function to handle crawling and storing pages with multithreading
def crawl_and_store(url, visited):
    # To prevent duplicates in multithreaded environment
    with db_lock:
        if url in visited:
            return []
        visited.add(url)

    links, content = crawl(url)
    if content:  # Only store the page if there's content
        rank = compute_rank(content)  # Compute the rank based on content
        store_page(url, content, rank)
    return links

# Compute a rank score for a page based on TF-IDF of the content
def compute_rank(content):
    vectorizer = TfidfVectorizer()
    vectors = vectorizer.fit_transform([content])
    rank = sum(vectors.data)  # Simplified ranking as the sum of TF-IDF values
    return rank

# Initialize database if not already set up
def create_index():
    conn = sqlite3.connect('search_engine.db')
    cursor = conn.cursor()
    cursor.execute('''CREATE TABLE IF NOT EXISTS pages (url TEXT PRIMARY KEY, content TEXT, rank REAL)''')
    conn.commit()
    conn.close()

# Crawl random websites to diversify crawling
def get_random_url():
    random_urls = [
        "https://www.example.com",
        "https://www.wikipedia.org",
        "https://www.python.org",
        "https://www.reddit.com",
        "https://www.bbc.com",
        "https://www.cnn.com"
    ]
    return random.choice(random_urls)

# Main function to start crawling with 50 workers
def start_crawling(seed_urls, max_depth=2):
    create_index()
    visited = set()  # Track visited URLs
    urls_to_crawl = set(seed_urls)

    with ThreadPoolExecutor(max_workers=50) as executor:
        for depth in range(max_depth):
            print(f"Depth {depth+1}/{max_depth}")
            futures = []
            current_level_urls = urls_to_crawl.copy()
            urls_to_crawl.clear()

            for url in current_level_urls:
                if url not in visited:
                    futures.append(executor.submit(crawl_and_store, url, visited))

            # Collect results (links) from crawled pages
            for future in futures:
                links = future.result()
                # Limit the number of new URLs to prevent explosion
                new_links = set(links) - visited
                urls_to_crawl.update(random.sample(new_links, min(len(new_links), 100)))  # Limit to 100 new URLs per page

            # Randomly get new websites to crawl and add them to the list
            random_url = get_random_url()
            if random_url not in visited:
                urls_to_crawl.add(random_url)

if __name__ == "__main__":
    seed_urls = [
        "https://www.wikipedia.org/",
        "https://commons.wikimedia.org/",
        "https://www.britannica.com/",
        "https://www.bbc.com/news",
        "https://www.cnn.com/",
        "https://www.nytimes.com/",
        "https://www.reuters.com/",
        "https://www.theguardian.com/",
        "https://ocw.mit.edu/",
        "https://www.stanford.edu/",
        "https://www.harvard.edu/",
        "https://www.usa.gov/",
        "https://www.un.org/",
        "https://europa.eu/",
        "https://github.com/",
        "https://stackoverflow.com/",
        "https://developer.mozilla.org/",
        "https://arxiv.org/",
        "https://pubmed.ncbi.nlm.nih.gov/",
        "https://www.nasa.gov/",
        "https://curlie.org/",
        "http://vlib.org/",
        "https://dmoztools.net/",
        "https://www.reddit.com/",
        "https://www.quora.com/",
        "https://www.linkedin.com/",
        "https://medium.com/",
        "https://www.data.gov/",
        "https://data.europa.eu/euodp/en/home",
        "https://www.gutenberg.org/",
        "https://www.amazon.com/",
        "https://www.ebay.com/",
        "https://www.youtube.com/",
        "https://www.imdb.com/",
        "https://soundcloud.com/",
        "https://www.tripadvisor.com/",
        "https://www.lonelyplanet.com/",
        "https://www.nationalgeographic.com/",
        "https://www.openstreetmap.org/",
        "https://doaj.org/",
        "https://www.scirp.org/",
        "https://artsandculture.google.com/",
        "https://www.metmuseum.org/",
        "https://www.britishmuseum.org/",
        "https://www.indeed.com/",
        "https://www.glassdoor.com/",
        "https://www.webmd.com/",
        "https://www.who.int/",
        "https://www.bloomberg.com/",
        "https://data.worldbank.org/",
        "https://www.ted.com/",
        "https://www.khanacademy.org/",
        "https://www.coursera.org/",
        "https://www.scribd.com/",
        "https://www.slideshare.net/",
        "https://techcrunch.com/",
        "https://mashable.com/",
        "https://www.theverge.com/",
        "https://stackexchange.com/",
        "https://disqus.com/",
        "https://wikimediafoundation.org/",
        "https://www.amnesty.org/",
        "https://www.habitat.org/",
        "https://archive.org/",
        "https://web.archive.org/"
    ]
    start_crawling(seed_urls, max_depth=3)

Is Getting This Error Traceback (most recent call last):
  File "C:\Users\tsuri\OneDrive\Pictures\search engine\crawler.py", line 252, in <module>
    start_crawling(seed_urls, max_depth=3)
  File "C:\Users\tsuri\OneDrive\Pictures\search engine\crawler.py", line 177, in start_crawling
    urls_to_crawl.update(random.sample(new_links, min(len(new_links), 100)))  # Limit to 100 new URLs per page
                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.12_3.12.2032.0_x64__qbz5n2kfra8p0\Lib\random.py", line 413, in sample
    raise TypeError("Population must be a sequence.  "
TypeError: Population must be a sequence.  For dicts or sets, use sorted(d).

Also Make The Code WAY FASTER USE MULTITHREADING ITS WAY TO SLOW