Can You Change This Code Where When You Start It Up It Asks How Many Instances You Want To Start So I Dont Have To Open A Bunch Of Terminals import asyncio
import aiohttp
from aiohttp import ClientSession, TCPConnector
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import sqlite3
import time
import random
import nltk
from collections import defaultdict
from sklearn.feature_extraction.text import TfidfVectorizer
import urllib.robotparser
import threading
import string

# Ensure nltk data is downloaded only once
try:
    nltk.data.find('corpora/stopwords')
except LookupError:
    nltk.download('stopwords')
from nltk.corpus import stopwords

# Set up stop words for TF-IDF
stop_words = set(stopwords.words('english'))

# Initialize the TfidfVectorizer once to save time
vectorizer = TfidfVectorizer()

# Custom User-Agent for polite crawling
USER_AGENT = 'MyWebCrawler/1.0 (+http://yourdomain.com/crawlerinfo)'

# Domain access times for throttling requests
domain_access_times = defaultdict(float)
DOMAIN_DELAY = 1  # Delay in seconds between requests to the same domain

# Robots.txt parsers cache
robots_parsers = {}
robots_lock = asyncio.Lock()

# Initialize database connection
conn = sqlite3.connect('search_engine.db', check_same_thread=False)
cursor = conn.cursor()
db_lock = threading.Lock()

# Normalize and canonicalize URLs
def normalize_url(url):
    parsed = urlparse(url)
    if parsed.scheme not in ('http', 'https'):
        return None
    # Remove fragment identifiers and query parameters (optional)
    normalized = parsed._replace(fragment='', query='').geturl()
    return normalized

# Function to parse and respect robots.txt
async def can_fetch(session, url):
    parsed_url = urlparse(url)
    base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"

    async with robots_lock:
        rp = robots_parsers.get(base_url)
        if rp is None:
            rp = urllib.robotparser.RobotFileParser()
            robots_url = urljoin(base_url, '/robots.txt')
            try:
                async with session.get(robots_url, timeout=10) as response:
                    if response.status == 200:
                        robots_content = await response.text()
                        rp.parse(robots_content.splitlines())
                    else:
                        # If robots.txt cannot be fetched, assume allowed
                        rp = None
                robots_parsers[base_url] = rp
            except Exception as e:
                print(f"Failed to read robots.txt at {base_url}: {e}")
                # If robots.txt cannot be fetched, assume allowed
                rp = None
                robots_parsers[base_url] = rp
        else:
            rp = robots_parsers[base_url]

    if rp is not None:
        return rp.can_fetch(USER_AGENT, url)
    else:
        # If no robots.txt, assume allowed
        return True

# Compute a rank score for a page based on TF-IDF of the content
def compute_rank(content):
    vectors = vectorizer.fit_transform([content])
    rank = sum(vectors.data)  # Simplified ranking as the sum of TF-IDF values
    return rank

# Store page content in SQLite database
def store_page(url, content, rank):
    with db_lock:
        cursor.execute("INSERT OR IGNORE INTO pages (url, content, rank) VALUES (?, ?, ?)", (url, content, rank))
        conn.commit()

# Crawl a single URL asynchronously
async def crawl(session, url, semaphore):
    async with semaphore:
        if not await can_fetch(session, url):
            print(f"Disallowed by robots.txt: {url}")
            return [], ""

        # Throttle requests to respect domain's crawling policy
        parsed_url = urlparse(url)
        domain = parsed_url.netloc
        elapsed = time.time() - domain_access_times[domain]
        if elapsed < DOMAIN_DELAY:
            await asyncio.sleep(DOMAIN_DELAY - elapsed)

        headers = {'User-Agent': USER_AGENT}
        try:
            print(f"Crawling: {url}")
            async with session.get(url, headers=headers, timeout=10) as response:
                domain_access_times[domain] = time.time()
                if response.status != 200:
                    print(f"Failed to retrieve {url}: Status code {response.status}")
                    return [], ""

                content = await response.text()

            soup = BeautifulSoup(content, 'html.parser')

            # Remove script and style elements
            for script_or_style in soup(["script", "style"]):
                script_or_style.decompose()

            text = soup.get_text(separator=' ')
            text = ' '.join([word for word in text.split() if word.lower() not in stop_words])

            # Get all links from the page and normalize them
            links = []
            for a in soup.find_all('a', href=True):
                link = urljoin(url, a.get('href'))
                norm_link = normalize_url(link)
                if norm_link:
                    links.append(norm_link)
            return links, text

        except Exception as e:
            print(f"Error crawling {url}: {e}")
            return [], ""

# Function to handle crawling and storing pages
async def crawl_and_store(url, visited, session, semaphore):
    if url in visited:
        return []

    visited.add(url)
    links, content = await crawl(session, url, semaphore)
    if content:  # Only store the page if there's content
        rank = compute_rank(content)  # Compute the rank based on content
        store_page(url, content, rank)
    return links

# Initialize database if not already set up
def create_index():
    cursor.execute('''CREATE TABLE IF NOT EXISTS pages (url TEXT PRIMARY KEY, content TEXT, rank REAL)''')
    conn.commit()

# Generate a random URL (Note: This method generates random domain names, which may not resolve)
def get_random_url():
    tlds = ['com', 'org', 'net', 'info', 'io', 'co']
    name = ''.join(random.choices(string.ascii_lowercase, k=random.randint(5, 10)))
    tld = random.choice(tlds)
    url = f"http://{name}.{tld}/"
    return url

# Main asynchronous function to start crawling
async def start_crawling(seed_urls, max_depth=2):
    create_index()
    visited = set()  # Track visited URLs
    urls_to_crawl = set(seed_urls)

    # Set the number of workers (concurrent requests)
    num_workers = 12  # For AMD Ryzen 5 5600 6-Core Processor with 12 threads
    semaphore = asyncio.Semaphore(num_workers)

    # Increase the number of allowed connections
    connector = TCPConnector(limit_per_host=num_workers)

    async with aiohttp.ClientSession(headers={'User-Agent': USER_AGENT}, connector=connector) as session:
        for depth in range(max_depth):
            print(f"Depth {depth+1}/{max_depth}")
            tasks = []
            current_level_urls = urls_to_crawl.copy()
            urls_to_crawl.clear()

            for url in current_level_urls:
                if url not in visited:
                    tasks.append(crawl_and_store(url, visited, session, semaphore))

            # If no tasks are found and it's the first depth, seed with random URLs
            if not tasks and depth == 0:
                for _ in range(num_workers):
                    random_url = get_random_url()
                    tasks.append(crawl_and_store(random_url, visited, session, semaphore))

            # Gather results concurrently
            results = await asyncio.gather(*tasks)
            for links in results:
                new_links = set(links) - visited
                # Limit to 100 new URLs per page
                sampled_links = random.sample(list(new_links), min(len(new_links), 100))
                urls_to_crawl.update(sampled_links)

            # Randomly get new websites to crawl and add them to the list
            for _ in range(num_workers):
                random_url = get_random_url()
                if random_url not in visited:
                    urls_to_crawl.add(random_url)

if __name__ == "__main__":
    seed_urls = [
        "https://www.wikipedia.org/",
        "https://commons.wikimedia.org/",
        "https://www.britannica.com/",
        "https://www.bbc.com/news",
        "https://www.cnn.com/",
        "https://www.nytimes.com/",
        "https://www.reuters.com/",
        "https://www.theguardian.com/",
        "https://ocw.mit.edu/",
        "https://www.stanford.edu/",
        "https://www.harvard.edu/",
        "https://www.usa.gov/",
        "https://www.un.org/",
        "https://europa.eu/",
        "https://github.com/",
        "https://stackoverflow.com/",
        "https://developer.mozilla.org/",
        "https://arxiv.org/",
        "https://pubmed.ncbi.nlm.nih.gov/",
        "https://www.nasa.gov/",
        "https://curlie.org/",
        "http://vlib.org/",
        "https://dmoztools.net/",
        "https://www.reddit.com/",
        "https://www.quora.com/",
        "https://www.linkedin.com/",
        "https://medium.com/",
        "https://www.data.gov/",
        "https://data.europa.eu/euodp/en/home",
        "https://www.gutenberg.org/",
        "https://www.amazon.com/",
        "https://www.ebay.com/",
        "https://www.youtube.com/",
        "https://www.imdb.com/",
        "https://soundcloud.com/",
        "https://www.tripadvisor.com/",
        "https://www.lonelyplanet.com/",
        "https://www.nationalgeographic.com/",
        "https://www.openstreetmap.org/",
        "https://doaj.org/",
        "https://www.scirp.org/",
        "https://artsandculture.google.com/",
        "https://www.metmuseum.org/",
        "https://www.britishmuseum.org/",
        "https://www.indeed.com/",
        "https://www.glassdoor.com/",
        "https://www.webmd.com/",
        "https://www.who.int/",
        "https://www.bloomberg.com/",
        "https://data.worldbank.org/",
        "https://www.ted.com/",
        "https://www.khanacademy.org/",
        "https://www.coursera.org/",
        "https://www.scribd.com/",
        "https://www.slideshare.net/",
        "https://techcrunch.com/",
        "https://mashable.com/",
        "https://www.theverge.com/",
        "https://stackexchange.com/",
        "https://disqus.com/",
        "https://wikimediafoundation.org/",
        "https://www.amnesty.org/",
        "https://www.habitat.org/",
        "https://archive.org/",
        "https://web.archive.org/"
        "https://music.youtube.com/"
        "https://www.roblox.com/home"
        "https://monkeygames.pages.dev/"
        "https://sketchfab.com/feed"
        "https://www.temu.com/"
    ]
    asyncio.run(start_crawling(seed_urls, max_depth=3))

    # Close the database connection
    conn.close()