import asyncio
import aiohttp
from aiohttp import ClientSession, TCPConnector
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import sqlite3
import time
import threading
import os
import shutil
import nltk
from collections import defaultdict
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
import urllib.robotparser
from colorama import Fore, Style, init

# Initialize colorama for colored terminal text
init(autoreset=True)

# Ensure nltk data is downloaded only once
try:
    nltk.data.find('corpora/stopwords')
except LookupError:
    nltk.download('stopwords')

# Configurations
USER_AGENT = 'OptimizedCrawler/1.0 (+http://yourdomain.com/crawlerinfo)'
DB_FILE = 'search_engine.db'
BACKUP_FILE = 'search_engine_backup.db'
DOMAIN_DELAY = 1  # Seconds between requests to the same domain
BACKUP_INTERVAL = 300  # Backup every 5 minutes

# Initialize stopwords
stop_words = set(stopwords.words('english'))

# Backup SQLite database
def backup_database():
    if os.path.exists(DB_FILE):
        shutil.copyfile(DB_FILE, BACKUP_FILE)
        print(f"Database backed up to {BACKUP_FILE}")

# Normalize URLs
def normalize_url(url):
    parsed = urlparse(url)
    if parsed.scheme not in ('http', 'https'):
        return None
    normalized = parsed._replace(fragment='', query='').geturl()
    return normalized

# Parse robots.txt rules
async def can_fetch(session, url, robots_lock, robots_parsers):
    parsed_url = urlparse(url)
    base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"

    async with robots_lock:
        rp = robots_parsers.get(base_url)
        if rp is None:
            rp = urllib.robotparser.RobotFileParser()
            robots_url = urljoin(base_url, '/robots.txt')
            try:
                async with session.get(robots_url, timeout=10) as response:
                    if response.status == 200:
                        robots_content = await response.text()
                        rp.parse(robots_content.splitlines())
                    robots_parsers[base_url] = rp
            except Exception:
                robots_parsers[base_url] = None

    if rp is not None:
        return rp.can_fetch(USER_AGENT, url)
    return True

# Create SQLite database and table
def create_database():
    conn = sqlite3.connect(DB_FILE)
    cursor = conn.cursor()
    cursor.execute("""
        CREATE TABLE IF NOT EXISTS pages (
            url TEXT PRIMARY KEY,
            content TEXT,
            rank REAL
        )
    """)
    conn.commit()
    conn.close()

# Store page data in SQLite
def store_page(url, content, rank, db_lock):
    with db_lock:
        conn = sqlite3.connect(DB_FILE)
        cursor = conn.cursor()
        cursor.execute("INSERT OR IGNORE INTO pages (url, content, rank) VALUES (?, ?, ?)", (url, content, rank))
        conn.commit()
        conn.close()

# Compute TF-IDF rank
def compute_rank(content, vectorizer):
    vectors = vectorizer.fit_transform([content])
    return sum(vectors.data)

# Crawl a single URL
async def crawl(session, url, semaphore, domain_access_times, robots_lock, robots_parsers, db_lock, vectorizer):
    async with semaphore:
        if not await can_fetch(session, url, robots_lock, robots_parsers):
            print(f"Disallowed by robots.txt: {url}")
            return []

        parsed_url = urlparse(url)
        domain = parsed_url.netloc
        elapsed = time.time() - domain_access_times.get(domain, 0)
        if elapsed < DOMAIN_DELAY:
            await asyncio.sleep(DOMAIN_DELAY - elapsed)

        print(f"{Fore.GREEN}Crawling: {url}{Style.RESET_ALL}")  # Show URL in green text

        try:
            async with session.get(url, headers={'User-Agent': USER_AGENT}, timeout=10) as response:
                if response.status != 200:
                    return []

                content = await response.text()
                soup = BeautifulSoup(content, 'html.parser')

                for script_or_style in soup(["script", "style"]):
                    script_or_style.decompose()

                text = soup.get_text(separator=' ')
                text = ' '.join([word for word in text.split() if word.lower() not in stop_words])

                links = [normalize_url(urljoin(url, a['href'])) for a in soup.find_all('a', href=True)]
                links = [link for link in links if link]

                rank = compute_rank(text, vectorizer)
                store_page(url, text, rank, db_lock)
                return links

        except Exception as e:
            print(f"Error crawling {url}: {e}")
            return []

# Crawl URLs
async def crawl_urls(seed_urls):
    semaphore = asyncio.Semaphore(os.cpu_count() * 2)
    domain_access_times = defaultdict(float)
    robots_lock = asyncio.Lock()
    robots_parsers = {}
    db_lock = threading.Lock()
    vectorizer = TfidfVectorizer()

    to_crawl = set(seed_urls)
    visited = set()

    async with aiohttp.ClientSession(headers={'User-Agent': USER_AGENT}, connector=TCPConnector(limit_per_host=50)) as session:
        while to_crawl:
            url = to_crawl.pop()
            if url in visited:
                continue
            visited.add(url)
            new_links = await crawl(session, url, semaphore, domain_access_times, robots_lock, robots_parsers, db_lock, vectorizer)
            to_crawl.update(new_links)

# Periodic backup
def backup_periodically():
    while True:
        time.sleep(BACKUP_INTERVAL)
        backup_database()

# Main function
def main():
    create_database()

    seed_urls = [
        "https://www.wikipedia.org/",
        "https://commons.wikimedia.org/",
        "https://www.britannica.com/",
        "https://www.bbc.com/news",
        "https://www.cnn.com/",
        "https://www.nytimes.com/",
        "https://www.reuters.com/",
        "https://www.theguardian.com/",
        "https://ocw.mit.edu/",
        "https://www.stanford.edu/",
        "https://www.harvard.edu/",
        "https://www.usa.gov/",
        "https://www.un.org/",
        "https://europa.eu/",
        "https://github.com/",
        "https://stackoverflow.com/",
        "https://developer.mozilla.org/",
        "https://arxiv.org/",
        "https://pubmed.ncbi.nlm.nih.gov/",
        "https://www.nasa.gov/",
        "https://curlie.org/",
        "http://vlib.org/",
        "https://dmoztools.net/",
        "https://www.reddit.com/",
        "https://www.quora.com/",
        "https://www.linkedin.com/",
        "https://medium.com/",
        "https://www.data.gov/",
        "https://data.europa.eu/euodp/en/home",
        "https://www.gutenberg.org/",
        "https://www.amazon.com/",
        "https://www.ebay.com/",
        "https://www.youtube.com/",
        "https://www.imdb.com/",
        "https://soundcloud.com/",
        "https://www.tripadvisor.com/",
        "https://www.lonelyplanet.com/",
        "https://www.nationalgeographic.com/",
        "https://www.openstreetmap.org/",
        "https://doaj.org/",
        "https://www.scirp.org/",
        "https://artsandculture.google.com/",
        "https://www.metmuseum.org/",
        "https://www.britishmuseum.org/",
        "https://www.indeed.com/",
        "https://www.glassdoor.com/",
        "https://www.webmd.com/",
        "https://www.who.int/",
        "https://www.bloomberg.com/",
        "https://data.worldbank.org/",
        "https://www.ted.com/",
        "https://www.khanacademy.org/",
        "https://www.coursera.org/",
        "https://www.scribd.com/",
        "https://www.slideshare.net/",
        "https://techcrunch.com/",
        "https://mashable.com/",
        "https://www.theverge.com/",
        "https://stackexchange.com/",
        "https://disqus.com/",
        "https://wikimediafoundation.org/",
        "https://www.amnesty.org/",
        "https://www.habitat.org/",
        "https://archive.org/",
        "https://web.archive.org/",
        "https://music.youtube.com/",
        "https://www.roblox.com/home",
        "https://monkeygames.pages.dev/",
        "https://sketchfab.com/feed",
        "https://www.temu.com/"
    ]

    backup_thread = threading.Thread(target=backup_periodically, daemon=True)
    backup_thread.start()

    asyncio.run(crawl_urls(seed_urls))

if __name__ == "__main__":
    main()


Can You Optimise My Code To Be WAY FASTER LIKE SUPER DUPER FAST But not use up all my ram (: