if the url is not text readable add links but not text: "import requests
from bs4 import BeautifulSoup
import tldextract
import time
import random
import sys
from urllib.parse import urljoin, urlparse
from colorama import Fore, Style, init
from difflib import SequenceMatcher
import requests_cache
import textstat



# Initialize colorama for colored output
init(autoreset=True)

requests_cache.install_cache('web_cache', expire_after=180)  # Cache por 3 minutos


class WebNavigatorGame:
    def __init__(self, start_url, duration_seconds=60):
        self.start_url = self.normalize_url(start_url)
        self.domain = self.get_domain(self.start_url)
        self.duration = duration_seconds
        self.visited = set()
        self.to_visit = []
        self.texts = []
        # Cache to store extracted data: {url: {'text': ..., 'links': set(...)}}
        self.cache = {}
        # Define headers to simulate a human browser
        self.headers = {
            "User-Agent": (
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                "AppleWebKit/537.36 (KHTML, like Gecko) "
                "Chrome/112.0.0.0 Safari/537.36"
            ),
            "Accept-Language": "en-US,en;q=0.9",
            "Accept-Encoding": "gzip, deflate",
            "Connection": "keep-alive",
        }
        # Store base URL path segments
        self.base_url = self.start_url
        self.base_path_segments = self.get_path_segments(self.base_url)

    def normalize_url(self, url):
        """Ensure the URL has a scheme."""
        parsed = urlparse(url)
        if not parsed.scheme:
            return "http://" + url
        return url

    def get_domain(self, url):
        """Extract the domain using tldextract."""
        extracted = tldextract.extract(url)
        domain = f"{extracted.domain}.{extracted.suffix}"
        return domain.lower()

    def get_page(self, url):
        """Fetch the content of a web page with headers."""
        for attempt in range(3):  # Intentar hasta 3 veces
            try:
                response = requests.get(url, headers=self.headers, timeout=10)
                response.raise_for_status()
                return response.text
            except requests.exceptions.Timeout:
                print(Fore.RED + f"Timeout occurred for {url}. Retrying...")
            except requests.RequestException as e:
                print(Fore.RED + f"Failed to retrieve {url}: {e}")
                break  # Salir si no es un error temporal
        return None

    def extract_text(self, html_content):
        """Extract text from HTML content."""
        soup = BeautifulSoup(html_content, 'lxml')
        for script_or_style in soup(['script', 'style']):
            script_or_style.decompose()
        text = soup.get_text(separator=' ', strip=True)
        return text

    def get_path_segments(self, url):
        """Return the list of path segments for a URL."""
        parsed = urlparse(url)
        segments = [segment for segment in parsed.path.split('/') if segment]
        return segments

    def is_valid_url(self, url):
        """Check if the URL satisfies the constraints."""
        url_segments = self.get_path_segments(url)
        base_segments = self.base_path_segments
        if len(url_segments) < len(base_segments):
            return False
        for base_seg, url_seg in zip(base_segments, url_segments):
            if base_seg != url_seg:
                return False
        return True

    def extract_links(self, html_content, base_url):
        """Extract all unique, valid links from HTML content."""
        soup = BeautifulSoup(html_content, 'html.parser')
        links = set()
        for link in soup.find_all('a', href=True):
            href = link['href']
            full_url = urljoin(base_url, href)
            normalized_url = self.normalize_url(full_url)
            if self.get_domain(normalized_url) == self.domain:
                parsed = urlparse(normalized_url)
                clean_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
                if self.is_valid_url(clean_url):
                    links.add(clean_url)
        return links

    def get_extracted_data(self, url):
        """Retrieve extracted text and links from the cache or extract them if not cached."""
        if url in self.cache:
            return self.cache[url]['text'], self.cache[url]['links'], False
        else:
            html_content = self.get_page(url)
            if html_content is None:
                return None, set(), True
            text = self.extract_text(html_content)
            links = self.extract_links(html_content, url)
            self.cache[url] = {'text': text, 'links': links}
            return text, links, True

    def is_text_unique(self, new_text):
        """Check if the new text is more than 50% different from existing texts."""
        for old_text in self.texts:
            similarity = SequenceMatcher(None, new_text, old_text).ratio()
            if similarity > 0.5:
                return False
        return True  # It's unique enough

    def is_text_readable(self, text, threshold=60):
        """Check if the text is readable based on the Flesch Reading Ease score."""
        if not text:
            return False
        score = textstat.flesch_reading_ease(text)
        print(f"Flesch Reading Ease score: {score}") 
        return score >= threshold

    def run(self):
        """Run the web navigation game."""
        start_time = time.time()
        current_url = self.start_url
        self.to_visit.append(current_url)

        while time.time() - start_time < self.duration and self.to_visit:
            current_url = self.to_visit.pop(0)
            if current_url != self.start_url and current_url in self.visited:
                continue
            print(Fore.CYAN + f"\nVisiting: {current_url}")

            text, links, request_made = self.get_extracted_data(current_url)
            if text is None:
                continue

            if not self.is_text_readable(text) and current_url != self.start_url:
                print(Fore.YELLOW + "Text is not readable. Skipping this URL.")
                continue  # Skip to the next URL

            if request_made:
                if self.is_text_unique(text):
                    self.visited.add(current_url)
                    self.texts.append(text)
                    print(Fore.GREEN + f"Extracted Text (first 200 chars):\n{text[:200]}...\n")
                else:
                    print(Fore.YELLOW + "Text is not unique enough. Skipping this URL.")
                    continue  # Skip to the next URL
            else:
                print(Fore.BLUE + f"Revisited {current_url}. Using cached data.\n")

            new_links = links - self.visited
            new_links.add(self.start_url)
            filtered_links = set()
            for link in new_links:
                if link == self.start_url or link not in self.visited:
                    filtered_links.add(link)
            if filtered_links:
                self.to_visit.extend(list(filtered_links))
                random.shuffle(self.to_visit)
            else:
                print(Fore.YELLOW + "No new links found on this page.")

            if request_made:
                print(Fore.MAGENTA + "Waiting for 5 seconds before the next request...\n")
                time.sleep(5)

        print(Fore.MAGENTA + f"\nGame Over! Visited {len(self.visited)} unique pages.")
        self.save_texts()

    def save_texts(self, filename="extracted_texts3.txt"):
        """Save all extracted texts to a file."""
        with open(filename, 'w', encoding='utf-8') as f:
            for idx, text in enumerate(self.texts, 1):
                f.write(f"--- Page {idx} ---\n")
                f.write(text + "\n\n")
        print(Fore.BLUE + f"All texts have been saved to {filename}.")

if __name__ == "__main__":
    if len(sys.argv) < 3:
        print("Usage: python web_navigator_game.py <start_url> <duration_seconds>")
        print("Example: python web_navigator_game.py https://example.com 120")
        sys.exit(1)

    start_url = sys.argv[1]
    try:
        duration = int(sys.argv[2])
    except ValueError:
        print("Duration must be an integer representing seconds.")
        sys.exit(1)

    game = WebNavigatorGame(start_url, duration)
    game.run()"