modifica lo siguiente: antes de aceptar el texto de una url hay que asegurarse de que al menos mas del 50% de su contenido es distinto a cualquier contenido de otras urls, por tanto para hacerlo guardaremos en una lista todo el contenido de las urls aceptadas y comprobaremos una a una que sea diferente en mas del 50%. Si no es diferente , vuelve a la pagina anteriror para ir a otro link aleatorio. Codigo: "import requests
from bs4 import BeautifulSoup
import tldextract
import time
import random
import sys
from urllib.parse import urljoin, urlparse
from colorama import Fore, Style, init

# Initialize colorama for colored output
init(autoreset=True)

class WebNavigatorGame:
    def __init__(self, start_url, duration_seconds=60):
        self.start_url = self.normalize_url(start_url)
        self.domain = self.get_domain(self.start_url)
        self.duration = duration_seconds
        self.visited = set()
        self.to_visit = []
        self.texts = []
        # Cache to store extracted data: {url: {'text': ..., 'links': set(...)}}
        self.cache = {}
        # Define headers to simulate a human browser
        self.headers = {
            "User-Agent": (
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                "AppleWebKit/537.36 (KHTML, like Gecko) "
                "Chrome/112.0.0.0 Safari/537.36"
            ),
            "Accept-Language": "en-US,en;q=0.9",
            "Accept-Encoding": "gzip, deflate",
            "Connection": "keep-alive",
        }
        # Store base URL path segments
        self.base_url = self.start_url
        self.base_path_segments = self.get_path_segments(self.base_url)

    def normalize_url(self, url):
        """Ensure the URL has a scheme."""
        parsed = urlparse(url)
        if not parsed.scheme:
            return "http://" + url
        return url

    def get_domain(self, url):
        """Extract the domain using tldextract."""
        extracted = tldextract.extract(url)
        domain = f"{extracted.domain}.{extracted.suffix}"
        return domain.lower()

    def get_page(self, url):
        """Fetch the content of a web page with headers."""
        try:
            response = requests.get(url, headers=self.headers, timeout=10)
            response.raise_for_status()
            return response.text
        except (requests.RequestException, requests.HTTPError) as e:
            print(Fore.RED + f"Failed to retrieve {url}: {e}")
            return None

    def extract_text(self, html_content):
        """Extract text from HTML content."""
        soup = BeautifulSoup(html_content, 'html.parser')
        # Remove script and style elements
        for script_or_style in soup(['script', 'style']):
            script_or_style.decompose()
        text = soup.get_text(separator=' ', strip=True)
        return text

    def get_path_segments(self, url):
        """Return the list of path segments for a URL."""
        parsed = urlparse(url)
        # Split the path and remove empty segments
        segments = [segment for segment in parsed.path.split('/') if segment]
        return segments

    def is_valid_url(self, url):
        """
        Check if the URL satisfies the constraints:
        - Contains at least the base URL in its path.
        - Does not have fewer segments than the base URL.
        - Does not modify any part of the base URL.
        """
        url_segments = self.get_path_segments(url)
        base_segments = self.base_path_segments
        # The URL must have at least as many segments as the base URL
        if len(url_segments) < len(base_segments):
            return False
        # The initial segments must match the base URL segments
        for base_seg, url_seg in zip(base_segments, url_segments):
            if base_seg != url_seg:
                return False
        return True

    def extract_links(self, html_content, base_url):
        """Extract all unique, valid links from HTML content."""
        soup = BeautifulSoup(html_content, 'html.parser')
        links = set()
        for link in soup.find_all('a', href=True):
            href = link['href']
            # Resolve relative URLs
            full_url = urljoin(base_url, href)
            normalized_url = self.normalize_url(full_url)
            if self.get_domain(normalized_url) == self.domain:
                # Remove fragment identifiers and query parameters for consistency
                parsed = urlparse(normalized_url)
                clean_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
                # Check if the URL meets the constraints
                if self.is_valid_url(clean_url):
                    links.add(clean_url)
        return links

    def get_extracted_data(self, url):
        """
        Retrieve extracted text and links from the cache or extract them if not cached.
        """
        if url in self.cache:
            # Data is cached; retrieve from cache
            return self.cache[url]['text'], self.cache[url]['links']
        else:
            # Data not cached; fetch and extract
            html_content = self.get_page(url)
            if html_content is None:
                return None, set()
            text = self.extract_text(html_content)
            links = self.extract_links(html_content, url)
            # Cache the extracted data
            self.cache[url] = {'text': text, 'links': links}
            return text, links

    def run(self):
        """Run the web navigation game."""
        start_time = time.time()
        current_url = self.start_url
        self.to_visit.append(current_url)

        while time.time() - start_time < self.duration and self.to_visit:
            current_url = self.to_visit.pop(0)
            # Allow revisiting the starter page
            if current_url != self.start_url and current_url in self.visited:
                continue
            print(Fore.CYAN + f"\nVisiting: {current_url}")

            if current_url not in self.cache:
                # First-time visit: extract and cache data
                text, links = self.get_extracted_data(current_url)
                if text is None:
                    continue  # Skip to next URL if failed to retrieve
                self.visited.add(current_url)
                self.texts.append(text)
                print(Fore.GREEN + f"Extracted Text (first 200 chars):\n{text[:200]}...\n")
            else:
                # Revisiting a page: retrieve from cache without extracting
                text = self.cache[current_url]['text']
                links = self.cache[current_url]['links']
                print(Fore.BLUE + f"Revisited {current_url}. Using cached data.\n")

            # Add valid links to the to_visit list
            new_links = links - self.visited
            # Always include the starter page to allow revisiting
            new_links.add(self.start_url)
            # Remove URLs that have been visited except the starter page
            filtered_links = set()
            for link in new_links:
                if link == self.start_url or link not in self.visited:
                    filtered_links.add(link)
            if filtered_links:
                self.to_visit.extend(list(filtered_links))
                random.shuffle(self.to_visit)  # Shuffle to ensure random navigation
            else:
                print(Fore.YELLOW + "No new links found on this page.")

            # Wait for 5 seconds before the next request to simulate human behavior
            print(Fore.MAGENTA + "Waiting for 5 seconds before the next request...\n")
            time.sleep(5)

        print(Fore.MAGENTA + f"\nGame Over! Visited {len(self.visited)} unique pages.")
        # Optionally, you can save or process the extracted texts further.
        # For example, saving to a file:
        self.save_texts()

    def save_texts(self, filename="extracted_texts3.txt"):
        """Save all extracted texts to a file."""
        with open(filename, 'w', encoding='utf-8') as f:
            for idx, text in enumerate(self.texts, 1):
                f.write(f"--- Page {idx} ---\n")
                f.write(text + "\n\n")
        print(Fore.BLUE + f"All texts have been saved to {filename}.")

if __name__ == "__main__":
    if len(sys.argv) < 3:
        print("Usage: python web_navigator_game.py <start_url> <duration_seconds>")
        print("Example: python web_navigator_game.py https://example.com 120")
        sys.exit(1)

    start_url = sys.argv[1]
    try:
        duration = int(sys.argv[2])
    except ValueError:
        print("Duration must be an integer representing seconds.")
        sys.exit(1)

    game = WebNavigatorGame(start_url, duration)
    game.run()"