Rewrite the following script to be just a proxy scraper and verifier which saves valid proxies to a file called validproxies.tx. add the following proxy sources as well

##new sources
https://github.com/gitrecon1455/ProxyScraper/raw/refs/heads/main/proxies.txt ips start on the line after "// успешный ответ проверки" in the format: ip:port
https://github.com/r00tee/Proxy-List/raw/refs/heads/main/Https.txt format: ip:port
https://github.com/themiralay/Proxy-List-World/raw/refs/heads/master/data.txt format: ip:port
https://github.com/zloi-user/hideip.me/raw/refs/heads/main/https.txt format: ip:port:country
https://github.com/jepluk/PROXYLIST/raw/refs/heads/main/all.json format: {"data": [{"address": ipv4}]}
https://github.com/proxifly/free-proxy-list/raw/refs/heads/main/proxies/protocols/https/data.txt format: http://ip:port
https://github.com/ErcinDedeoglu/proxies/raw/refs/heads/main/proxies/https.txt format: ip:port
https://github.com/SevenworksDev/proxy-list/raw/refs/heads/main/proxies/https.txt format: ip:port
https://github.com/andigwandi/free-proxy/raw/refs/heads/main/proxy_list.txt format: ip:port
https://github.com/roosterkid/openproxylist/raw/refs/heads/main/HTTPS_RAW.txt format: ip:port
https://raw.githubusercontent.com/Tsprnay/Proxy-lists/master/proxies/https.txt format: ip:port
https://github.com/vakhov/fresh-proxy-list/raw/refs/heads/master/https.txt format: ip:port
https://github.com/MrMarble/proxy-list/raw/refs/heads/main/all.txt format: ip:port
https://github.com/sunny9577/proxy-scraper/raw/refs/heads/master/generated/http_proxies.txt format: ip:port
https://github.com/mmpx12/proxy-list/raw/refs/heads/master/https.txt format: ip:port



##the script
import random
import string
import requests
import time
import re
import threading
import asyncio
import aiohttp  # Asynchronous HTTP client
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timedelta
import argparse  # For command-line argument parsing

# Import colorama for colored console output
from colorama import init, Fore

# Initialize colorama
init(autoreset=True)

# Constants
GOFILE_URL_TEMPLATE = "https://gofile.io/d/{}"
DISCORD_WEBHOOK_URL = "https://discord.com/api/webhooks/..."  # Replace with your webhook URL

# List of proxy sources
PROXY_SOURCES = [
    "https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt",
    "https://github.com/hendrikbgr/Free-Proxy-Repo/raw/refs/heads/master/proxy_list.txt",
    "https://raw.githubusercontent.com/Vann-Dev/proxy-list/refs/heads/main/proxies/https-tested/google.txt",
    "https://github.com/proxifly/free-proxy-list/raw/main/proxies/protocols/https/data.txt",
    "https://raw.githubusercontent.com/Zaeem20/FREE_PROXIES_LIST/refs/heads/master/https.txt",
    "https://raw.githubusercontent.com/babyhagey74/free-proxies/refs/heads/main/proxies/https/https.txt",
]

# List of endpoints to use for proxy validation
PROXY_VALIDATION_ENDPOINTS = [
    "https://www.google.com/humans.txt",                  # Current endpoint
    "http://httpbin.org/ip",
    "https://humanstxt.org/humans.txt",
    "https://www.berkshirehathaway.com/disclaimer.html",
    "https://fastestwebsite.net/",
]

MAX_PROXY_VALIDATION_WORKERS = 100  # Limit the number of concurrent validation tasks

# Thread-local storage for per-thread Session objects
thread_local = threading.local()

# Shared variables and locks for thread safety
valid_urls = []
valid_urls_lock = threading.Lock()
rate_limited_lock = threading.Lock()
proxies_list_lock = threading.Lock()
print_lock = threading.Lock()  # Lock for printing to console

# Rate limiting variables
rate_limited = False
using_proxies = False  # Indicates if we are currently using proxies
proxies_use_start_time = None  # Time when we started using proxies for rate limiting

# Shared proxy list and index
proxies_list = []
proxy_index = 0  # Index to track which proxy to use next

# Command-line arguments
args = None  # Will be set by parse_arguments()


def parse_arguments():
    """Parse command-line arguments."""
    parser = argparse.ArgumentParser(description="GoFile URL Checker with Proxy Support")
    group = parser.add_mutually_exclusive_group()
    group.add_argument(
        '--no-proxy',
        action='store_true',
        help='Disable proxies completely.'
    )
    group.add_argument(
        '--only-proxy',
        action='store_true',
        help='Use only proxies for all requests.'
    )
    group.add_argument(
        '--rate-limit-proxy',
        action='store_true',
        help='Use proxies for one minute each time rate limiting is detected. (Default behavior)'
    )
    return parser.parse_args()


def download_and_update_proxies():
    """Download proxies from multiple sources, validate them, and update the proxy list."""
    global proxies_list, proxy_index
    raw_proxies = set()

    with print_lock:
        print(f"{Fore.CYAN}[INFO] Downloading proxies from multiple sources...")

    for source_url in PROXY_SOURCES:
        try:
            response = requests.get(source_url, timeout=10)
            if response.status_code == 200:
                proxy_content = response.text.strip()
                proxies = [line.strip() for line in proxy_content.splitlines() if line.strip()]
                raw_proxies.update(proxies)
                with print_lock:
                    print(f"{Fore.CYAN}[INFO] Fetched {len(proxies)} proxies from {source_url}")
            else:
                with print_lock:
                    print(f"{Fore.YELLOW}[WARNING] Failed to download proxies from {source_url}. Status code: {response.status_code}")
        except requests.RequestException as e:
            with print_lock:
                print(f"{Fore.RED}[ERROR] Exception occurred while downloading proxies from {source_url}: {e}")

    if not raw_proxies:
        with print_lock:
            print(f"{Fore.RED}[ERROR] No proxies fetched from any source.")
        proxies_list = []
        return

    with print_lock:
        print(f"{Fore.CYAN}[INFO] Total unique proxies fetched: {len(raw_proxies)}")

    # Save proxies to file
    with open("gofileproxies.txt", 'w') as f:
        for proxy in raw_proxies:
            f.write(f"{proxy}\n")

    # Validate proxies
    validated_proxies = asyncio.run(validate_proxies_async(list(raw_proxies)))
    with proxies_list_lock:
        proxies_list = validated_proxies
        proxy_index = 0  # Reset the index when proxies are updated

    with print_lock:
        print(f"{Fore.CYAN}[INFO] Total proxies after validation: {len(proxies_list)}")


async def validate_proxy_async(session, proxy, semaphore):
    """Asynchronously validate a single proxy using a random endpoint and ensure load time under 1.3 seconds."""
    async with semaphore:
        # Normalize proxy format
        if proxy.startswith("http://") or proxy.startswith("https://"):
            proxy_url = proxy
        else:
            proxy_url = f"http://{proxy}"

        # Choose a random endpoint for validating this proxy
        test_url = random.choice(PROXY_VALIDATION_ENDPOINTS)

        try:
            start_time = time.time()
            async with session.get(test_url, proxy=proxy_url, timeout=1.3) as response:
                elapsed_time = time.time() - start_time
                if response.status == 200 and elapsed_time <= 1.3:
                    with print_lock:
                        print(f"{Fore.GREEN}[VALID PROXY] {proxy} (Load time: {elapsed_time:.2f}s)")
                    return proxy_url  # Return the normalized proxy URL
        except Exception:
            pass  # Ignore exceptions for invalid proxies

        with print_lock:
            print(f"{Fore.RED}[INVALID PROXY] {proxy}")
        return None


async def validate_proxies_async(proxy_list):
    """Asynchronously validate proxies with controlled concurrency."""
    valid_proxies = []
    semaphore = asyncio.Semaphore(MAX_PROXY_VALIDATION_WORKERS)
    connector = aiohttp.TCPConnector(limit_per_host=MAX_PROXY_VALIDATION_WORKERS * 2, ssl=False)
    timeout = aiohttp.ClientTimeout(total=1.5)
    async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
        tasks = []
        for proxy in proxy_list:
            task = asyncio.create_task(validate_proxy_async(session, proxy, semaphore))
            tasks.append(task)
        results = []
        for task in asyncio.as_completed(tasks):
            result = await task
            if result:
                valid_proxies.append(result)
    return valid_proxies


def generate_random_string(length=6):
    """Generate a random string of letters and digits."""
    return ''.join(random.choices(string.ascii_letters + string.digits, k=length))


def get_session():
    """Get a Session object for the current thread to enable connection pooling."""
    if not hasattr(thread_local, "session"):
        session = requests.Session()
        # Set up retry strategy
        retries = requests.adapters.Retry(
            total=2,  # Reduced total retries for faster failure
            backoff_factor=0.1,  # Reduced backoff for faster retry
            status_forcelist=[500, 502, 503, 504],
            allowed_methods=["HEAD", "GET", "OPTIONS"]
        )
        adapter = requests.adapters.HTTPAdapter(max_retries=retries, pool_connections=1000, pool_maxsize=1000)
        session.mount('http://', adapter)
        session.mount('https://', adapter)
        thread_local.session = session
    return thread_local.session


def get_proxy():
    """Get the next proxy from the list in a thread-safe manner."""
    global proxy_index
    with proxies_list_lock:
        if proxies_list:
            # Rotate through the proxy list
            proxy = proxies_list[proxy_index % len(proxies_list)]
            proxy_index += 1
            # Format the proxy for requests
            proxy_dict = {"http": proxy, "https": proxy}
            return proxy_dict
        else:
            return None


def check_rate_limit(response):
    """Check if the response indicates rate limiting."""
    # Check for HTTP 429 Too Many Requests
    if response.status_code == 429:
        retry_after = response.headers.get("Retry-After")
        return True, retry_after
    # Additional checks can be added here
    return False, None


def check_gofile_url(random_string):
    """
    Check if the randomly generated GoFile URL is valid by looking for a specific meta tag.
    Returns a tuple of (valid, url).
    """
    global rate_limited, using_proxies, proxies_use_start_time
    session = get_session()
    url = GOFILE_URL_TEMPLATE.format(random_string)
    download_meta_pattern = r"<meta name='description' content='\d+ downloads' />"

    # Determine if we need to use a proxy
    with rate_limited_lock:
        current_using_proxies = using_proxies

    if args.only_proxy:
        current_using_proxies = True
    elif args.no_proxy:
        current_using_proxies = False
    else:
        # If rate-limit-proxy mode (default), adjust based on rate limiting detection
        pass  # current_using_proxies remains as per rate limiting status

    if current_using_proxies:
        if not args.only_proxy:
            # Check if one minute has elapsed since we started using proxies
            with rate_limited_lock:
                elapsed_time = (datetime.now() - proxies_use_start_time).total_seconds()
                if elapsed_time >= 60:
                    using_proxies = False
                    current_using_proxies = False
                    with print_lock:
                        print(f"{Fore.YELLOW}[INFO] One minute has passed. Switching back to no proxies.")
        proxy = get_proxy()
        if proxy is None:
            with print_lock:
                print(f"{Fore.YELLOW}[WARNING] No proxies available.")
            proxy = None
    else:
        proxy = None

    try:
        response = session.get(url, timeout=10, proxies=proxy)
        # Check for rate limiting
        is_rate_limited, retry_after = check_rate_limit(response)
        if is_rate_limited:
            with rate_limited_lock:
                if not using_proxies and args.rate_limit_proxy:
                    using_proxies = True
                    proxies_use_start_time = datetime.now()
                    with print_lock:
                        print(f"{Fore.YELLOW}[INFO] Rate limiting detected. Switching to proxies for one minute.")
            return None, url

        # Process the response
        if response.status_code == 200:
            html_content = response.text
            if re.search(download_meta_pattern, html_content, re.IGNORECASE):
                return True, url
            return False, url
        else:
            return False, url
    except requests.RequestException as e:
        with print_lock:
            print(f"{Fore.RED}[ERROR] Exception for URL {url}: {e}")
        return None, url


def send_to_discord(urls):
    """Send the valid URLs to the specified Discord webhook in batches."""
    # Split the URLs into chunks due to Discord message length limits
    max_message_length = 2000
    content = "**Valid URLs found:**\n"
    for i, url in enumerate(urls, 1):
        entry = f"{i}. {url}\n"
        if len(content) + len(entry) > max_message_length:
            # Send the current content and start a new message
            data = {"content": content}
            try:
                response = requests.post(DISCORD_WEBHOOK_URL, json=data, timeout=10)
                if response.status_code not in (200, 204):
                    with print_lock:
                        print(f"{Fore.RED}[ERROR] Discord webhook failed with status code {response.status_code}: {response.text}")
            except requests.RequestException as e:
                with print_lock:
                    print(f"{Fore.RED}[ERROR] Failed to send data to Discord: {e}")
            content = ""
        content += entry
    if content:
        data = {"content": content}
        try:
            response = requests.post(DISCORD_WEBHOOK_URL, json=data, timeout=10)
            if response.status_code not in (200, 204):
                with print_lock:
                    print(f"{Fore.RED}[ERROR] Discord webhook failed with status code {response.status_code}: {response.text}")
        except requests.RequestException as e:
            with print_lock:
                print(f"{Fore.RED}[ERROR] Failed to send data to Discord: {e}")


def worker():
    """Worker function to generate a URL, check it, and collect valid URLs in batches."""
    random_string = generate_random_string()
    url = GOFILE_URL_TEMPLATE.format(random_string)

    # Print that we're checking the URL
    with print_lock:
        print(f"{Fore.CYAN}[CHECKING] {url}")

    valid, url = check_gofile_url(random_string)

    if valid is True:
        with print_lock:
            print(f"{Fore.GREEN}[FOUND] URL exists: {url}")
        should_send = False
        with valid_urls_lock:
            valid_urls.append(url)
            if len(valid_urls) >= 5:
                should_send = True
                urls_to_send = valid_urls.copy()
                valid_urls.clear()
        if should_send:
            send_to_discord(urls_to_send)
    elif valid is False:
        # You may choose to log not found URLs or ignore them
        pass
    else:
        # None indicates an error or rate limit; handled inside check_gofile_url
        pass


def main():
    global args
    args = parse_arguments()

    if args.no_proxy:
        with print_lock:
            print(f"{Fore.CYAN}[INFO] Running without proxies.")
    else:
        # Download and validate proxies only once at the start
        download_and_update_proxies()
        if not proxies_list and (args.only_proxy or args.rate_limit_proxy):
            with print_lock:
                print(f"{Fore.RED}[ERROR] No valid proxies available. Exiting.")
            return

    max_workers = 100  # Adjust based on your system's capabilities
    max_pending_tasks = 1000  # Adjust to handle more tasks

    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        futures = set()
        try:
            while True:
                # Remove completed futures from the set
                futures = {future for future in futures if not future.done()}

                # If the number of pending tasks is below the threshold, submit new tasks
                if len(futures) < max_pending_tasks:
                    future = executor.submit(worker)
                    futures.add(future)
                else:
                    # Avoid tight loop when max pending tasks is reached
                    time.sleep(0.01)
        except KeyboardInterrupt:
            with print_lock:
                print(f"{Fore.CYAN}[INFO] Stopping...")
        finally:
            # Optionally cancel pending futures
            for future in futures:
                future.cancel()
            # Send any remaining valid URLs
            with valid_urls_lock:
                if valid_urls:
                    send_to_discord(valid_urls.copy())
                    valid_urls.clear()


if __name__ == "__main__":
    main()