Repository: my-dev-app/AProxyRelay (language: python)

Issue:
Feature/additional proxy lists



Discussion / hints:

Fix patch:
diff --git a/aproxyrelay/req.py b/aproxyrelay/req.py
index b4f4d4d..63fc283 100644
--- a/aproxyrelay/req.py
+++ b/aproxyrelay/req.py
@@ -120,6 +120,7 @@ async def _test_proxy_link(self, proxy_url, data, session) -> None:
             ClientOSError,
             ServerTimeoutError,
             InvalidURL,
+            ConnectionResetError,
         ):
             self._filtered_failed = self._filtered_failed + 1
 
@@ -178,7 +179,7 @@ async def _obtain_targets(self, proxy_url, target, session) -> None:
                 },
             ) as response:
                 status = response.status
-                if status == 200:
+                if status in (200, 202,):
                     self.proxies.put(proxy_url)
                     data = await response.json()
                     if data:
diff --git a/aproxyrelay/scrapers/__init__.py b/aproxyrelay/scrapers/__init__.py
index b28d16a..9842336 100644
--- a/aproxyrelay/scrapers/__init__.py
+++ b/aproxyrelay/scrapers/__init__.py
@@ -29,6 +29,11 @@
 from .parser_murongpig_proxy_master_http import ParserMurongpigProxyMasterHttp
 from .parser_murongpig_proxy_master_socks4 import ParserMurongpigProxyMasterSocks4
 from .parser_murongpig_proxy_master_socks5 import ParserMurongpigProxyMasterSocks5
+from .parser_geonode import ParserGeonodeProxy
+from .parser_proxyscraper_com import ParserScraperComProxy
+from .parser_the_speed_http import ParserTheSpeedHTTPProxy
+from .parser_the_speed_sock4 import ParserTheSpeedSock4Proxy
+from .parser_the_speed_sock5 import ParserTheSpeedSock5Proxy
 
 
 proxy_list = [
@@ -108,4 +113,24 @@
         'url': 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',
         'parser': ParserMurongpigProxyMasterSocks5,
     },
+    {
+        'url': 'https://proxylist.geonode.com/api/proxy-list?country=US&limit=500&page=1&sort_by=lastChecked&sort_type=desc',
+        'parser': ParserGeonodeProxy
+    },
+    {
+        'url': 'https://api.proxyscrape.com/v3/free-proxy-list/get?request=displayproxies&country=us&proxy_format=protocolipport&format=text&timeout=20000',  # noqa: B950
+        'parser': ParserScraperComProxy
+    },
+    {
+        'url': 'https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt',
+        'parser': ParserTheSpeedHTTPProxy
+    },
+    {
+        'url': 'https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt',
+        'parser': ParserTheSpeedSock4Proxy
+    },
+    {
+        'url': 'https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt',
+        'parser': ParserTheSpeedSock5Proxy
+    },
 ]
diff --git a/aproxyrelay/scrapers/parser_freeproxylist.py b/aproxyrelay/scrapers/parser_freeproxylist.py
index 90e6443..312f106 100644
--- a/aproxyrelay/scrapers/parser_freeproxylist.py
+++ b/aproxyrelay/scrapers/parser_freeproxylist.py
@@ -54,7 +54,7 @@ async def format_data(cls, zone: str, data: dict, queue: Queue) -> None:
         """Data formatter, formats data and returns is back in the process Queue"""
         if data['Code'] == zone.upper():
             queue.put({
-                'zone': data['Code'],
+                'zone': data['Code'].upper(),
                 'method': 'http' if data['Https'] != 'yes' else 'https',
                 'anonymity': 'anonymous' if data['Anonymity'].lower() in ['elite', 'anonymous', 'elite proxy'] else 'transparent',
                 'protocol': 'http' if data['Https'] != 'yes' else 'https',
diff --git a/aproxyrelay/scrapers/parser_geonode.py b/aproxyrelay/scrapers/parser_geonode.py
new file mode 100644
index 0000000..c193d60
--- /dev/null
+++ b/aproxyrelay/scrapers/parser_geonode.py
@@ -0,0 +1,48 @@
+# -*- mode: python ; coding: utf-8 -*-
+"""
+░░      ░░       ░░       ░░░      ░░  ░░░░  ░  ░░░░  ░       ░░        ░  ░░░░░░░░      ░░  ░░░░  ░
+▒  ▒▒▒▒  ▒  ▒▒▒▒  ▒  ▒▒▒▒  ▒  ▒▒▒▒  ▒▒  ▒▒  ▒▒▒  ▒▒  ▒▒  ▒▒▒▒  ▒  ▒▒▒▒▒▒▒  ▒▒▒▒▒▒▒  ▒▒▒▒  ▒▒  ▒▒  ▒▒
+▓  ▓▓▓▓  ▓       ▓▓       ▓▓  ▓▓▓▓  ▓▓▓    ▓▓▓▓▓    ▓▓▓       ▓▓      ▓▓▓  ▓▓▓▓▓▓▓  ▓▓▓▓  ▓▓▓    ▓▓▓
+█        █  ███████  ███  ██  ████  ██  ██  █████  ████  ███  ██  ███████  ███████        ████  ████
+█  ████  █  ███████  ████  ██      ██  ████  ████  ████  ████  █        █        █  ████  ████  ████
+By undeƒined
+------------
+
+Main parser example, other parsers can inherit from this class
+"""
+from queue import Queue
+
+from .parser import MainScraper
+
+
+class ParserGeonodeProxy(MainScraper):
+    def __init__(self) -> None:
+        MainScraper.__init__(self)
+        self.zone = None
+
+    @classmethod
+    async def format_url(cls, url, *args, **kwargs) -> str:
+        """Formats URL before scraping, let us adjust query parameters for each parser"""
+        cls.zone = kwargs.get("country", "US")
+        return url.replace('country=US', f'country={cls.zone.upper()}')
+
+    @classmethod
+    async def format_raw(cls, html: str) -> list:
+        """Parse text/html pages, customized method for the parser of this website"""
+        return
+
+    @classmethod
+    async def format_data(cls, zone: str, data: dict, queue: Queue) -> None:
+        """Data formatter, formats data and returns is back in the process Queue"""
+        for item in data['data']:
+            queue.put(
+                {
+                    'zone': zone.upper(),
+                    'method': item['protocols'][0],
+                    'anonymity': 'unknown',
+                    'protocol': item['protocols'][0],
+                    'port': item['port'],
+                    'ip': item['ip']
+                }
+            )
+        return queue
diff --git a/aproxyrelay/scrapers/parser_lumiproxy.py b/aproxyrelay/scrapers/parser_lumiproxy.py
index ff7e364..9217a6a 100644
--- a/aproxyrelay/scrapers/parser_lumiproxy.py
+++ b/aproxyrelay/scrapers/parser_lumiproxy.py
@@ -32,7 +32,7 @@ async def format_data(cls, zone: str, data: dict, queue: Queue) -> None:
         for item in data['data']['list']:
             if item['country_code'] == zone.upper():
                 queue.put({
-                    'zone': item['country_code'],
+                    'zone': item['country_code'].upper(),
                     'method': cls._get_protocol(item['protocol']),
                     'anonymity': 'transparent' if item['anonymity'] not in [1, 2] else 'anonymous',
                     'protocol': cls._get_protocol(item['protocol']),
diff --git a/aproxyrelay/scrapers/parser_proxyscrape.py b/aproxyrelay/scrapers/parser_proxyscrape.py
index 195f6e1..607df0c 100644
--- a/aproxyrelay/scrapers/parser_proxyscrape.py
+++ b/aproxyrelay/scrapers/parser_proxyscrape.py
@@ -36,7 +36,7 @@ async def format_data(cls, zone: str, data: dict, queue: Queue) -> None:
         for item in data['proxies']:
             if item['ip_data']['countryCode'] == zone.upper():
                 queue.put({
-                    'zone': item['ip_data']['countryCode'],
+                    'zone': item['ip_data']['countryCode'].upper(),
                     'method': item['protocol'],
                     'anonymity': 'anonymous' if item['anonymity'] in ['elite', 'elite proxy', 'anonymous'] else 'transparent',
                     'protocol': item['protocol'],
diff --git a/aproxyrelay/scrapers/parser_proxyscraper_com.py b/aproxyrelay/scrapers/parser_proxyscraper_com.py
new file mode 100644
index 0000000..682dae4
--- /dev/null
+++ b/aproxyrelay/scrapers/parser_proxyscraper_com.py
@@ -0,0 +1,53 @@
+# -*- mode: python ; coding: utf-8 -*-
+"""
+░░      ░░       ░░       ░░░      ░░  ░░░░  ░  ░░░░  ░       ░░        ░  ░░░░░░░░      ░░  ░░░░  ░
+▒  ▒▒▒▒  ▒  ▒▒▒▒  ▒  ▒▒▒▒  ▒  ▒▒▒▒  ▒▒  ▒▒  ▒▒▒  ▒▒  ▒▒  ▒▒▒▒  ▒  ▒▒▒▒▒▒▒  ▒▒▒▒▒▒▒  ▒▒▒▒  ▒▒  ▒▒  ▒▒
+▓  ▓▓▓▓  ▓       ▓▓       ▓▓  ▓▓▓▓  ▓▓▓    ▓▓▓▓▓    ▓▓▓       ▓▓      ▓▓▓  ▓▓▓▓▓▓▓  ▓▓▓▓  ▓▓▓    ▓▓▓
+█        █  ███████  ███  ██  ████  ██  ██  █████  ████  ███  ██  ███████  ███████        ████  ████
+█  ████  █  ███████  ████  ██      ██  ████  ████  ████  ████  █        █        █  ████  ████  ████
+By undeƒined
+------------
+
+Main parser example, other parsers can inherit from this class
+"""
+from queue import Queue
+
+from .parser import MainScraper
+
+
+class ParserScraperComProxy(MainScraper):
+    def __init__(self) -> None:
+        MainScraper.__init__(self)
+        self.zone = None
+
+    @classmethod
+    async def format_url(cls, url, *args, **kwargs) -> str:
+        """Formats URL before scraping, let us adjust query parameters for each parser"""
+        cls.zone = kwargs.get("country", "us")
+        return url
+
+    @classmethod
+    async def format_raw(cls, html: str) -> list:
+        """Parse text/html pages, customized method for the parser of this website"""
+        result = []
+        data = html.split('\r\n')
+        for proxy in data:
+            if proxy:
+                protocol = proxy.split('://')[0]
+                ip = proxy.split('://')[1].split(':')[0]
+                port = proxy.split('://')[1].split(':')[-1]
+                result.append({
+                    'zone': cls.zone.upper(),
+                    'method': protocol,
+                    'anonymity': 'unknown',
+                    'protocol': protocol,
+                    'port': port,
+                    'ip': ip
+                })
+        return result
+
+    @classmethod
+    async def format_data(cls, zone: str, data: dict, queue: Queue) -> None:
+        """Data formatter, formats data and returns is back in the process Queue"""
+        queue.put(data)
+        return queue
diff --git a/aproxyrelay/scrapers/parser_socks_proxy.py b/aproxyrelay/scrapers/parser_socks_proxy.py
index 0ecbdcf..4e52ec2 100644
--- a/aproxyrelay/scrapers/parser_socks_proxy.py
+++ b/aproxyrelay/scrapers/parser_socks_proxy.py
@@ -54,7 +54,7 @@ async def format_data(cls, zone: str, data: dict, queue: Queue) -> None:
         """Data formatter, formats data and returns is back in the process Queue"""
         if data['Code'] == zone.upper():
             queue.put({
-                'zone': data['Code'],
+                'zone': data['Code'].upper(),
                 'method': data['Version'].lower(),
                 'anonymity': 'anonymous' if data['Anonymity'].lower() in ['elite', 'anonymous', 'elite proxy'] else 'transparent',
                 'protocol': data['Version'].lower(),
diff --git a/aproxyrelay/scrapers/parser_ssl_proxies.py b/aproxyrelay/scrapers/parser_ssl_proxies.py
index 7fc78e3..f2a2161 100644
--- a/aproxyrelay/scrapers/parser_ssl_proxies.py
+++ b/aproxyrelay/scrapers/parser_ssl_proxies.py
@@ -54,7 +54,7 @@ async def format_data(cls, zone: str, data: dict, queue: Queue) -> None:
         """Data formatter, formats data and returns is back in the process Queue"""
         if data['Code'] == zone.upper():
             queue.put({
-                'zone': data['Code'],
+                'zone': data['Code'].upper(),
                 'method': 'https',
                 'anonymity': 'anonymous' if data['Anonymity'].lower() in ['elite', 'anonymous', 'elite proxy'] else 'transparent',
                 'protocol': 'https',
diff --git a/aproxyrelay/scrapers/parser_the_speed_http.py b/aproxyrelay/scrapers/parser_the_speed_http.py
new file mode 100644
index 0000000..32c2577
--- /dev/null
+++ b/aproxyrelay/scrapers/parser_the_speed_http.py
@@ -0,0 +1,53 @@
+# -*- mode: python ; coding: utf-8 -*-
+"""
+░░      ░░       ░░       ░░░      ░░  ░░░░  ░  ░░░░  ░       ░░        ░  ░░░░░░░░      ░░  ░░░░  ░
+▒  ▒▒▒▒  ▒  ▒▒▒▒  ▒  ▒▒▒▒  ▒  ▒▒▒▒  ▒▒  ▒▒  ▒▒▒  ▒▒  ▒▒  ▒▒▒▒  ▒  ▒▒▒▒▒▒▒  ▒▒▒▒▒▒▒  ▒▒▒▒  ▒▒  ▒▒  ▒▒
+▓  ▓▓▓▓  ▓       ▓▓       ▓▓  ▓▓▓▓  ▓▓▓    ▓▓▓▓▓    ▓▓▓       ▓▓      ▓▓▓  ▓▓▓▓▓▓▓  ▓▓▓▓  ▓▓▓    ▓▓▓
+█        █  ███████  ███  ██  ████  ██  ██  █████  ████  ███  ██  ███████  ███████        ████  ████
+█  ████  █  ███████  ████  ██      ██  ████  ████  ████  ████  █        █        █  ████  ████  ████
+By undeƒined
+------------
+
+Main parser example, other parsers can inherit from this class
+"""
+from queue import Queue
+
+from .parser import MainScraper
+
+
+class ParserTheSpeedHTTPProxy(MainScraper):
+    def __init__(self) -> None:
+        MainScraper.__init__(self)
+        self.zone = None
+
+    @classmethod
+    async def format_url(cls, url, *args, **kwargs) -> str:
+        """Formats URL before scraping, let us adjust query parameters for each parser"""
+        cls.zone = kwargs.get("country", "us")
+        return url
+
+    @classmethod
+    async def format_raw(cls, html: str) -> list:
+        """Parse text/html pages, customized method for the parser of this website"""
+        result = []
+        data = html.split('\n')
+        for proxy in data:
+            if proxy:
+                protocol = 'http'
+                ip = proxy.split(':')[0]
+                port = proxy.split(':')[-1]
+                result.append({
+                    'zone': cls.zone.upper(),
+                    'method': protocol,
+                    'anonymity': 'unknown',
+                    'protocol': protocol,
+                    'port': port,
+                    'ip': ip
+                })
+        return result
+
+    @classmethod
+    async def format_data(cls, zone: str, data: dict, queue: Queue) -> None:
+        """Data formatter, formats data and returns is back in the process Queue"""
+        queue.put(data)
+        return queue
diff --git a/aproxyrelay/scrapers/parser_the_speed_sock4.py b/aproxyrelay/scrapers/parser_the_speed_sock4.py
new file mode 100644
index 0000000..f00b5e0
--- /dev/null
+++ b/aproxyrelay/scrapers/parser_the_speed_sock4.py
@@ -0,0 +1,53 @@
+# -*- mode: python ; coding: utf-8 -*-
+"""
+░░      ░░       ░░       ░░░      ░░  ░░░░  ░  ░░░░  ░       ░░        ░  ░░░░░░░░      ░░  ░░░░  ░
+▒  ▒▒▒▒  ▒  ▒▒▒▒  ▒  ▒▒▒▒  ▒  ▒▒▒▒  ▒▒  ▒▒  ▒▒▒  ▒▒  ▒▒  ▒▒▒▒  ▒  ▒▒▒▒▒▒▒  ▒▒▒▒▒▒▒  ▒▒▒▒  ▒▒  ▒▒  ▒▒
+▓  ▓▓▓▓  ▓       ▓▓       ▓▓  ▓▓▓▓  ▓▓▓    ▓▓▓▓▓    ▓▓▓       ▓▓      ▓▓▓  ▓▓▓▓▓▓▓  ▓▓▓▓  ▓▓▓    ▓▓▓
+█        █  ███████  ███  ██  ████  ██  ██  █████  ████  ███  ██  ███████  ███████        ████  ████
+█  ████  █  ███████  ████  ██      ██  ████  ████  ████  ████  █        █        █  ████  ████  ████
+By undeƒined
+------------
+
+Main parser example, other parsers can inherit from this class
+"""
+from queue import Queue
+
+from .parser import MainScraper
+
+
+class ParserTheSpeedSock4Proxy(MainScraper):
+    def __init__(self) -> None:
+        MainScraper.__init__(self)
+        self.zone = None
+
+    @classmethod
+    async def format_url(cls, url, *args, **kwargs) -> str:
+        """Formats URL before scraping, let us adjust query parameters for each parser"""
+        cls.zone = kwargs.get("country", "us")
+        return url
+
+    @classmethod
+    async def format_raw(cls, html: str) -> list:
+        """Parse text/html pages, customized method for the parser of this website"""
+        result = []
+        data = html.split('\n')
+        for proxy in data:
+            if proxy:
+                protocol = 'http'
+                ip = proxy.split(':')[0]
+                port = proxy.split(':')[-1]
+                result.append({
+                    'zone': cls.zone.upper(),
+                    'method': protocol,
+                    'anonymity': 'unknown',
+                    'protocol': protocol,
+                    'port': port,
+                    'ip': ip
+                })
+        return result
+
+    @classmethod
+    async def format_data(cls, zone: str, data: dict, queue: Queue) -> None:
+        """Data formatter, formats data and returns is back in the process Queue"""
+        queue.put(data)
+        return queue
diff --git a/aproxyrelay/scrapers/parser_the_speed_sock5.py b/aproxyrelay/scrapers/parser_the_speed_sock5.py
new file mode 100644
index 0000000..82d65b7
--- /dev/null
+++ b/aproxyrelay/scrapers/parser_the_speed_sock5.py
@@ -0,0 +1,53 @@
+# -*- mode: python ; coding: utf-8 -*-
+"""
+░░      ░░       ░░       ░░░      ░░  ░░░░  ░  ░░░░  ░       ░░        ░  ░░░░░░░░      ░░  ░░░░  ░
+▒  ▒▒▒▒  ▒  ▒▒▒▒  ▒  ▒▒▒▒  ▒  ▒▒▒▒  ▒▒  ▒▒  ▒▒▒  ▒▒  ▒▒  ▒▒▒▒  ▒  ▒▒▒▒▒▒▒  ▒▒▒▒▒▒▒  ▒▒▒▒  ▒▒  ▒▒  ▒▒
+▓  ▓▓▓▓  ▓       ▓▓       ▓▓  ▓▓▓▓  ▓▓▓    ▓▓▓▓▓    ▓▓▓       ▓▓      ▓▓▓  ▓▓▓▓▓▓▓  ▓▓▓▓  ▓▓▓    ▓▓▓
+█        █  ███████  ███  ██  ████  ██  ██  █████  ████  ███  ██  ███████  ███████        ████  ████
+█  ████  █  ███████  ████  ██      ██  ████  ████  ████  ████  █        █        █  ████  ████  ████
+By undeƒined
+------------
+
+Main parser example, other parsers can inherit from this class
+"""
+from queue import Queue
+
+from .parser import MainScraper
+
+
+class ParserTheSpeedSock5Proxy(MainScraper):
+    def __init__(self) -> None:
+        MainScraper.__init__(self)
+        self.zone = None
+
+    @classmethod
+    async def format_url(cls, url, *args, **kwargs) -> str:
+        """Formats URL before scraping, let us adjust query parameters for each parser"""
+        cls.zone = kwargs.get("country", "us")
+        return url
+
+    @classmethod
+    async def format_raw(cls, html: str) -> list:
+        """Parse text/html pages, customized method for the parser of this website"""
+        result = []
+        data = html.split('\n')
+        for proxy in data:
+            if proxy:
+                protocol = 'http'
+                ip = proxy.split(':')[0]
+                port = proxy.split(':')[-1]
+                result.append({
+                    'zone': cls.zone.upper(),
+                    'method': protocol,
+                    'anonymity': 'unknown',
+                    'protocol': protocol,
+                    'port': port,
+                    'ip': ip
+                })
+        return result
+
+    @classmethod
+    async def format_data(cls, zone: str, data: dict, queue: Queue) -> None:
+        """Data formatter, formats data and returns is back in the process Queue"""
+        queue.put(data)
+        return queue
diff --git a/setup.py b/setup.py
index f68f92d..68a2686 100644
--- a/setup.py
+++ b/setup.py
@@ -22,6 +22,7 @@
             'build',
             'flake8-bugbear',
             'pytest',
+            'pytest-asyncio',
             'pytest-cov',
             'pytest-sugar',
             'pytest-xdist',


Test patch:
diff --git a/tests/test_endpoint_availability.py b/tests/test_endpoint_availability.py
new file mode 100644
index 0000000..9751762
--- /dev/null
+++ b/tests/test_endpoint_availability.py
@@ -0,0 +1,40 @@
+import pytest
+import requests
+
+from aproxyrelay.agents import UserAgents
+from aproxyrelay.scrapers import proxy_list
+
+
+def is_website_reachable(url, agent):
+    headers = {
+        'User-Agent': agent,
+    }
+    try:
+        response = requests.get(url, timeout=5, headers=headers)
+        if response.status_code in (200, 202,):
+            return response
+        return False
+    except requests.exceptions.RequestException:
+        return False
+
+
+@pytest.mark.asyncio
+async def test_parse_proxy_data():
+    agents = UserAgents()
+    for endpoint in proxy_list:
+        url = endpoint.get("url")
+        if not url:
+            pytest.fail(f"Endpoint is missing 'url' key: {endpoint}")
+
+        parser = endpoint.get("parser")
+        if not parser:
+            pytest.fail(f"Parser for endpoint is required key: {endpoint}")
+
+        url = await parser.format_url(url)
+
+        # Check if the website is reachable without mocking (real test)
+        if not is_website_reachable(url, agents.random()):
+            pytest.fail(f"Website {url} is not reachable, cannot proceed with scraping.")
+
+        # You would continue your scraping logic here if the site is reachable
+        print(f"Website {url} is reachable, proceeding with tests.")