Repository: datosgobar/pydatajson

Issue:
Modificar criterio de análisis de broken URL: es válida a menos que 4xx o 5xx
Excepciones: 429 "Too Many Requests" no se debe considerar broken.

Discussion:

Patch:
diff --git a/pydatajson/constants.py b/pydatajson/constants.py
index dc49ecb..cbfbd03 100644
--- a/pydatajson/constants.py
+++ b/pydatajson/constants.py
@@ -1,6 +1,7 @@
 REQUESTS_TIMEOUT = 30
 DEFAULT_TIMEZONE = "America/Buenos_Aires"
 
-VALID_STATUS_CODES = [200, 203, 302]
+INVALID_STATUS_CODES_REGEX = ["^4[0-9]+$", "^5[0-9]+$"]
+EXCEPTION_STATUS_CODES = [429]
 
 CANT_THREADS_BROKEN_URL_VALIDATOR = 10
diff --git a/pydatajson/helpers.py b/pydatajson/helpers.py
index 638a9a8..dde2dea 100644
--- a/pydatajson/helpers.py
+++ b/pydatajson/helpers.py
@@ -18,13 +18,15 @@ from contextlib import contextmanager
 
 import requests
 from openpyxl import load_workbook
-from requests import RequestException
+from requests import RequestException, Timeout
 from six.moves.urllib_parse import urlparse
 
 from six import string_types, iteritems
 from unidecode import unidecode
 
-from pydatajson.constants import VALID_STATUS_CODES
+from pydatajson.constants import \
+    INVALID_STATUS_CODES_REGEX, \
+    EXCEPTION_STATUS_CODES
 from pydatajson.download import download_to_file
 
 logger = logging.getLogger('pydatajson.helpers')
@@ -571,6 +573,13 @@ def fields_to_uppercase(fields):
 def is_working_url(url):
     try:
         response = requests.head(url, timeout=1)
-        return response.status_code in VALID_STATUS_CODES, response.status_code
+        matches = []
+        if response.status_code not in EXCEPTION_STATUS_CODES:
+            matches = \
+                [re.match(pattern, str(response.status_code)) is not None
+                 for pattern in INVALID_STATUS_CODES_REGEX]
+        return True not in matches, response.status_code
+    except Timeout:
+        return False, 408
     except (RequestException, Exception):
         return False, None


Test patch:
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
index c5edfa6..5a7e91b 100644
--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -294,15 +294,23 @@ class HelpersTestCase(unittest.TestCase):
         req_mock.head('http://test.com/', status_code=400)
         self.assertEqual((False, 400), is_working_url('http://test.com/'))
 
+    @requests_mock.Mocker()
+    def test_validate_too_many_requests_response(self, req_mock):
+        too_many_request_status_code = 429
+        req_mock.head('http://test.com/',
+                      status_code=too_many_request_status_code)
+        self.assertEqual((True, too_many_request_status_code),
+                         is_working_url('http://test.com/'))
+
     @requests_mock.Mocker()
     def test_validate_url_with_exception(self, req_mock):
         req_mock.head('http://test.com/', exc=ConnectionError)
         self.assertEqual((False, None), is_working_url('http://test.com/'))
 
     @requests_mock.Mocker()
-    def validate_url_with_timeout(self, req_mock):
+    def test_validate_url_with_timeout(self, req_mock):
         req_mock.head('http://test.com/', exc=Timeout)
-        self.assertEqual((False, None), is_working_url('http://test.com/'))
+        self.assertEqual((False, 408), is_working_url('http://test.com/'))
 
     def test_validate_malformed_values(self):