Repository: LeMyst/WikibaseIntegrator

Issue:
Add retry when getting 502 HTTP errors
![bild](https://user-images.githubusercontent.com/68460690/134591492-3ad2deb2-eb62-45a7-ae3e-dd7c36ae4c98.png)


Discussion:

Patch:
diff --git a/wikibaseintegrator/datatypes/basedatatype.py b/wikibaseintegrator/datatypes/basedatatype.py
index 6804b1e..4f0393c 100644
--- a/wikibaseintegrator/datatypes/basedatatype.py
+++ b/wikibaseintegrator/datatypes/basedatatype.py
@@ -1,5 +1,4 @@
 from wikibaseintegrator.models import Claim, Reference, References, Snak, Snaks
-from wikibaseintegrator.wbi_enums import WikibaseSnakType
 
 
 class BaseDataType(Claim):
@@ -18,24 +17,7 @@ class BaseDataType(Claim):
         """
         Constructor, will be called by all data types.
 
-        :param value: Data value of the Wikibase data snak
-        :type value: str or int or tuple
         :param prop_nr: The property number a Wikibase snak belongs to
-        :type prop_nr: A string with a prefixed 'P' and several digits e.g. 'P715' (Drugbank ID) or an int
-        :param datatype: The Wikibase data type declaration of this snak
-        :type datatype: str
-        :param snaktype: One of the values in the enum WikibaseSnakValueType denoting the state of the value:
-            KNOWN_VALUE, NO_VALUE or UNKNOWN_VALUE
-        :type snaktype: WikibaseSnakType
-        :param references: A one level nested list with reference Wikibase snaks of base type BaseDataType,
-            e.g. references=[[<BaseDataType>, <BaseDataType>], [<BaseDataType>]]
-            This will create two references, the first one with two statements, the second with one
-        :type references: A one level nested list with instances of BaseDataType or children of it.
-        :param qualifiers: A list of qualifiers for the Wikibase mainsnak
-        :type qualifiers: A list with instances of BaseDataType or children of it.
-        :param rank: The rank of a Wikibase mainsnak, should determine the status of a value
-        :type rank: A string of one of three allowed values: 'normal', 'deprecated', 'preferred'
-        :return:
         """
 
         super().__init__(**kwargs)
diff --git a/wikibaseintegrator/wbi_helpers.py b/wikibaseintegrator/wbi_helpers.py
index ac0c990..9b3e1a8 100644
--- a/wikibaseintegrator/wbi_helpers.py
+++ b/wikibaseintegrator/wbi_helpers.py
@@ -21,7 +21,7 @@ class BColors:
     UNDERLINE = '\033[4m'
 
 
-def mediawiki_api_call(method, mediawiki_api_url=None, session=None, max_retries=1000, retry_after=60, **kwargs):
+def mediawiki_api_call(method, mediawiki_api_url=None, session=None, max_retries=100, retry_after=60, **kwargs):
     """
     :param method: 'GET' or 'POST'
     :param mediawiki_api_url:
@@ -53,8 +53,8 @@ def mediawiki_api_call(method, mediawiki_api_url=None, session=None, max_retries
             print(f"Connection error: {e}. Sleeping for {retry_after} seconds.")
             sleep(retry_after)
             continue
-        if response.status_code == 503:  # pragma: no cover
-            print(f"service unavailable. sleeping for {retry_after} seconds")
+        if response.status_code in (500, 502, 503, 504):
+            print(f"Service unavailable (HTTP Code {response.status_code}). Sleeping for {retry_after} seconds.")
             sleep(retry_after)
             continue
 
@@ -205,8 +205,8 @@ def execute_sparql_query(query, prefix=None, endpoint=None, user_agent=None, max
             print(f"Connection error: {e}. Sleeping for {retry_after} seconds.")
             sleep(retry_after)
             continue
-        if response.status_code == 503:
-            print(f"Service unavailable (503). Sleeping for {retry_after} seconds")
+        if response.status_code in (500, 502, 503, 504):
+            print(f"Service unavailable (HTTP Code {response.status_code}). Sleeping for {retry_after} seconds.")
             sleep(retry_after)
             continue
         if response.status_code == 429:


Test patch:
diff --git a/test/test_wbi_helpers.py b/test/test_wbi_helpers.py
index d361164..6a80431 100644
--- a/test/test_wbi_helpers.py
+++ b/test/test_wbi_helpers.py
@@ -7,13 +7,27 @@ from wikibaseintegrator.wbi_helpers import mediawiki_api_call_helper, get_user_a
 
 
 def test_connection():
+    data = {'format': 'json', 'action': 'wbgetentities', 'ids': 'Q42'}
+
+    mediawiki_api_call_helper(data=data, max_retries=2, retry_after=1, allow_anonymous=True)
+
     with unittest.TestCase().assertRaises(MWApiError):
-        mediawiki_api_call_helper(data={'format': 'json', 'action': 'wbgetentities', 'ids': 'Q42'}, mediawiki_api_url="https://www.wikidataaaaaaa.org", max_retries=3,
-                                  retry_after=1, allow_anonymous=True)
-    with unittest.TestCase().assertRaises(requests.HTTPError):
-        mediawiki_api_call_helper(data=None, mediawiki_api_url="https://httpbin.org/status/400", max_retries=3, retry_after=1, allow_anonymous=True)
+        mediawiki_api_call_helper(data=data, mediawiki_api_url="https://www.wikidataaaaaaa.org", max_retries=2, retry_after=1, allow_anonymous=True)
 
-    mediawiki_api_call_helper(data={'format': 'json', 'action': 'wbgetentities', 'ids': 'Q42'}, max_retries=3, retry_after=1, allow_anonymous=True)
+    with unittest.TestCase().assertRaises(MWApiError):
+        mediawiki_api_call_helper(data=data, mediawiki_api_url="https://httpbin.org/status/500", max_retries=2, retry_after=1, allow_anonymous=True)
+
+    with unittest.TestCase().assertRaises(MWApiError):
+        mediawiki_api_call_helper(data=data, mediawiki_api_url="https://httpbin.org/status/502", max_retries=2, retry_after=1, allow_anonymous=True)
+
+    with unittest.TestCase().assertRaises(MWApiError):
+        mediawiki_api_call_helper(data=data, mediawiki_api_url="https://httpbin.org/status/503", max_retries=2, retry_after=1, allow_anonymous=True)
+
+    with unittest.TestCase().assertRaises(MWApiError):
+        mediawiki_api_call_helper(data=data, mediawiki_api_url="https://httpbin.org/status/504", max_retries=2, retry_after=1, allow_anonymous=True)
+
+    with unittest.TestCase().assertRaises(requests.HTTPError):
+        mediawiki_api_call_helper(data=data, mediawiki_api_url="https://httpbin.org/status/400", max_retries=2, retry_after=1, allow_anonymous=True)
 
 
 def test_user_agent(capfd):