Repository: django-extensions/django-extensions

Issue:
pipchecker - PyPI rate limit
django-extensions==3.0.9
pip==20.2.3

NB: I'm fairly certain this is *not* related to existing GitHub-specific rate-limiting issues.

Pointing pipchecker at a sufficiently long `requirements.txt` results in the following error (redacted for readability):

```
  File "~/.virtualenvs/env/lib/python3.7/site-packages/django_extensions/management/commands/pipchecker.py", line 111, in handle
    self.check_pypi()
  File "~/.virtualenvs/env/lib/python3.7/site-packages/django_extensions/management/commands/pipchecker.py", line 148, in check_pypi
    available = pypi.package_releases(req["pip_req"].name, True) or pypi.package_releases(req["pip_req"].name.replace('-', '_'), True)
  File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/xmlrpc/client.py", line 1112, in __call__
    return self.__send(self.__name, args)
  File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/xmlrpc/client.py", line 1452, in __request
    verbose=self.__verbose
  File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/xmlrpc/client.py", line 1154, in request
    return self.single_request(host, handler, request_body, verbose)
  File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/xmlrpc/client.py", line 1170, in single_request
    return self.parse_response(resp)
  File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/xmlrpc/client.py", line 1342, in parse_response
    return u.close()
  File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/xmlrpc/client.py", line 656, in close
    raise Fault(**self._stack[0])
xmlrpc.client.Fault: <Fault -32500: 'HTTPTooManyRequests: The action could not be performed because there were too many requests by the client. Limit may reset in 58 seconds.'>
```

I believe this is related to recently added rate limiting on PyPI (can't find a news announcement but swear I remember seeing something).

In terms of a PR, would a general approach of "back off and retry" be appropriate/welcome? (Unclear if the response actually includes the retry time in e.g. a header)



Discussion:
Yes :-) PR would be much appreciated... if there is a retry-time in the header that would be the best obvious but otherwise back off and retry be great as well.

Resolving patch:
diff --git a/django_extensions/management/commands/pipchecker.py b/django_extensions/management/commands/pipchecker.py
--- a/django_extensions/management/commands/pipchecker.py
+++ b/django_extensions/management/commands/pipchecker.py
@@ -6,9 +6,10 @@
 from urllib.parse import urlparse
 from urllib.error import HTTPError
 from urllib.request import Request, urlopen
-from xmlrpc.client import ServerProxy
+from xmlrpc.client import ServerProxy, Fault
 
 import pip
+from time import sleep
 from django.core.management.base import BaseCommand, CommandError
 from django_extensions.management.color import color_style
 from django_extensions.management.utils import signalcommand
@@ -145,7 +146,17 @@ def check_pypi(self):
             elif "dist" in req:
                 dist = req["dist"]
                 dist_version = LooseVersion(dist.version)
-                available = pypi.package_releases(req["pip_req"].name, True) or pypi.package_releases(req["pip_req"].name.replace('-', '_'), True)
+                retry = True
+                available = None
+                while retry:
+                    try:
+                        available = pypi.package_releases(req["pip_req"].name, True) or pypi.package_releases(req["pip_req"].name.replace('-', '_'), True)
+                        retry = False
+                    except Fault as err:
+                        self.stdout.write(err.faultString)
+                        self.stdout.write("Retrying in 60 seconds!")
+                        sleep(60)
+
                 available_version = self._available_version(dist_version, available)
 
                 if not available_version:


Test patch:
diff --git a/tests/management/commands/test_pipchecker.py b/tests/management/commands/test_pipchecker.py
--- a/tests/management/commands/test_pipchecker.py
+++ b/tests/management/commands/test_pipchecker.py
@@ -115,3 +115,57 @@ def test_pipchecker_with_outdated_requirement_on_pip20_1(self):
         os.remove(requirements_path)
 
         self.assertTrue(value.endswith('available\n'))
+
+    def test_pipchecker_with_long_up_to_date_requirements(self):
+        requirements_path = './requirements.txt'
+        out = StringIO()
+
+        f = open(requirements_path, 'wt')
+        f.write('appdirs')
+        f.write('asgiref')
+        f.write('attrs')
+        f.write('black')
+        f.write('certifi')
+        f.write('chardet')
+        f.write('click')
+        f.write('distlib')
+        f.write('Django')
+        f.write('django-cors-headers')
+        f.write('django-debug-toolbar')
+        f.write('djangorestframework')
+        f.write('filelock')
+        f.write('idna')
+        f.write('iniconfig')
+        f.write('mypy-extensions')
+        f.write('packaging')
+        f.write('pathspec')
+        f.write('Pillow')
+        f.write('pluggy')
+        f.write('psycopg2-binary')
+        f.write('py')
+        f.write('pyparsing')
+        f.write('pytest')
+        f.write('pytz')
+        f.write('regex')
+        f.write('requests')
+        f.write('sentry-sdk')
+        f.write('shortuuid')
+        f.write('six')
+        f.write('sqlparse')
+        f.write('toml')
+        f.write('typed-ast')
+        f.write('typing-extensions')
+        f.write('urllib3')
+        f.write('whitenoise')
+        f.write('zipp')
+
+        subprocess.call([sys.executable, '-m', 'pip', 'install', '-r', requirements_path])
+        pip._vendor.pkg_resources = importlib.reload(pip._vendor.pkg_resources)
+        call_command('pipchecker', '-r', requirements_path, stdout=out)
+
+        value = out.getvalue()
+
+        subprocess.call([sys.executable, '-m', 'pip', 'uninstall', '--yes', '-r', requirements_path])
+        os.remove(requirements_path)
+
+        self.assertTrue(value.endswith("Retrying in 60 seconds!") or value == '')