Repository: python-gitlab/python-gitlab

Issue:
gitlab-artifact-cleanup script unable to delete some artifacts
The script worked fine on most projects but not this last one.
Here is the error we get.
powerserver7 / Worklist
Traceback (most recent call last):
  File "/opt/gitlab-artifact-cleanup/gitlab-artifact-cleanup", line 181, in <module>
    main()
------ more logs-----
------ more logs-----
 File "/usr/local/lib/python2.7/dist-packages/python_gitlab-0.15.1-py2.7.egg/gitlab/__init__.py", line 392, in _raw_list
    results.extend(self.list(cls, **args))
  File "/usr/local/lib/python2.7/dist-packages/python_gitlab-0.15.1-py2.7.egg/gitlab/__init__.py", line 468, in list
    return self._raw_list(url, obj_class, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/python_gitlab-0.15.1-py2.7.egg/gitlab/__init__.py", line 392, in _raw_list
    results.extend(self.list(cls, **args))
  File "/usr/local/lib/python2.7/dist-packages/python_gitlab-0.15.1-py2.7.egg/gitlab/__init__.py", line 468, in list
    return self._raw_list(url, obj_class, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/python_gitlab-0.15.1-py2.7.egg/gitlab/__init__.py", line 374, in _raw_list
    r = self._raw_get(path_, **params)
  File "/usr/local/lib/python2.7/dist-packages/python_gitlab-0.15.1-py2.7.egg/gitlab/__init__.py", line 360, in _raw_get
    "Can't connect to GitLab server (%s)" % e)
gitlab.exceptions.GitlabConnectionError: Can't connect to GitLab server (maximum recursion depth exceeded)


And this is our system info.
System information
System:         Ubuntu 14.04
Current User:   git
Using RVM:      no
Ruby Version:   2.3.1p112
Gem Version:    2.6.6
Bundler Version:1.13.6
Rake Version:   10.5.0
Sidekiq Version:4.2.1

GitLab information
Version:        8.14.4
Revision:       3ea6c8d
Directory:      /opt/gitlab/embedded/service/gitlab-rails
DB Adapter:     postgresql
URL:            	
HTTP Clone URL: 	
SSH Clone URL:  	
Using LDAP:     yes
Using Omniauth: no

GitLab Shell
Version:        4.0.3
Repository storage paths:
- default:      /var/opt/gitlab/git-data/repositories
Hooks:          /opt/gitlab/embedded/service/gitlab-shell/hooks/
Git:            /opt/gitlab/embedded/bin/git

Please help


Discussion:
Could you check with the gitlab-artifact-cleanup project what could happen? https://github.com/JonathonReinhart/gitlab-artifact-cleanup

I can't be sure that your problem is related to python-gitlab. Thanks.

Resolving patch:
diff --git a/gitlab/__init__.py b/gitlab/__init__.py
--- a/gitlab/__init__.py
+++ b/gitlab/__init__.py
@@ -112,8 +112,8 @@ def __init__(self, url, private_token=None, email=None, password=None,
         # build the "submanagers"
         for parent_cls in six.itervalues(globals()):
             if (not inspect.isclass(parent_cls)
-               or not issubclass(parent_cls, GitlabObject)
-               or parent_cls == CurrentUser):
+                or not issubclass(parent_cls, GitlabObject)
+                    or parent_cls == CurrentUser):
                 continue
 
             if not parent_cls.managers:
@@ -312,11 +312,13 @@ def _raw_list(self, path_, cls, extra_attrs={}, **kwargs):
         params = extra_attrs.copy()
         params.update(kwargs.copy())
 
-        get_all_results = kwargs.get('all', False)
+        catch_recursion_limit = kwargs.get('safe_all', False)
+        get_all_results = (kwargs.get('all', False) is True
+                           or catch_recursion_limit)
 
         # Remove these keys to avoid breaking the listing (urls will get too
         # long otherwise)
-        for key in ['all', 'next_url']:
+        for key in ['all', 'next_url', 'safe_all']:
             if key in params:
                 del params[key]
 
@@ -334,12 +336,20 @@ def _raw_list(self, path_, cls, extra_attrs={}, **kwargs):
 
         results = [cls(self, item, **params) for item in r.json()
                    if item is not None]
-        if ('next' in r.links and 'url' in r.links['next']
-           and get_all_results is True):
-            args = kwargs.copy()
-            args.update(extra_attrs)
-            args['next_url'] = r.links['next']['url']
-            results.extend(self.list(cls, **args))
+        try:
+            if ('next' in r.links and 'url' in r.links['next']
+                    and get_all_results):
+                args = kwargs.copy()
+                args.update(extra_attrs)
+                args['next_url'] = r.links['next']['url']
+                results.extend(self.list(cls, **args))
+        except Exception as e:
+            # Catch the recursion limit exception if the 'safe_all'
+            # kwarg was provided
+            if not (catch_recursion_limit and
+                    "maximum recursion depth exceeded" in str(e)):
+                raise e
+
         return results
 
     def _raw_post(self, path_, data=None, content_type=None, **kwargs):


Test patch:
diff --git a/gitlab/tests/test_gitlab.py b/gitlab/tests/test_gitlab.py
--- a/gitlab/tests/test_gitlab.py
+++ b/gitlab/tests/test_gitlab.py
@@ -26,6 +26,7 @@
 from httmock import HTTMock  # noqa
 from httmock import response  # noqa
 from httmock import urlmatch  # noqa
+import six
 
 import gitlab
 from gitlab import *  # noqa
@@ -243,6 +244,75 @@ def resp_two(url, request):
             self.assertEqual(data[0].ref, "b")
             self.assertEqual(len(data), 2)
 
+    def test_list_recursion_limit_caught(self):
+        @urlmatch(scheme="http", netloc="localhost",
+                  path='/api/v3/projects/1/repository/branches', method="get")
+        def resp_one(url, request):
+            """First request:
+
+            http://localhost/api/v3/projects/1/repository/branches?per_page=1
+            """
+            headers = {
+                'content-type': 'application/json',
+                'link': '<http://localhost/api/v3/projects/1/repository/branc'
+                'hes?page=2&per_page=0>; rel="next", <http://localhost/api/v3'
+                '/projects/1/repository/branches?page=2&per_page=0>; rel="las'
+                't", <http://localhost/api/v3/projects/1/repository/branches?'
+                'page=1&per_page=0>; rel="first"'
+            }
+            content = ('[{"branch_name": "otherbranch", '
+                       '"project_id": 1, "ref": "b"}]').encode("utf-8")
+            resp = response(200, content, headers, None, 5, request)
+            return resp
+
+        @urlmatch(scheme="http", netloc="localhost",
+                  path='/api/v3/projects/1/repository/branches', method="get",
+                  query=r'.*page=2.*')
+        def resp_two(url, request):
+            # Mock a runtime error
+            raise RuntimeError("maximum recursion depth exceeded")
+
+        with HTTMock(resp_two, resp_one):
+            data = self.gl.list(ProjectBranch, project_id=1, per_page=1,
+                                safe_all=True)
+            self.assertEqual(data[0].branch_name, "otherbranch")
+            self.assertEqual(data[0].project_id, 1)
+            self.assertEqual(data[0].ref, "b")
+            self.assertEqual(len(data), 1)
+
+    def test_list_recursion_limit_not_caught(self):
+        @urlmatch(scheme="http", netloc="localhost",
+                  path='/api/v3/projects/1/repository/branches', method="get")
+        def resp_one(url, request):
+            """First request:
+
+            http://localhost/api/v3/projects/1/repository/branches?per_page=1
+            """
+            headers = {
+                'content-type': 'application/json',
+                'link': '<http://localhost/api/v3/projects/1/repository/branc'
+                'hes?page=2&per_page=0>; rel="next", <http://localhost/api/v3'
+                '/projects/1/repository/branches?page=2&per_page=0>; rel="las'
+                't", <http://localhost/api/v3/projects/1/repository/branches?'
+                'page=1&per_page=0>; rel="first"'
+            }
+            content = ('[{"branch_name": "otherbranch", '
+                       '"project_id": 1, "ref": "b"}]').encode("utf-8")
+            resp = response(200, content, headers, None, 5, request)
+            return resp
+
+        @urlmatch(scheme="http", netloc="localhost",
+                  path='/api/v3/projects/1/repository/branches', method="get",
+                  query=r'.*page=2.*')
+        def resp_two(url, request):
+            # Mock a runtime error
+            raise RuntimeError("maximum recursion depth exceeded")
+
+        with HTTMock(resp_two, resp_one):
+            with six.assertRaisesRegex(self, GitlabError,
+                                       "(maximum recursion depth exceeded)"):
+                self.gl.list(ProjectBranch, project_id=1, per_page=1, all=True)
+
     def test_list_401(self):
         @urlmatch(scheme="http", netloc="localhost",
                   path="/api/v3/projects/1/repository/branches", method="get")