Repository: readthedocs/readthedocs.org

Issue:
Can't sync outdated Bitbucket repositories
For some reason, the refresh token for Bitbucket is failing. Is this a Bitbucket problem? Problem with our detection of expired tokens? I get this locally as well as on the community site. Removing and adding the Bitbucket connection solves the problem at least temporarily.

Traceback:

```
[05/Jul/2017 14:44:13] celery.worker.job:282[33151]: ERROR Task readthedocs.oauth.tasks.SyncRemoteRepositories[86bd1cbd-0c99-44a6-b5d7-9beadb917a7b] raised unexpected: InvalidClientIdError(u'(invalid_request) (invalid_request) Invalid
 refresh_token',)
Traceback (most recent call last):
  File "/Users/anthony/.pyenv/versions/rtd-2.7.8/lib/python2.7/site-packages/celery/app/trace.py", line 240, in trace_task
    R = retval = fun(*args, **kwargs)
  File "/Users/anthony/.pyenv/versions/rtd-2.7.8/lib/python2.7/site-packages/celery/app/trace.py", line 438, in __protected_call__
    return self.run(*args, **kwargs)
  File "/Users/anthony/dev/readthedocs.org/readthedocs/core/utils/tasks/public.py", line 71, in run
    result = self.run_public(*args, **kwargs)
  File "/Users/anthony/dev/readthedocs.org/readthedocs/oauth/tasks.py", line 22, in run_public
    service.sync()
  File "/Users/anthony/dev/readthedocs.org/readthedocs/oauth/services/bitbucket.py", line 38, in sync
    self.sync_repositories()
  File "/Users/anthony/dev/readthedocs.org/readthedocs/oauth/services/bitbucket.py", line 46, in sync_repositories
    'https://bitbucket.org/api/2.0/repositories/?role=member')
  File "/Users/anthony/dev/readthedocs.org/readthedocs/oauth/services/bitbucket.py", line 172, in paginate
    resp = self.get_session().get(url)
  File "/Users/anthony/.pyenv/versions/rtd-2.7.8/lib/python2.7/site-packages/requests/sessions.py", line 468, in get
    return self.request('GET', url, **kwargs)
  File "/Users/anthony/.pyenv/versions/rtd-2.7.8/lib/python2.7/site-packages/requests_oauthlib/oauth2_session.py", line 343, in request
    self.auto_refresh_url, auth=auth, **kwargs
  File "/Users/anthony/.pyenv/versions/rtd-2.7.8/lib/python2.7/site-packages/requests_oauthlib/oauth2_session.py", line 309, in refresh_token
    self.token = self._client.parse_request_body_response(r.text, scope=self.scope)
  File "/Users/anthony/.pyenv/versions/rtd-2.7.8/lib/python2.7/site-packages/oauthlib/oauth2/rfc6749/clients/base.py", line 409, in parse_request_body_response
    self.token = parse_token_response(body, scope=scope)
  File "/Users/anthony/.pyenv/versions/rtd-2.7.8/lib/python2.7/site-packages/oauthlib/oauth2/rfc6749/parameters.py", line 376, in parse_token_response
    validate_token_parameters(params)
  File "/Users/anthony/.pyenv/versions/rtd-2.7.8/lib/python2.7/site-packages/oauthlib/oauth2/rfc6749/parameters.py", line 383, in validate_token_parameters
    raise_from_error(params.get('error'), params)
  File "/Users/anthony/.pyenv/versions/rtd-2.7.8/lib/python2.7/site-packages/oauthlib/oauth2/rfc6749/errors.py", line 325, in raise_from_error
    raise cls(**kwargs)
InvalidClientIdError: (invalid_request) (invalid_request) Invalid refresh_token
```


Discussion:

Resolving patch:
diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py
--- a/readthedocs/oauth/services/base.py
+++ b/readthedocs/oauth/services/base.py
@@ -1,13 +1,14 @@
-"""OAuth utility functions"""
+"""OAuth utility functions."""
 
 from __future__ import absolute_import
-from builtins import str
 from builtins import object
 import logging
 from datetime import datetime
 
 from django.conf import settings
 from requests_oauthlib import OAuth2Session
+from requests.exceptions import RequestException
+from oauthlib.oauth2.rfc6749.errors import InvalidClientIdError
 from allauth.socialaccount.models import SocialAccount
 
 
@@ -69,13 +70,13 @@ def create_session(self):
             return None
 
         token_config = {
-            'access_token': str(token.token),
+            'access_token': token.token,
             'token_type': 'bearer',
         }
         if token.expires_at is not None:
             token_expires = (token.expires_at - datetime.now()).total_seconds()
             token_config.update({
-                'refresh_token': str(token.token_secret),
+                'refresh_token': token.token_secret,
                 'expires_in': token_expires,
             })
 
@@ -114,6 +115,34 @@ def _updater(data):
 
         return _updater
 
+    def paginate(self, url):
+        """Recursively combine results from service's pagination.
+
+        :param url: start url to get the data from.
+        :type url: unicode
+        """
+        try:
+            resp = self.get_session().get(url)
+            next_url = self.get_next_url_to_paginate(resp)
+            results = self.get_paginated_results(resp)
+            if next_url:
+                results.extend(self.paginate(next_url))
+            return results
+        # Catch specific exception related to OAuth
+        except InvalidClientIdError:
+            log.error('access_token or refresh_token failed: %s', url)
+            raise Exception('You should reconnect your account')
+        # Catch exceptions with request or deserializing JSON
+        except (RequestException, ValueError):
+            # Response data should always be JSON, still try to log if not though
+            try:
+                debug_data = resp.json()
+            except ValueError:
+                debug_data = resp.content
+            log.debug('paginate failed at %s with response: %s', url, debug_data)
+        finally:
+            return []
+
     def sync(self):
         raise NotImplementedError
 
@@ -124,6 +153,22 @@ def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL,
     def create_organization(self, fields):
         raise NotImplementedError
 
+    def get_next_url_to_paginate(self, response):
+        """Return the next url to feed the `paginate` method.
+
+        :param response: response from where to get the `next_url` attribute
+        :type response: requests.Response
+        """
+        raise NotImplementedError
+
+    def get_paginated_results(self, response):
+        """Return the results for the current response/page.
+
+        :param response: response from where to get the results.
+        :type response: requests.Response
+        """
+        raise NotImplementedError
+
     def setup_webhook(self, project):
         raise NotImplementedError
 
diff --git a/readthedocs/oauth/services/bitbucket.py b/readthedocs/oauth/services/bitbucket.py
--- a/readthedocs/oauth/services/bitbucket.py
+++ b/readthedocs/oauth/services/bitbucket.py
@@ -164,18 +164,11 @@ def create_organization(self, fields):
         organization.save()
         return organization
 
-    def paginate(self, url):
-        """Recursively combine results from Bitbucket pagination
+    def get_next_url_to_paginate(self, response):
+        return response.json().get('next')
 
-        :param url: start url to get the data from.
-        """
-        resp = self.get_session().get(url)
-        data = resp.json()
-        results = data.get('values', [])
-        next_url = data.get('next')
-        if next_url:
-            results.extend(self.paginate(next_url))
-        return results
+    def get_paginated_results(self, response):
+        return response.json().get('values', [])
 
     def get_webhook_data(self, project, integration):
         """Get webhook JSON data to post to the API"""
diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py
--- a/readthedocs/oauth/services/github.py
+++ b/readthedocs/oauth/services/github.py
@@ -148,19 +148,11 @@ def create_organization(self, fields):
         organization.save()
         return organization
 
-    def paginate(self, url):
-        """Combines return from GitHub pagination
+    def get_next_url_to_paginate(self, response):
+        return response.links.get('next', {}).get('url')
 
-        :param url: start url to get the data from.
-
-        See https://developer.github.com/v3/#pagination
-        """
-        resp = self.get_session().get(url)
-        result = resp.json()
-        next_url = resp.links.get('next', {}).get('url')
-        if next_url:
-            result.extend(self.paginate(next_url))
-        return result
+    def get_paginated_results(self, response):
+        return response.json()
 
     def get_webhook_data(self, project, integration):
         """Get webhook JSON data to post to the API"""


Test patch: