Repository: googleapis/google-cloud-python

Issue:
oauth2client 2.0.0 breaks gcloud-python
Looks like `oauth2client` just cut a 2.0.0 release; since `'oauth2client >= 1.4.6'` is pinned in `setup.py`, this will likely break for users.

For us, the error surfaced as

```
    :   File "/www/venv/local/lib/python2.7/site-packages/oauth2client/contrib/gce.py", line 102, in _refresh
    :     raise HttpAccessTokenRefreshError(content, status=response.status)
    : HttpAccessTokenRefreshError: <!DOCTYPE html>
    : <html lang=en>
    :   <meta charset=utf-8>
    :   <meta name=viewport content="initial-scale=1, minimum-scale=1, width=device-width">
    :   <title>Error 404 (Not Found)!!1</title>
    :   <style>
    :     *{margin:0;padding:0}html,code{font:15px/22px arial,sans-serif}html{background:#fff;color:#222;padding:15px}body{margin:7% auto 0;max-width:390px;min-height:180px;padding:30px 0 15px}* > body{background:url(//www.google.com/images/errors/robot.png) 100% 5px no-repeat;padding-right:205px}p{margin:11px 0 22px;overflow:hidden}ins{color:#777;text-decoration:none}a img{border:0}@media screen and (max-width:772px){body{background:none;margin-top:0;max-width:none;padding-right:0}}#logo{background:url(//www.google.com/images/branding/googlelogo/1x/googlelogo_color_150x54dp.png) no-repeat;margin-left:-5px}@media only screen and (min-resolution:192dpi){#logo{background:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) no-repeat 0% 0%/100% 100%;-moz-border-image:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) 0}}@media only screen and (-webkit-min-device-pixel-ratio:2){#logo{background:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) no-repeat;-webkit-background-size:100% 100%}}#logo{display:inline-block;height:54px;width:150px}
    :   </style>
    :   <a href=//www.google.com/><span id=logo aria-label=Google></span></a>
    :   <p><b>404.</b> <ins>That\u2019s an error.</ins>
    :   <p>The requested URL <code>/computeMetadata/v1/instance/service-accounts/default/acquire?scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdatastore%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fuserinfo.email</code> was not found on this server.  <ins>That\u2019s all we know.</ins>
    :  This can occur if a VM was created with no service account or scopes.
```



Discussion:
Thanks. We saw it first in #1475 (surprisingly broken build).


Resolving patch:
diff --git a/gcloud/credentials.py b/gcloud/credentials.py
--- a/gcloud/credentials.py
+++ b/gcloud/credentials.py
@@ -29,7 +29,7 @@
 from oauth2client import client
 from oauth2client.client import _get_application_default_credential_from_file
 from oauth2client import crypt
-from oauth2client import service_account
+from oauth2client.service_account import ServiceAccountCredentials
 try:
     from oauth2client.appengine import AppAssertionCredentials as _GAECreds
 except ImportError:
@@ -95,7 +95,7 @@ def get_credentials():
     :rtype: :class:`oauth2client.client.GoogleCredentials`,
             :class:`oauth2client.contrib.appengine.AppAssertionCredentials`,
             :class:`oauth2client.contrib.gce.AppAssertionCredentials`,
-            :class:`oauth2client.service_account._ServiceAccountCredentials`
+            :class:`oauth2client.service_account.ServiceAccountCredentials`
     :returns: A new credentials instance corresponding to the implicit
               environment.
     """
@@ -120,7 +120,7 @@ def get_for_service_account_json(json_credentials_path, scope=None):
                   particular API.)
 
     :rtype: :class:`oauth2client.client.GoogleCredentials`,
-            :class:`oauth2client.service_account._ServiceAccountCredentials`
+            :class:`oauth2client.service_account.ServiceAccountCredentials`
     :returns: New service account or Google (for a user JSON key file)
               credentials object.
     """
@@ -154,21 +154,18 @@ def get_for_service_account_p12(client_email, private_key_path, scope=None):
                   scope is required for the different levels of access to any
                   particular API.)
 
-    :rtype: :class:`oauth2client.client.SignedJwtAssertionCredentials`
-    :returns: A new ``SignedJwtAssertionCredentials`` instance with the
+    :rtype: :class:`oauth2client.service_account.ServiceAccountCredentials`
+    :returns: A new ``ServiceAccountCredentials`` instance with the
               needed service account settings.
     """
-    return client.SignedJwtAssertionCredentials(
-        service_account_name=client_email,
-        private_key=open(private_key_path, 'rb').read(),
-        scope=scope)
+    return ServiceAccountCredentials.from_p12_keyfile(
+        client_email, private_key_path, scopes=scope)
 
 
 def _get_pem_key(credentials):
     """Gets private key for a PEM payload from a credentials object.
 
-    :type credentials: :class:`client.SignedJwtAssertionCredentials`,
-                       :class:`service_account._ServiceAccountCredentials`
+    :type credentials: :class:`service_account.ServiceAccountCredentials`,
     :param credentials: The credentials used to create a private key
                         for signing text.
 
@@ -177,12 +174,14 @@ def _get_pem_key(credentials):
     :raises: `TypeError` if `credentials` is the wrong type.
              `EnvironmentError` if `crypto` did not import successfully.
     """
-    if isinstance(credentials, client.SignedJwtAssertionCredentials):
-        # Take our PKCS12 (.p12) text and convert to PEM text.
-        pem_text = crypt.pkcs12_key_as_pem(credentials.private_key,
-                                           credentials.private_key_password)
-    elif isinstance(credentials, service_account._ServiceAccountCredentials):
-        pem_text = credentials._private_key_pkcs8_text
+    if isinstance(credentials, ServiceAccountCredentials):
+        if credentials._private_key_pkcs12 is not None:
+            # Take our PKCS12 (.p12) text and convert to PEM text.
+            pem_text = crypt.pkcs12_key_as_pem(
+                credentials._private_key_pkcs12,
+                credentials._private_key_password)
+        else:
+            pem_text = credentials._private_key_pkcs8_pem
     else:
         raise TypeError((credentials,
                          'not a valid service account credentials type'))
@@ -196,8 +195,7 @@ def _get_pem_key(credentials):
 def _get_signature_bytes(credentials, string_to_sign):
     """Uses crypto attributes of credentials to sign a string/bytes.
 
-    :type credentials: :class:`client.SignedJwtAssertionCredentials`,
-                       :class:`service_account._ServiceAccountCredentials`,
+    :type credentials: :class:`service_account.ServiceAccountCredentials`,
                        :class:`_GAECreds`
     :param credentials: The credentials used for signing text (typically
                         involves the creation of a PKey).
@@ -227,8 +225,7 @@ def _get_signature_bytes(credentials, string_to_sign):
 def _get_service_account_name(credentials):
     """Determines service account name from a credentials object.
 
-    :type credentials: :class:`client.SignedJwtAssertionCredentials`,
-                       :class:`service_account._ServiceAccountCredentials`,
+    :type credentials: :class:`service_account.ServiceAccountCredentials`,
                        :class:`_GAECreds`
     :param credentials: The credentials used to determine the service
                         account name.
@@ -239,10 +236,8 @@ def _get_service_account_name(credentials):
              account type.
     """
     service_account_name = None
-    if isinstance(credentials, client.SignedJwtAssertionCredentials):
-        service_account_name = credentials.service_account_name
-    elif isinstance(credentials, service_account._ServiceAccountCredentials):
-        service_account_name = credentials._service_account_email
+    if isinstance(credentials, ServiceAccountCredentials):
+        service_account_name = credentials.service_account_email
     elif isinstance(credentials, _GAECreds):
         service_account_name = app_identity.get_service_account_name()
 
@@ -255,8 +250,7 @@ def _get_service_account_name(credentials):
 def _get_signed_query_params(credentials, expiration, string_to_sign):
     """Gets query parameters for creating a signed URL.
 
-    :type credentials: :class:`client.SignedJwtAssertionCredentials`,
-                       :class:`service_account._ServiceAccountCredentials`
+    :type credentials: :class:`service_account.ServiceAccountCredentials`
     :param credentials: The credentials used to create a private key
                         for signing text.
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -14,7 +14,7 @@
 REQUIREMENTS = [
     'httplib2 >= 0.9.1',
     'googleapis-common-protos',
-    'oauth2client >= 1.4.6',
+    'oauth2client >= 2.0.0',
     'protobuf >= 3.0.0b2',
     'pyOpenSSL',
     'six',


Test patch:
diff --git a/gcloud/test_credentials.py b/gcloud/test_credentials.py
--- a/gcloud/test_credentials.py
+++ b/gcloud/test_credentials.py
@@ -90,47 +90,31 @@ def _callFUT(self, client_email, private_key_path, scope=None):
     def test_it(self):
         from gcloud import credentials as MUT
         from gcloud._testing import _Monkey
-        from gcloud._testing import _NamedTemporaryFile
 
         CLIENT_EMAIL = 'phred@example.com'
-        PRIVATE_KEY = b'SEEkR1t'
-        client = _Client()
-        with _Monkey(MUT, client=client):
-            with _NamedTemporaryFile() as temp:
-                with open(temp.name, 'wb') as file_obj:
-                    file_obj.write(PRIVATE_KEY)
-                found = self._callFUT(CLIENT_EMAIL, temp.name)
+        MOCK_FILENAME = 'foo.path'
+        MOCK_CRED_CLASS = _MockServiceAccountCredentials()
+        with _Monkey(MUT, ServiceAccountCredentials=MOCK_CRED_CLASS):
+            found = self._callFUT(CLIENT_EMAIL, MOCK_FILENAME)
 
-        self.assertTrue(found is client._signed)
-        expected_called_with = {
-            'service_account_name': CLIENT_EMAIL,
-            'private_key': PRIVATE_KEY,
-            'scope': None,
-        }
-        self.assertEqual(client._called_with, expected_called_with)
+        self.assertTrue(found is MOCK_CRED_CLASS._result)
+        self.assertEqual(MOCK_CRED_CLASS.p12_called,
+                         [(CLIENT_EMAIL, MOCK_FILENAME, None)])
 
     def test_it_with_scope(self):
         from gcloud import credentials as MUT
         from gcloud._testing import _Monkey
-        from gcloud._testing import _NamedTemporaryFile
 
         CLIENT_EMAIL = 'phred@example.com'
-        PRIVATE_KEY = b'SEEkR1t'
         SCOPE = 'SCOPE'
-        client = _Client()
-        with _Monkey(MUT, client=client):
-            with _NamedTemporaryFile() as temp:
-                with open(temp.name, 'wb') as file_obj:
-                    file_obj.write(PRIVATE_KEY)
-                found = self._callFUT(CLIENT_EMAIL, temp.name, SCOPE)
+        MOCK_FILENAME = 'foo.path'
+        MOCK_CRED_CLASS = _MockServiceAccountCredentials()
+        with _Monkey(MUT, ServiceAccountCredentials=MOCK_CRED_CLASS):
+            found = self._callFUT(CLIENT_EMAIL, MOCK_FILENAME, SCOPE)
 
-        self.assertTrue(found is client._signed)
-        expected_called_with = {
-            'service_account_name': CLIENT_EMAIL,
-            'private_key': PRIVATE_KEY,
-            'scope': SCOPE,
-        }
-        self.assertEqual(client._called_with, expected_called_with)
+        self.assertTrue(found is MOCK_CRED_CLASS._result)
+        self.assertEqual(MOCK_CRED_CLASS.p12_called,
+                         [(CLIENT_EMAIL, MOCK_FILENAME, SCOPE)])
 
 
 class Test_get_for_service_account_json(unittest2.TestCase):
@@ -279,8 +263,7 @@ def _run_with_fake_crypto(self, credentials, private_key_text,
             result = self._callFUT(credentials, string_to_sign)
 
         if crypt._pkcs12_key_as_pem_called:
-            self.assertEqual(crypt._private_key_text,
-                             base64.b64encode(private_key_text))
+            self.assertEqual(crypt._private_key_text, private_key_text)
             self.assertEqual(crypt._private_key_password, 'notasecret')
             self.assertEqual(openssl_crypto._loaded,
                              [(openssl_crypto.FILETYPE_PEM, _Crypt._KEY)])
@@ -296,22 +279,28 @@ def _run_with_fake_crypto(self, credentials, private_key_text,
         self.assertEqual(result, sign_result)
 
     def test_p12_type(self):
-        from oauth2client.client import SignedJwtAssertionCredentials
+        from oauth2client.service_account import ServiceAccountCredentials
         ACCOUNT_NAME = 'dummy_service_account_name'
         PRIVATE_KEY_TEXT = b'dummy_private_key_text'
         STRING_TO_SIGN = b'dummy_signature'
-        CREDENTIALS = SignedJwtAssertionCredentials(
-            ACCOUNT_NAME, PRIVATE_KEY_TEXT, [])
+        SIGNER = object()
+        CREDENTIALS = ServiceAccountCredentials(
+            ACCOUNT_NAME, SIGNER)
+        CREDENTIALS._private_key_pkcs12 = PRIVATE_KEY_TEXT
+        CREDENTIALS._private_key_password = 'notasecret'
         self._run_with_fake_crypto(CREDENTIALS, PRIVATE_KEY_TEXT,
                                    STRING_TO_SIGN)
 
     def test_p12_type_non_bytes_to_sign(self):
-        from oauth2client.client import SignedJwtAssertionCredentials
+        from oauth2client.service_account import ServiceAccountCredentials
         ACCOUNT_NAME = 'dummy_service_account_name'
         PRIVATE_KEY_TEXT = b'dummy_private_key_text'
         STRING_TO_SIGN = u'dummy_signature'
-        CREDENTIALS = SignedJwtAssertionCredentials(
-            ACCOUNT_NAME, PRIVATE_KEY_TEXT, [])
+        SIGNER = object()
+        CREDENTIALS = ServiceAccountCredentials(
+            ACCOUNT_NAME, SIGNER)
+        CREDENTIALS._private_key_pkcs12 = PRIVATE_KEY_TEXT
+        CREDENTIALS._private_key_password = 'notasecret'
         self._run_with_fake_crypto(CREDENTIALS, PRIVATE_KEY_TEXT,
                                    STRING_TO_SIGN)
 
@@ -321,21 +310,16 @@ def test_json_type(self):
 
         PRIVATE_KEY_TEXT = 'dummy_private_key_pkcs8_text'
         STRING_TO_SIGN = b'dummy_signature'
-
-        def _get_private_key(private_key_pkcs8_text):
-            return private_key_pkcs8_text
-
-        with _Monkey(service_account, _get_private_key=_get_private_key):
-            CREDENTIALS = service_account._ServiceAccountCredentials(
-                'dummy_service_account_id', 'dummy_service_account_email',
-                'dummy_private_key_id', PRIVATE_KEY_TEXT, [])
-
+        SIGNER = object()
+        CREDENTIALS = service_account.ServiceAccountCredentials(
+            'dummy_service_account_email', SIGNER)
+        CREDENTIALS._private_key_pkcs8_pem = PRIVATE_KEY_TEXT
         self._run_with_fake_crypto(CREDENTIALS, PRIVATE_KEY_TEXT,
                                    STRING_TO_SIGN)
 
     def test_gae_type(self):
         # Relies on setUp fixing up App Engine imports.
-        from oauth2client.appengine import AppAssertionCredentials
+        from oauth2client.contrib.appengine import AppAssertionCredentials
         from gcloud._testing import _Monkey
         from gcloud import credentials
 
@@ -387,33 +371,20 @@ def test_bad_type(self):
                                         None, None, None)
         self.assertRaises(ValueError, self._callFUT, CREDENTIALS)
 
-    def test_p12_type(self):
-        from oauth2client.client import SignedJwtAssertionCredentials
-        SERVICE_ACCOUNT_NAME = 'SERVICE_ACCOUNT_NAME'
-        CREDENTIALS = SignedJwtAssertionCredentials(SERVICE_ACCOUNT_NAME,
-                                                    b'bogus_key', [])
-        found = self._callFUT(CREDENTIALS)
-        self.assertEqual(found, SERVICE_ACCOUNT_NAME)
-
-    def test_json_type(self):
+    def test_service_account_type(self):
         from oauth2client import service_account
-        from gcloud._testing import _Monkey
-
-        def _get_private_key(private_key_pkcs8_text):
-            return private_key_pkcs8_text
 
         SERVICE_ACCOUNT_NAME = 'SERVICE_ACCOUNT_NAME'
-        with _Monkey(service_account, _get_private_key=_get_private_key):
-            CREDENTIALS = service_account._ServiceAccountCredentials(
-                'bogus_id', SERVICE_ACCOUNT_NAME, 'bogus_id',
-                'bogus_key_text', [])
+        SIGNER = object()
+        CREDENTIALS = service_account.ServiceAccountCredentials(
+            SERVICE_ACCOUNT_NAME, SIGNER)
 
         found = self._callFUT(CREDENTIALS)
         self.assertEqual(found, SERVICE_ACCOUNT_NAME)
 
     def test_gae_type(self):
         # Relies on setUp fixing up App Engine imports.
-        from oauth2client.appengine import AppAssertionCredentials
+        from oauth2client.contrib.appengine import AppAssertionCredentials
         from gcloud._testing import _Monkey
         from gcloud import credentials
 
@@ -483,15 +454,17 @@ def test_bad_argument(self):
         self.assertRaises(TypeError, self._callFUT, None)
 
     def test_signed_jwt_for_p12(self):
-        import base64
-        from oauth2client import client
+        from oauth2client import service_account
         from gcloud._testing import _Monkey
         from gcloud import credentials as MUT
 
-        scopes = []
         PRIVATE_KEY = b'dummy_private_key_text'
-        credentials = client.SignedJwtAssertionCredentials(
-            'dummy_service_account_name', PRIVATE_KEY, scopes)
+        SIGNER = object()
+        credentials = service_account.ServiceAccountCredentials(
+            'dummy_service_account_email', SIGNER)
+        credentials._private_key_pkcs12 = PRIVATE_KEY
+        credentials._private_key_password = password = 'password-nope'
+
         crypt = _Crypt()
         load_result = object()
         openssl_crypto = _OpenSSLCrypto(load_result, None)
@@ -499,9 +472,8 @@ def test_signed_jwt_for_p12(self):
         with _Monkey(MUT, crypt=crypt, crypto=openssl_crypto):
             result = self._callFUT(credentials)
 
-        self.assertEqual(crypt._private_key_text,
-                         base64.b64encode(PRIVATE_KEY))
-        self.assertEqual(crypt._private_key_password, 'notasecret')
+        self.assertEqual(crypt._private_key_text, PRIVATE_KEY)
+        self.assertEqual(crypt._private_key_password, password)
         self.assertEqual(result, load_result)
         self.assertEqual(openssl_crypto._loaded,
                          [(openssl_crypto.FILETYPE_PEM, _Crypt._KEY)])
@@ -515,14 +487,10 @@ def test_service_account_via_json_key(self):
         scopes = []
 
         PRIVATE_TEXT = 'dummy_private_key_pkcs8_text'
-
-        def _get_private_key(private_key_pkcs8_text):
-            return private_key_pkcs8_text
-
-        with _Monkey(service_account, _get_private_key=_get_private_key):
-            credentials = service_account._ServiceAccountCredentials(
-                'dummy_service_account_id', 'dummy_service_account_email',
-                'dummy_private_key_id', PRIVATE_TEXT, scopes)
+        SIGNER = object()
+        credentials = service_account.ServiceAccountCredentials(
+            'dummy_service_account_email', SIGNER, scopes=scopes)
+        credentials._private_key_pkcs8_pem = PRIVATE_TEXT
 
         load_result = object()
         openssl_crypto = _OpenSSLCrypto(load_result, None)
@@ -541,14 +509,11 @@ def test_without_pyopenssl(self):
         from gcloud import credentials as credentials_mod
 
         PRIVATE_TEXT = 'dummy_private_key_pkcs8_text'
+        SIGNER = object()
 
-        def _get_private_key(private_key_pkcs8_text):
-            return private_key_pkcs8_text
-
-        with _Monkey(service_account, _get_private_key=_get_private_key):
-            credentials = service_account._ServiceAccountCredentials(
-                'dummy_service_account_id', 'dummy_service_account_email',
-                'dummy_private_key_id', PRIVATE_TEXT, '')
+        credentials = service_account.ServiceAccountCredentials(
+            'dummy_service_account_email', SIGNER)
+        credentials._private_key_pkcs8_pem = PRIVATE_TEXT
 
         with _Monkey(credentials_mod, crypto=None):
             with self.assertRaises(EnvironmentError):
@@ -648,6 +613,7 @@ def create_scoped(self, scopes):
 
 
 class _Client(object):
+
     def __init__(self):
         self._signed = _Credentials()
 
@@ -659,10 +625,6 @@ def get_application_default():
 
         self.GoogleCredentials = GoogleCredentials
 
-    def SignedJwtAssertionCredentials(self, **kw):
-        self._called_with = kw
-        return self._signed
-
 
 class _Crypt(object):
 
@@ -724,3 +686,14 @@ def non_transactional(*args, **kwargs):
         def do_nothing_wrapper(func):
             return func
         return do_nothing_wrapper
+
+
+class _MockServiceAccountCredentials(object):
+
+    def __init__(self):
+        self.p12_called = []
+        self._result = _Credentials()
+
+    def from_p12_keyfile(self, email, path, scopes=None):
+        self.p12_called.append((email, path, scopes))
+        return self._result