Repository: googleapis/google-cloud-python

Issue:
`from_service_account_json` throws error
Currently, I'm getting the following error while using gcloud in a python 3.4.3 virtualenv:

```
Traceback (most recent call last):
  File "/usr/lib/python3.4/configparser.py", line 762, in get
    value = d[option]
  File "/var/lib/jenkins/workspace/d-files-with-client-library-J2HHEFIKRSP5J5SH4BYKZ7MKYZEJFGDHIM4IBDJ7MCBN24XYA53A/venv/lib/python3.4/collections/__init__.py", line 814, in __getitem__
    return self.__missing__(key)            # support subclasses that define __missing__
  File "/var/lib/jenkins/workspace/d-files-with-client-library-J2HHEFIKRSP5J5SH4BYKZ7MKYZEJFGDHIM4IBDJ7MCBN24XYA53A/venv/lib/python3.4/collections/__init__.py", line 806, in __missing__
    raise KeyError(key)
KeyError: 'project'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "deploy/deploy.py", line 128, in <module>
    cli()
  File "/var/lib/jenkins/workspace/d-files-with-client-library-J2HHEFIKRSP5J5SH4BYKZ7MKYZEJFGDHIM4IBDJ7MCBN24XYA53A/venv/lib/python3.4/site-packages/click/core.py", line 716, in __call__
    return self.main(*args, **kwargs)
  File "/var/lib/jenkins/workspace/d-files-with-client-library-J2HHEFIKRSP5J5SH4BYKZ7MKYZEJFGDHIM4IBDJ7MCBN24XYA53A/venv/lib/python3.4/site-packages/click/core.py", line 696, in main
    rv = self.invoke(ctx)
  File "/var/lib/jenkins/workspace/d-files-with-client-library-J2HHEFIKRSP5J5SH4BYKZ7MKYZEJFGDHIM4IBDJ7MCBN24XYA53A/venv/lib/python3.4/site-packages/click/core.py", line 1060, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/var/lib/jenkins/workspace/d-files-with-client-library-J2HHEFIKRSP5J5SH4BYKZ7MKYZEJFGDHIM4IBDJ7MCBN24XYA53A/venv/lib/python3.4/site-packages/click/core.py", line 889, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/var/lib/jenkins/workspace/d-files-with-client-library-J2HHEFIKRSP5J5SH4BYKZ7MKYZEJFGDHIM4IBDJ7MCBN24XYA53A/venv/lib/python3.4/site-packages/click/core.py", line 534, in invoke
    return callback(*args, **kwargs)
  File "deploy/deploy.py", line 26, in upload_code_to_CDN_command
    upload_code_to_CDN(branch)
  File "deploy/deploy.py", line 41, in upload_code_to_CDN
    '/path/to/keyfile) <---- This is not a typo, it has been redacted
  File "/var/lib/jenkins/workspace/d-files-with-client-library-J2HHEFIKRSP5J5SH4BYKZ7MKYZEJFGDHIM4IBDJ7MCBN24XYA53A/venv/lib/python3.4/site-packages/google/cloud/client.py", line 61, in from_service_account_json
    return cls(*args, **kwargs)
  File "/var/lib/jenkins/workspace/d-files-with-client-library-J2HHEFIKRSP5J5SH4BYKZ7MKYZEJFGDHIM4IBDJ7MCBN24XYA53A/venv/lib/python3.4/site-packages/google/cloud/storage/client.py", line 53, in __init__
    http=http)
  File "/var/lib/jenkins/workspace/d-files-with-client-library-J2HHEFIKRSP5J5SH4BYKZ7MKYZEJFGDHIM4IBDJ7MCBN24XYA53A/venv/lib/python3.4/site-packages/google/cloud/client.py", line 185, in __init__
    _ClientProjectMixin.__init__(self, project=project)
  File "/var/lib/jenkins/workspace/d-files-with-client-library-J2HHEFIKRSP5J5SH4BYKZ7MKYZEJFGDHIM4IBDJ7MCBN24XYA53A/venv/lib/python3.4/site-packages/google/cloud/client.py", line 141, in __init__
    project = self._determine_default(project)
  File "/var/lib/jenkins/workspace/d-files-with-client-library-J2HHEFIKRSP5J5SH4BYKZ7MKYZEJFGDHIM4IBDJ7MCBN24XYA53A/venv/lib/python3.4/site-packages/google/cloud/client.py", line 154, in _determine_default
    return _determine_default_project(project)
  File "/var/lib/jenkins/workspace/d-files-with-client-library-J2HHEFIKRSP5J5SH4BYKZ7MKYZEJFGDHIM4IBDJ7MCBN24XYA53A/venv/lib/python3.4/site-packages/google/cloud/_helpers.py", line 312, in _determine_default_project
    project = _default_service_project_id()
  File "/var/lib/jenkins/workspace/d-files-with-client-library-J2HHEFIKRSP5J5SH4BYKZ7MKYZEJFGDHIM4IBDJ7MCBN24XYA53A/venv/lib/python3.4/site-packages/google/cloud/_helpers.py", line 245, in _default_service_project_id
    return config.get(_GCLOUD_CONFIG_SECTION, _GCLOUD_CONFIG_KEY)
  File "/usr/lib/python3.4/configparser.py", line 765, in get
    raise NoOptionError(option, section)
```



Discussion:
The .json file DOES exist, and the user does have access to it.

Thanks for reporting!

@csaintc, it looks like the key file is missing the project ID somehow?
If you look at the file, can you confirm that the project ID is in your .json file?
The actual key in the JSON file is `project_id`.

yeah, I have the `project_id` in my keyfile. It's fresh from gcloud

@daspecster @csaintc older key files may be missing the project id, so it's possible. If it is, you can just generate a new key file for the same service account from the API / Cloud Console.

I'll go a head and re-generate my keyfile for a sanity check

Went ahead and did so, same error.

@csaintc, is this on google app engine?

@daspecster No, it's on a Google Compute Engine instance running Jenkins. This is a service account keyfile.json that I upload to the box in order to authenticate with gcloud, so i can upload my compiled front-end code to a CDN.

@csaintc, have you exported the `GOOGLE_APPLICATION_CREDENTIALS` environment variable?

`$ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json`

It appears to not have found that, and then is continuing on to use the `gcloud cli` configuration project which also doesn't exist. I think we should probably throw a better error for this.


Resolving patch:
diff --git a/core/google/cloud/_helpers.py b/core/google/cloud/_helpers.py
--- a/core/google/cloud/_helpers.py
+++ b/core/google/cloud/_helpers.py
@@ -240,7 +240,10 @@ def _default_service_project_id():
     config.read(search_paths)
 
     if config.has_section(_GCLOUD_CONFIG_SECTION):
-        return config.get(_GCLOUD_CONFIG_SECTION, _GCLOUD_CONFIG_KEY)
+        try:
+            return config.get(_GCLOUD_CONFIG_SECTION, _GCLOUD_CONFIG_KEY)
+        except configparser.NoOptionError:
+            return None
 
 
 def _compute_engine_id():


Test patch:
diff --git a/core/unit_tests/test__helpers.py b/core/unit_tests/test__helpers.py
--- a/core/unit_tests/test__helpers.py
+++ b/core/unit_tests/test__helpers.py
@@ -254,6 +254,26 @@ def mock_get_path():
 
             self.assertEqual(result, project_id)
 
+    def test_nix_missing_prject_key(self):
+        from google.cloud import _helpers as MUT
+        from google.cloud._testing import _Monkey
+        from google.cloud._testing import _NamedTemporaryFile
+
+        with _NamedTemporaryFile() as temp:
+            config_value = '[%s]' % (MUT._GCLOUD_CONFIG_SECTION,)
+            with open(temp.name, 'w') as config_file:
+                config_file.write(config_value)
+
+            def mock_get_path():
+                return temp.name
+
+            with _Monkey(os, name='not-nt'):
+                with _Monkey(MUT, _get_nix_config_path=mock_get_path,
+                             _USER_ROOT='not-None'):
+                    result = self._callFUT()
+
+            self.assertEqual(result, None)
+
     def test_windows(self):
         from google.cloud import _helpers as MUT
         from google.cloud._testing import _Monkey
@@ -694,7 +714,6 @@ def test_w_utc_datetime(self):
 
     def test_w_non_utc_datetime(self):
         import datetime
-        from google.cloud._helpers import _UTC
 
         zone = self._make_timezone(offset=datetime.timedelta(hours=-1))
         TIMESTAMP = datetime.datetime(2016, 4, 5, 13, 30, 0, tzinfo=zone)
@@ -703,7 +722,6 @@ def test_w_non_utc_datetime(self):
 
     def test_w_non_utc_datetime_and_ignore_zone(self):
         import datetime
-        from google.cloud._helpers import _UTC
 
         zone = self._make_timezone(offset=datetime.timedelta(hours=-1))
         TIMESTAMP = datetime.datetime(2016, 4, 5, 13, 30, 0, tzinfo=zone)