Repository: googleapis/google-cloud-python

Issue:
Stop using GOOGLE_CLOUD_TESTS_PROJECT_ID
Related to #2259.

With the project ID from `GOOGLE_APPLICATION_CREDENTIALS` support added by @daspecster, we don't need to specify the project ID any longer. Though I may be wrong on this, e.g. if some API doesn't use credentials but does use a project ID.



Discussion:
Can you say a little more about what you have in mind and point to the changes by @daspecster that you mentioned?

For a number of reasons, it's important that the automated system tests continue to associate their RPCs with your own GCP project (rather than using a shared project ID built into the auth library).
- It gives you control over your RPC quota limits.
- It lets you monitor your RPC quota usage.
- It gives you control over which APIs are enabled.
- It enables Google on-call engineers to associate the RPCs with this project in case there are issues that need to be investigated.

I might be worried about nothing. With GOOGLE_APPLICATION_CREDENTIALS pointing at credentials for a service account, the project owning that service account will be the issuing project for the RPCs, and all will be well.

@rimey Check out the [implementation](https://github.com/GoogleCloudPlatform/google-cloud-python/blob/1f75bde2fe63420d9687610762920f8e5e6bd65c/google/cloud/_helpers.py#L290). In particular, `_file_project_id()` gets the project ID from the `GOOGLE_APPLICATION_CREDENTIALS` file (which is already the same as the one we set in tests).

---

> rather than using a shared project ID built into the auth library

This doesn't exist (and can't really for most requests).

---

> I might be worried about nothing. With GOOGLE_APPLICATION_CREDENTIALS pointing at credentials for a service account, the project owning that service account will be the issuing project for the RPCs, and all will be well.

Exactly

Thanks. This all sounds good.

> > rather than using a shared project ID built into the auth library
> 
> This doesn't exist (and can't really for most requests).

I should have said built into the Cloud SDK.


Resolving patch:
diff --git a/google/cloud/environment_vars.py b/google/cloud/environment_vars.py
--- a/google/cloud/environment_vars.py
+++ b/google/cloud/environment_vars.py
@@ -21,9 +21,6 @@
 PROJECT = 'GOOGLE_CLOUD_PROJECT'
 """Environment variable defining default project."""
 
-TESTS_PROJECT = 'GOOGLE_CLOUD_TESTS_PROJECT_ID'
-"""Environment variable defining project for tests."""
-
 GCD_DATASET = 'DATASTORE_DATASET'
 """Environment variable defining default dataset ID under GCD."""
 


Test patch:
diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py
--- a/system_tests/bigquery.py
+++ b/system_tests/bigquery.py
@@ -16,8 +16,6 @@
 
 import unittest
 
-from google.cloud import _helpers
-from google.cloud.environment_vars import TESTS_PROJECT
 from google.cloud import bigquery
 from google.cloud.exceptions import Forbidden
 
@@ -52,7 +50,6 @@ class Config(object):
 
 
 def setUpModule():
-    _helpers.PROJECT = TESTS_PROJECT
     Config.CLIENT = bigquery.Client()
 
 
diff --git a/system_tests/bigtable.py b/system_tests/bigtable.py
--- a/system_tests/bigtable.py
+++ b/system_tests/bigtable.py
@@ -17,7 +17,6 @@
 
 import unittest
 
-from google.cloud import _helpers
 from google.cloud._helpers import _datetime_from_microseconds
 from google.cloud._helpers import _microseconds_from_datetime
 from google.cloud._helpers import UTC
@@ -29,7 +28,6 @@
 from google.cloud.bigtable.row_filters import RowFilterUnion
 from google.cloud.bigtable.row_data import Cell
 from google.cloud.bigtable.row_data import PartialRowData
-from google.cloud.environment_vars import TESTS_PROJECT
 
 from retry import RetryErrors
 from retry import RetryResult
@@ -92,7 +90,6 @@ def _retry_on_unavailable(exc):
 
 def setUpModule():
     from grpc._channel import _Rendezvous
-    _helpers.PROJECT = TESTS_PROJECT
     Config.CLIENT = Client(admin=True)
     Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, LOCATION_ID)
     retry = RetryErrors(_Rendezvous, error_predicate=_retry_on_unavailable)
diff --git a/system_tests/clear_datastore.py b/system_tests/clear_datastore.py
--- a/system_tests/clear_datastore.py
+++ b/system_tests/clear_datastore.py
@@ -21,7 +21,6 @@
 import six
 
 from google.cloud import datastore
-from google.cloud.environment_vars import TESTS_PROJECT
 
 
 FETCH_MAX = 20
@@ -90,7 +89,7 @@ def remove_kind(kind, client):
 def remove_all_entities(client=None):
     if client is None:
         # Get a client that uses the test dataset.
-        client = datastore.Client(project=os.getenv(TESTS_PROJECT))
+        client = datastore.Client()
     for kind in ALL_KINDS:
         remove_kind(kind, client)
 
diff --git a/system_tests/datastore.py b/system_tests/datastore.py
--- a/system_tests/datastore.py
+++ b/system_tests/datastore.py
@@ -18,12 +18,10 @@
 
 import httplib2
 
-from google.cloud import _helpers
 from google.cloud._helpers import UTC
 from google.cloud import datastore
 from google.cloud.datastore.helpers import GeoPoint
 from google.cloud.environment_vars import GCD_DATASET
-from google.cloud.environment_vars import TESTS_PROJECT
 from google.cloud.exceptions import Conflict
 
 import clear_datastore
@@ -55,7 +53,6 @@ def setUpModule():
     # Isolated namespace so concurrent test runs don't collide.
     test_namespace = 'ns' + unique_resource_id()
     if emulator_dataset is None:
-        _helpers.PROJECT = TESTS_PROJECT
         Config.CLIENT = datastore.Client(namespace=test_namespace)
     else:
         credentials = EmulatorCreds()
diff --git a/system_tests/language.py b/system_tests/language.py
--- a/system_tests/language.py
+++ b/system_tests/language.py
@@ -14,8 +14,6 @@
 
 import unittest
 
-from google.cloud import _helpers
-from google.cloud.environment_vars import TESTS_PROJECT
 from google.cloud import exceptions
 from google.cloud import language
 from google.cloud import storage
@@ -39,7 +37,6 @@ class Config(object):
 
 
 def setUpModule():
-    _helpers.PROJECT = TESTS_PROJECT
     Config.CLIENT = language.Client()
     # Now create a bucket for GCS stored content.
     storage_client = storage.Client()
diff --git a/system_tests/local_test_setup.sample b/system_tests/local_test_setup.sample
--- a/system_tests/local_test_setup.sample
+++ b/system_tests/local_test_setup.sample
@@ -1,4 +1,3 @@
 export GOOGLE_APPLICATION_CREDENTIALS="app_credentials.json.sample"
-export GOOGLE_CLOUD_TESTS_PROJECT_ID="my-project"
 export GOOGLE_CLOUD_REMOTE_FOR_LINT="upstream"
 export GOOGLE_CLOUD_BRANCH_FOR_LINT="master"
diff --git a/system_tests/logging_.py b/system_tests/logging_.py
--- a/system_tests/logging_.py
+++ b/system_tests/logging_.py
@@ -20,8 +20,6 @@
 from google.cloud.logging.handlers.handlers import CloudLoggingHandler
 from google.cloud.logging.handlers.transports import SyncTransport
 from google.cloud.logging import client
-from google.cloud import _helpers
-from google.cloud.environment_vars import TESTS_PROJECT
 
 from retry import RetryErrors
 from retry import RetryResult
@@ -52,7 +50,6 @@ class Config(object):
 
 
 def setUpModule():
-    _helpers.PROJECT = TESTS_PROJECT
     Config.CLIENT = client.Client()
 
 
diff --git a/system_tests/monitoring.py b/system_tests/monitoring.py
--- a/system_tests/monitoring.py
+++ b/system_tests/monitoring.py
@@ -14,8 +14,6 @@
 
 import unittest
 
-from google.cloud import _helpers
-from google.cloud.environment_vars import TESTS_PROJECT
 from google.cloud.exceptions import InternalServerError
 from google.cloud.exceptions import NotFound
 from google.cloud.exceptions import ServiceUnavailable
@@ -30,10 +28,6 @@
 retry_503 = RetryErrors(ServiceUnavailable)
 
 
-def setUpModule():
-    _helpers.PROJECT = TESTS_PROJECT
-
-
 class TestMonitoring(unittest.TestCase):
 
     def test_fetch_metric_descriptor(self):
diff --git a/system_tests/populate_datastore.py b/system_tests/populate_datastore.py
--- a/system_tests/populate_datastore.py
+++ b/system_tests/populate_datastore.py
@@ -22,7 +22,6 @@
 import six
 
 from google.cloud import datastore
-from google.cloud.environment_vars import TESTS_PROJECT
 
 
 ANCESTOR = ('Book', 'GoT')
@@ -91,7 +90,7 @@ def print_func(message):
 def add_characters(client=None):
     if client is None:
         # Get a client that uses the test dataset.
-        client = datastore.Client(project=os.getenv(TESTS_PROJECT))
+        client = datastore.Client()
     with client.transaction() as xact:
         for key_path, character in six.moves.zip(KEY_PATHS, CHARACTERS):
             if key_path[-1] != character['name']:
diff --git a/system_tests/pubsub.py b/system_tests/pubsub.py
--- a/system_tests/pubsub.py
+++ b/system_tests/pubsub.py
@@ -23,7 +23,6 @@
 # pylint: disable=ungrouped-imports
 from google.cloud import _helpers
 from google.cloud.environment_vars import PUBSUB_EMULATOR
-from google.cloud.environment_vars import TESTS_PROJECT
 from google.cloud.pubsub import client
 # pylint: enable=ungrouped-imports
 
@@ -51,7 +50,6 @@ class Config(object):
 
 
 def setUpModule():
-    _helpers.PROJECT = TESTS_PROJECT
     if os.getenv(PUBSUB_EMULATOR) is None:
         Config.CLIENT = client.Client()
     else:
diff --git a/system_tests/storage.py b/system_tests/storage.py
--- a/system_tests/storage.py
+++ b/system_tests/storage.py
@@ -20,8 +20,6 @@
 import httplib2
 import six
 
-from google.cloud import _helpers
-from google.cloud.environment_vars import TESTS_PROJECT
 from google.cloud import exceptions
 from google.cloud import storage
 from google.cloud.storage._helpers import _base64_md5hash
@@ -33,7 +31,6 @@
 
 retry_429 = RetryErrors(exceptions.TooManyRequests)
 HTTP = httplib2.Http()
-_helpers.PROJECT = TESTS_PROJECT
 
 
 class Config(object):
diff --git a/system_tests/system_test_utils.py b/system_tests/system_test_utils.py
--- a/system_tests/system_test_utils.py
+++ b/system_tests/system_test_utils.py
@@ -18,11 +18,9 @@
 import time
 
 from google.cloud.environment_vars import CREDENTIALS as TEST_CREDENTIALS
-from google.cloud.environment_vars import TESTS_PROJECT
 
 
 # From shell environ. May be None.
-PROJECT_ID = os.getenv(TESTS_PROJECT)
 CREDENTIALS = os.getenv(TEST_CREDENTIALS)
 
 ENVIRON_ERROR_MSG = """\
@@ -44,24 +42,15 @@ def create_scoped_required():
 
 
 def check_environ():
-    missing = []
-    extra = ''
-
-    if PROJECT_ID is None:
-        missing.append(TESTS_PROJECT)
-
+    err_msg = None
     if CREDENTIALS is None:
-        missing.append(TEST_CREDENTIALS)
+        err_msg = '\nMissing variables: ' + TEST_CREDENTIALS
     elif not os.path.isfile(CREDENTIALS):
-        extra = '\nThe %s path %r is not a file.' % (TEST_CREDENTIALS,
-                                                     CREDENTIALS)
+        err_msg = '\nThe %s path %r is not a file.' % (TEST_CREDENTIALS,
+                                                       CREDENTIALS)
 
-    if missing or extra:
-        msg = ENVIRON_ERROR_MSG
-        if missing:
-            msg += '\nMissing variables: ' + ', '.join(missing)
-        if extra:
-            msg += extra
+    if err_msg is not None:
+        msg = ENVIRON_ERROR_MSG + err_msg
         print(msg, file=sys.stderr)
         sys.exit(1)