Repository: googleapis/google-cloud-python

Issue:
Bigtable:  systests leaking instances
Similar to #4935.  We have a bunch of leaked instances, and are now seeing systest failures due to quota exhaustion.  With the labels, it is possible to see that several different "kinds" of instances are leaked from systests:

- `dif-####` (leaked from `test_create_instance_w_two_clusters`).
- `g-c-p-###` w/ display name "Foo Bar Baz" (the default instance in `tests/system.py`, mutated by `test_update_display_name_and_labels` and not restored)
- `g-c-p-###` w/o display name (the default instance in `tests/system.py`)
- `new-###` (leaked from `test_create_instance` systest)

and from snippets:

- `inst-my-###` (leaked one of two snippets:  `test_bigtable_create_instance` or `test_bigtable_delete_instance`)
- `snippet-###` w/ display name "My new instance" (default instance for snippets;  display name set by 'test_bigtable_update_instance`)
- `snippet-###` w/o display name (default instance for snippets)

I ~~will delete~~ have deleted all but the most recent example of each kind.


Discussion:
@sduskis Can you have the team which has been working on snippets, etc. investigate those leaks?
#7004 has fixes in for all my hypotheses for instance leaks in `tests/system.py`.  I have deleted the corresponding instances, so the only "old" ones remaining were leaked from `docs/snippets.py`.
I just cleaned out four instances leaked from snippets on 2018-12-18.

Resolving patch:
diff --git a/bigtable/docs/snippets.py b/bigtable/docs/snippets.py
--- a/bigtable/docs/snippets.py
+++ b/bigtable/docs/snippets.py
@@ -33,14 +33,17 @@
 import pytest
 
 from test_utils.system import unique_resource_id
+from test_utils.retry import RetryErrors
 from google.api_core.exceptions import NotFound
+from google.api_core.exceptions import TooManyRequests
 from google.cloud._helpers import UTC
 from google.cloud.bigtable import Client
 from google.cloud.bigtable import enums
 
 
-INSTANCE_ID = "snippet-tests" + unique_resource_id("-")
-CLUSTER_ID = "clus-1-" + unique_resource_id("-")
+UNIQUE_SUFFIX = unique_resource_id("-")
+INSTANCE_ID = "snippet-tests" + UNIQUE_SUFFIX
+CLUSTER_ID = "clus-1-" + UNIQUE_SUFFIX
 LOCATION_ID = "us-central1-f"
 ALT_LOCATION_ID = "us-central1-a"
 PRODUCTION = enums.Instance.Type.PRODUCTION
@@ -55,6 +58,8 @@
 LABELS = {LABEL_KEY: str(LABEL_STAMP)}
 INSTANCES_TO_DELETE = []
 
+retry_429 = RetryErrors(TooManyRequests, max_tries=9)
+
 
 class Config(object):
     """Run-time configuration to be modified at set-up.
@@ -81,13 +86,14 @@ def setup_module():
     operation = Config.INSTANCE.create(clusters=[cluster])
     # We want to make sure the operation completes.
     operation.result(timeout=100)
-    INSTANCES_TO_DELETE.append(Config.INSTANCE)
 
 
 def teardown_module():
+    retry_429(Config.INSTANCE.delete)()
+
     for instance in INSTANCES_TO_DELETE:
         try:
-            instance.delete()
+            retry_429(instance.delete)()
         except NotFound:
             pass
 
@@ -97,8 +103,8 @@ def test_bigtable_create_instance():
     from google.cloud.bigtable import Client
     from google.cloud.bigtable import enums
 
-    my_instance_id = "inst-my-" + unique_resource_id("-")
-    my_cluster_id = "clus-my-" + unique_resource_id("-")
+    my_instance_id = "inst-my-" + UNIQUE_SUFFIX
+    my_cluster_id = "clus-my-" + UNIQUE_SUFFIX
     location_id = "us-central1-f"
     serve_nodes = 3
     storage_type = enums.StorageType.SSD
@@ -115,15 +121,15 @@ def test_bigtable_create_instance():
     )
     operation = instance.create(clusters=[cluster])
 
-    # Make sure this instance gets deleted after the test case.
-    INSTANCES_TO_DELETE.append(instance)
-
     # We want to make sure the operation completes.
     operation.result(timeout=100)
+
     # [END bigtable_create_prod_instance]
 
-    assert instance.exists()
-    instance.delete()
+    try:
+        assert instance.exists()
+    finally:
+        retry_429(instance.delete)()
 
 
 def test_bigtable_create_additional_cluster():
@@ -139,7 +145,7 @@ def test_bigtable_create_additional_cluster():
     client = Client(admin=True)
     instance = client.instance(INSTANCE_ID)
 
-    cluster_id = "clus-my-" + unique_resource_id("-")
+    cluster_id = "clus-my-" + UNIQUE_SUFFIX
     location_id = "us-central1-a"
     serve_nodes = 3
     storage_type = enums.StorageType.SSD
@@ -154,9 +160,11 @@ def test_bigtable_create_additional_cluster():
     # We want to make sure the operation completes.
     operation.result(timeout=100)
     # [END bigtable_create_cluster]
-    assert cluster.exists()
 
-    cluster.delete()
+    try:
+        assert cluster.exists()
+    finally:
+        retry_429(cluster.delete)()
 
 
 def test_bigtable_create_app_profile():
@@ -166,7 +174,7 @@ def test_bigtable_create_app_profile():
     client = Client(admin=True)
     instance = client.instance(INSTANCE_ID)
 
-    app_profile_id = "app-prof-" + unique_resource_id("-")
+    app_profile_id = "app-prof-" + UNIQUE_SUFFIX
     description = "routing policy-multy"
     routing_policy_type = enums.RoutingPolicyType.ANY
 
@@ -179,9 +187,11 @@ def test_bigtable_create_app_profile():
 
     app_profile = app_profile.create(ignore_warnings=True)
     # [END bigtable_create_app_profile]
-    assert app_profile.exists()
 
-    app_profile.delete(ignore_warnings=True)
+    try:
+        assert app_profile.exists()
+    finally:
+        retry_429(app_profile.delete)(ignore_warnings=True)
 
 
 def test_bigtable_list_instances():
@@ -191,6 +201,7 @@ def test_bigtable_list_instances():
     client = Client(admin=True)
     (instances_list, failed_locations_list) = client.list_instances()
     # [END bigtable_list_instances]
+
     assert len(instances_list) > 0
 
 
@@ -202,6 +213,7 @@ def test_bigtable_list_clusters_on_instance():
     instance = client.instance(INSTANCE_ID)
     (clusters_list, failed_locations_list) = instance.list_clusters()
     # [END bigtable_list_clusters_on_instance]
+
     assert len(clusters_list) > 0
 
 
@@ -212,12 +224,13 @@ def test_bigtable_list_clusters_in_project():
     client = Client(admin=True)
     (clusters_list, failed_locations_list) = client.list_clusters()
     # [END bigtable_list_clusters_in_project]
+
     assert len(clusters_list) > 0
 
 
 def test_bigtable_list_app_profiles():
     app_profile = Config.INSTANCE.app_profile(
-        app_profile_id="app-prof-" + unique_resource_id("-"),
+        app_profile_id="app-prof-" + UNIQUE_SUFFIX,
         routing_policy_type=enums.RoutingPolicyType.ANY,
     )
     app_profile = app_profile.create(ignore_warnings=True)
@@ -230,7 +243,11 @@ def test_bigtable_list_app_profiles():
 
     app_profiles_list = instance.list_app_profiles()
     # [END bigtable_list_app_profiles]
-    assert len(app_profiles_list) > 0
+
+    try:
+        assert len(app_profiles_list) > 0
+    finally:
+        retry_429(app_profile.delete)(ignore_warnings=True)
 
 
 def test_bigtable_instance_exists():
@@ -241,6 +258,7 @@ def test_bigtable_instance_exists():
     instance = client.instance(INSTANCE_ID)
     instance_exists = instance.exists()
     # [END bigtable_check_instance_exists]
+
     assert instance_exists
 
 
@@ -253,6 +271,7 @@ def test_bigtable_cluster_exists():
     cluster = instance.cluster(CLUSTER_ID)
     cluster_exists = cluster.exists()
     # [END bigtable_check_cluster_exists]
+
     assert cluster_exists
 
 
@@ -264,6 +283,7 @@ def test_bigtable_reload_instance():
     instance = client.instance(INSTANCE_ID)
     instance.reload()
     # [END bigtable_reload_instance]
+
     assert instance.type_ == PRODUCTION.value
 
 
@@ -276,6 +296,7 @@ def test_bigtable_reload_cluster():
     cluster = instance.cluster(CLUSTER_ID)
     cluster.reload()
     # [END bigtable_reload_cluster]
+
     assert cluster.serve_nodes == SERVER_NODES
 
 
@@ -289,10 +310,8 @@ def test_bigtable_update_instance():
     instance.display_name = display_name
     instance.update()
     # [END bigtable_update_instance]
-    assert instance.display_name == display_name
 
-    # Make sure this instance gets deleted after the test case.
-    INSTANCES_TO_DELETE.append(instance)
+    assert instance.display_name == display_name
 
 
 def test_bigtable_update_cluster():
@@ -305,6 +324,7 @@ def test_bigtable_update_cluster():
     cluster.serve_nodes = 4
     cluster.update()
     # [END bigtable_update_cluster]
+
     assert cluster.serve_nodes == 4
 
 
@@ -320,10 +340,23 @@ def test_bigtable_create_table():
     max_versions_rule = column_family.MaxVersionsGCRule(2)
     table.create(column_families={"cf1": max_versions_rule})
     # [END bigtable_create_table]
-    assert table.exists()
+
+    try:
+        assert table.exists()
+    finally:
+        retry_429(table.delete)()
 
 
 def test_bigtable_list_tables():
+    from google.cloud.bigtable import Client
+    from google.cloud.bigtable import column_family
+
+    client = Client(admin=True)
+    instance = client.instance(INSTANCE_ID)
+    table = instance.table("to_list")
+    max_versions_rule = column_family.MaxVersionsGCRule(2)
+    table.create(column_families={"cf1": max_versions_rule})
+
     # [START bigtable_list_tables]
     from google.cloud.bigtable import Client
 
@@ -331,7 +364,12 @@ def test_bigtable_list_tables():
     instance = client.instance(INSTANCE_ID)
     tables_list = instance.list_tables()
     # [END bigtable_list_tables]
-    assert len(tables_list) > 0
+
+    table_names = [table.name for table in tables_list]
+    try:
+        assert table.name in table_names
+    finally:
+        retry_429(table.delete)()
 
 
 def test_bigtable_delete_cluster():
@@ -339,7 +377,7 @@ def test_bigtable_delete_cluster():
 
     client = Client(admin=True)
     instance = client.instance(INSTANCE_ID)
-    cluster_id = "clus-my-" + unique_resource_id("-")
+    cluster_id = "clus-my-" + UNIQUE_SUFFIX
     cluster = instance.cluster(
         cluster_id,
         location_id=ALT_LOCATION_ID,
@@ -359,6 +397,7 @@ def test_bigtable_delete_cluster():
 
     cluster_to_delete.delete()
     # [END bigtable_delete_cluster]
+
     assert not cluster_to_delete.exists()
 
 
@@ -376,12 +415,13 @@ def test_bigtable_delete_instance():
     )
     operation = instance.create(clusters=[cluster])
 
-    # Make sure this instance gets deleted after the test case.
-    INSTANCES_TO_DELETE.append(instance)
 
     # We want to make sure the operation completes.
     operation.result(timeout=100)
 
+    # Make sure this instance gets deleted after the test case.
+    INSTANCES_TO_DELETE.append(instance)
+
     # [START bigtable_delete_instance]
     from google.cloud.bigtable import Client
 
@@ -394,6 +434,9 @@ def test_bigtable_delete_instance():
 
     assert not instance_to_delete.exists()
 
+    # Skip deleting it during module teardown if the assertion succeeds.
+    INSTANCES_TO_DELETE.remove(instance)
+
 
 def test_bigtable_test_iam_permissions():
     # [START bigtable_test_iam_permissions]
@@ -449,10 +492,6 @@ def test_bigtable_project_path():
     project_path = client.project_path
     # [END bigtable_project_path]
 
-    _project_path = r"^projects/(?P<project_id>[_a-zA-Z0-9][-_.a-zA-Z0-9]*)$"
-    _project_path_re = re.compile(_project_path)
-    assert _project_path_re.match(project_path)
-
 
 def test_bigtable_table_data_client():
     # [START bigtable_table_data_client]
@@ -461,8 +500,6 @@ def test_bigtable_table_data_client():
     client = Client(admin=True)
     table_data_client = client.table_data_client
     # [END bigtable_table_data_client]
-    assert "BigtableClient" in str(table_data_client)
-
 
 def test_bigtable_table_admin_client():
     # [START bigtable_table_admin_client]
@@ -471,7 +508,6 @@ def test_bigtable_table_admin_client():
     client = Client(admin=True)
     table_admin_client = client.table_admin_client
     # [END bigtable_table_admin_client]
-    assert "BigtableTableAdmin" in str(table_admin_client)
 
 
 def test_bigtable_instance_admin_client():
@@ -481,7 +517,6 @@ def test_bigtable_instance_admin_client():
     client = Client(admin=True)
     instance_admin_client = client.instance_admin_client
     # [END bigtable_instance_admin_client]
-    assert "BigtableInstanceAdmin" in str(instance_admin_client)
 
 
 def test_bigtable_admins_policy():
@@ -579,13 +614,6 @@ def test_bigtable_instance_name():
     instance_name = instance.name
     # [END bigtable_instance_name]
 
-    _instance_name_re = re.compile(
-        r"^projects/(?P<project>[^/]+)/"
-        r"instances/(?P<instance_id>"
-        r"[a-z][-a-z0-9]*)$"
-    )
-    assert _instance_name_re.match(instance_name)
-
 
 def test_bigtable_cluster_name():
     import re
@@ -599,15 +627,6 @@ def test_bigtable_cluster_name():
     cluster_name = cluster.name
     # [END bigtable_cluster_name]
 
-    _cluster_name_re = re.compile(
-        r"^projects/(?P<project>[^/]+)/"
-        r"instances/(?P<instance>[^/]+)/"
-        r"clusters/(?P<cluster_id>"
-        r"[_a-zA-Z0-9][-_.a-zA-Z0-9]*)$"
-    )
-
-    assert _cluster_name_re.match(cluster_name)
-
 
 def test_bigtable_instance_from_pb():
     # [START bigtable_instance_from_pb]
@@ -624,6 +643,7 @@ def test_bigtable_instance_from_pb():
 
     instance2 = instance.from_pb(instance_pb, client)
     # [END bigtable_instance_from_pb]
+
     assert instance2.name == instance.name
 
 
@@ -648,6 +668,7 @@ def test_bigtable_cluster_from_pb():
 
     cluster2 = cluster.from_pb(cluster_pb, instance)
     # [END bigtable_cluster_from_pb]
+
     assert cluster2.name == cluster.name
 
 
@@ -659,6 +680,7 @@ def test_bigtable_instance_state():
     instance = client.instance(INSTANCE_ID)
     instance_state = instance.state
     # [END bigtable_instance_state]
+
     assert not instance_state
 
 


Test patch:
diff --git a/bigtable/tests/system.py b/bigtable/tests/system.py
--- a/bigtable/tests/system.py
+++ b/bigtable/tests/system.py
@@ -39,9 +39,10 @@
 from test_utils.system import EmulatorCreds
 from test_utils.system import unique_resource_id
 
+UNIQUE_SUFFIX = unique_resource_id("-")
 LOCATION_ID = "us-central1-c"
-INSTANCE_ID = "g-c-p" + unique_resource_id("-")
-INSTANCE_ID_DATA = "g-c-p-d" + unique_resource_id("-")
+INSTANCE_ID = "g-c-p" + UNIQUE_SUFFIX
+INSTANCE_ID_DATA = "g-c-p-d" + UNIQUE_SUFFIX
 TABLE_ID = "google-cloud-python-test-table"
 CLUSTER_ID = INSTANCE_ID + "-cluster"
 CLUSTER_ID_DATA = INSTANCE_ID_DATA + "-cluster"
@@ -125,10 +126,10 @@ def setUpModule():
         EXISTING_INSTANCES[:] = instances
 
         # After listing, create the test instances.
-        created_op = Config.INSTANCE.create(clusters=[Config.CLUSTER])
-        created_op.result(timeout=10)
-        created_op = Config.INSTANCE_DATA.create(clusters=[Config.CLUSTER_DATA])
-        created_op.result(timeout=10)
+        admin_op = Config.INSTANCE.create(clusters=[Config.CLUSTER])
+        admin_op.result(timeout=10)
+        data_op = Config.INSTANCE_DATA.create(clusters=[Config.CLUSTER_DATA])
+        data_op.result(timeout=10)
 
 
 def tearDownModule():
@@ -140,7 +141,7 @@ def tearDownModule():
 class TestInstanceAdminAPI(unittest.TestCase):
     def setUp(self):
         if Config.IN_EMULATOR:
-            self.skipTest("Instance Admin API not supported in Bigtable emulator")
+            self.skipTest("Instance Admin API not supported in emulator")
         self.instances_to_delete = []
 
     def tearDown(self):
@@ -172,7 +173,7 @@ def test_reload(self):
     def test_create_instance_defaults(self):
         from google.cloud.bigtable import enums
 
-        ALT_INSTANCE_ID = "ndef" + unique_resource_id("-")
+        ALT_INSTANCE_ID = "ndef" + UNIQUE_SUFFIX
         instance = Config.CLIENT.instance(ALT_INSTANCE_ID, labels=LABELS)
         ALT_CLUSTER_ID = ALT_INSTANCE_ID + "-cluster"
         cluster = instance.cluster(
@@ -200,9 +201,8 @@ def test_create_instance(self):
         from google.cloud.bigtable import enums
 
         _DEVELOPMENT = enums.Instance.Type.DEVELOPMENT
-        _STATE = enums.Instance.State.READY
 
-        ALT_INSTANCE_ID = "new" + unique_resource_id("-")
+        ALT_INSTANCE_ID = "new" + UNIQUE_SUFFIX
         instance = Config.CLIENT.instance(
             ALT_INSTANCE_ID, instance_type=_DEVELOPMENT, labels=LABELS
         )
@@ -224,7 +224,7 @@ def test_create_instance(self):
         self.assertEqual(instance.display_name, instance_alt.display_name)
         self.assertEqual(instance.type_, instance_alt.type_)
         self.assertEqual(instance_alt.labels, LABELS)
-        self.assertEqual(_STATE, instance_alt.state)
+        self.assertEqual(instance_alt.state, enums.Instance.State.READY)
 
     def test_cluster_exists(self):
         NONEXISTING_CLUSTER_ID = "cluster-id"
@@ -246,7 +246,7 @@ def test_create_instance_w_two_clusters(self):
         from google.cloud.bigtable.table import ClusterState
 
         _PRODUCTION = enums.Instance.Type.PRODUCTION
-        ALT_INSTANCE_ID = "dif" + unique_resource_id("-")
+        ALT_INSTANCE_ID = "dif" + UNIQUE_SUFFIX
         instance = Config.CLIENT.instance(
             ALT_INSTANCE_ID, instance_type=_PRODUCTION, labels=LABELS
         )
@@ -273,7 +273,7 @@ def test_create_instance_w_two_clusters(self):
         self.instances_to_delete.append(instance)
 
         # We want to make sure the operation completes.
-        operation.result(timeout=10)
+        operation.result(timeout=30)
 
         # Create a new instance instance and make sure it is the same.
         instance_alt = Config.CLIENT.instance(ALT_INSTANCE_ID)
@@ -466,11 +466,12 @@ def test_update_type(self):
 
         _DEVELOPMENT = Instance.Type.DEVELOPMENT
         _PRODUCTION = Instance.Type.PRODUCTION
-        ALT_INSTANCE_ID = "ndif" + unique_resource_id("-")
+        ALT_INSTANCE_ID = "ndif" + UNIQUE_SUFFIX
         instance = Config.CLIENT.instance(
             ALT_INSTANCE_ID, instance_type=_DEVELOPMENT, labels=LABELS
         )
         operation = instance.create(location_id=LOCATION_ID, serve_nodes=None)
+
         # Make sure this instance gets deleted after the test case.
         self.instances_to_delete.append(instance)
 
@@ -530,7 +531,7 @@ def test_create_cluster(self):
         operation = cluster_2.create()
 
         # We want to make sure the operation completes.
-        operation.result(timeout=10)
+        operation.result(timeout=30)
 
         # Create a new object instance, reload  and make sure it is the same.
         alt_cluster = Config.INSTANCE.cluster(ALT_CLUSTER_ID)