Repository: googleapis/google-cloud-python

Issue:
Bigtable: 'test_create_instance_w_two_clusters' flakes with '504 Deadline Exceeded'
/cc @sduskis, @vikas-jamdar 

From:

- https://circleci.com/gh/GoogleCloudPlatform/google-cloud-python/7977
- https://circleci.com/gh/GoogleCloudPlatform/google-cloud-python/7978

```python
___________ TestInstanceAdminAPI.test_create_instance_w_two_clusters ___________

self = <tests.system.TestInstanceAdminAPI testMethod=test_create_instance_w_two_clusters>

    def test_create_instance_w_two_clusters(self):
        from google.cloud.bigtable import enums
        from google.cloud.bigtable.table import ClusterState
        _PRODUCTION = enums.Instance.Type.PRODUCTION
        ALT_INSTANCE_ID = 'dif' + unique_resource_id('-')
        instance = Config.CLIENT.instance(ALT_INSTANCE_ID,
                                          instance_type=_PRODUCTION,
                                          labels=LABELS)
    
        ALT_CLUSTER_ID_1 = ALT_INSTANCE_ID + '-c1'
        ALT_CLUSTER_ID_2 = ALT_INSTANCE_ID + '-c2'
        LOCATION_ID_2 = 'us-central1-f'
        STORAGE_TYPE = enums.StorageType.HDD
        cluster_1 = instance.cluster(
            ALT_CLUSTER_ID_1, location_id=LOCATION_ID, serve_nodes=SERVE_NODES,
            default_storage_type=STORAGE_TYPE)
        cluster_2 = instance.cluster(
            ALT_CLUSTER_ID_2, location_id=LOCATION_ID_2,
            serve_nodes=SERVE_NODES, default_storage_type=STORAGE_TYPE)
        operation = instance.create(clusters=[cluster_1, cluster_2])
        # We want to make sure the operation completes.
        operation.result(timeout=10)
    
        # Make sure this instance gets deleted after the test case.
        self.instances_to_delete.append(instance)
    
        # Create a new instance instance and make sure it is the same.
        instance_alt = Config.CLIENT.instance(ALT_INSTANCE_ID)
        instance_alt.reload()
    
        self.assertEqual(instance, instance_alt)
        self.assertEqual(instance.display_name, instance_alt.display_name)
        self.assertEqual(instance.type_, instance_alt.type_)
    
        clusters, failed_locations = instance_alt.list_clusters()
        self.assertEqual(failed_locations, [])
    
        clusters.sort(key=lambda x: x.name)
        alt_cluster_1, alt_cluster_2 = clusters
    
        self.assertEqual(cluster_1.location_id, alt_cluster_1.location_id)
        self.assertEqual(alt_cluster_1.state, enums.Cluster.State.READY)
        self.assertEqual(cluster_1.serve_nodes, alt_cluster_1.serve_nodes)
        self.assertEqual(cluster_1.default_storage_type,
                         alt_cluster_1.default_storage_type)
        self.assertEqual(cluster_2.location_id, alt_cluster_2.location_id)
        self.assertEqual(alt_cluster_2.state, enums.Cluster.State.READY)
        self.assertEqual(cluster_2.serve_nodes, alt_cluster_2.serve_nodes)
        self.assertEqual(cluster_2.default_storage_type,
                         alt_cluster_2.default_storage_type)
    
        # Test list clusters in project via 'client.list_clusters'
        clusters, failed_locations = Config.CLIENT.list_clusters()
        self.assertFalse(failed_locations)
        found = set([cluster.name for cluster in clusters])
        self.assertTrue({alt_cluster_1.name,
                         alt_cluster_2.name,
                         Config.CLUSTER.name}.issubset(found))
    
        temp_table_id = 'test-get-cluster-states'
        temp_table = instance.table(temp_table_id)
>       temp_table.create()

tests/system.py:280: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
google/cloud/bigtable/table.py:218: in create
    table=table, initial_splits=splits)
google/cloud/bigtable_admin_v2/gapic/bigtable_table_admin_client.py:327: in create_table
    request, retry=retry, timeout=timeout, metadata=metadata)
../api_core/google/api_core/gapic_v1/method.py:139: in __call__
    return wrapped_func(*args, **kwargs)
../api_core/google/api_core/retry.py:260: in retry_wrapped_func
    on_error=on_error,
../api_core/google/api_core/retry.py:177: in retry_target
    return target()
../api_core/google/api_core/timeout.py:206: in func_with_timeout
    return func(*args, **kwargs)
../api_core/google/api_core/grpc_helpers.py:61: in error_remapped_callable
    six.raise_from(exceptions.from_grpc_error(exc), exc)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

value = DeadlineExceeded('Deadline Exceeded',)
from_value = <_Rendezvous of RPC that terminated with:
	status = StatusCode.DEADLINE_EXCEED...all.cc","file_line":1099,"grpc_message":"Deadline Exceeded","grpc_status":4}"
>

    def raise_from(value, from_value):
>       raise value
E       DeadlineExceeded: 504 Deadline Exceeded

../.nox/sys-2-7/lib/python2.7/site-packages/six.py:737: DeadlineExceeded
```


Discussion:
We probably need to increase the time out from the 130 seconds it is now, to more like 15 minutes.  Create table can take longer than 2 minutes, depending on conditions.
@sduskis The deadline in [bigtable_table_admin_client_config.py](https://github.com/GoogleCloudPlatform/google-cloud-python/blob/64caa5f0a319313ccc5dbe515ae91c4029548447/bigtable/google/cloud/bigtable_admin_v2/gapic/bigtable_table_admin_client_config.py#L39) is set via autosynth.  We would need to get the upstream configuration fixed, rather than changing here manually.
Understood.  I am in progress in making those upstream changes now.
Another two failures today:

- https://circleci.com/gh/GoogleCloudPlatform/google-cloud-python/8187
- https://circleci.com/gh/GoogleCloudPlatform/google-cloud-python/8188
FYI, my internal changes touched more than just `CreateTable`.  I wanted to fix other RPCs as well.  Hopefully, we'll get a fix out this week.
@tseaver, I got the changes out.  See this [googleapis commit](https://github.com/googleapis/googleapis/commit/59036b030f4a02b1eef2425d76064992b50c4d9e#diff-05c73320211201124119d8c88eb6d5d2) for details.

A synth regen should hopefully fix this flake.
Hmmm, just saw the same failure yesterday (all for 2.7 and 3.7 systests):

- https://circleci.com/gh/GoogleCloudPlatform/google-cloud-python/8459
- https://circleci.com/gh/GoogleCloudPlatform/google-cloud-python/8463
- https://circleci.com/gh/GoogleCloudPlatform/google-cloud-python/8465
- https://circleci.com/gh/GoogleCloudPlatform/google-cloud-python/8461
Another failure [today](https://source.cloud.google.com/results/invocations/4bd4d22d-4d25-4ea3-963d-2f97dc170566/targets/cloud-devrel%2Fclient-libraries%2Fgoogle-cloud-python%2Fpresubmit%2Fbigtable/log).
@sduskis I just saw the `temp_table.create()` [fail again with a 504](https://source.cloud.google.com/results/invocations/683d6ae1-e434-4634-9952-a79cafbfd947/targets/cloud-devrel%2Fclient-libraries%2Fgoogle-cloud-python%2Fpresubmit%2Fbigtable/log).  It looks to me like the `googleapis` commit you linked did *not* up the [timeout for `CreateInstance`](https://github.com/googleapis/googleapis/commit/59036b030f4a02b1eef2425d76064992b50c4d9e#diff-05c73320211201124119d8c88eb6d5d2L77).

Resolving patch:
diff --git a/bigtable/docs/snippets.py b/bigtable/docs/snippets.py
--- a/bigtable/docs/snippets.py
+++ b/bigtable/docs/snippets.py
@@ -415,7 +415,6 @@ def test_bigtable_delete_instance():
     )
     operation = instance.create(clusters=[cluster])
 
-
     # We want to make sure the operation completes.
     operation.result(timeout=100)
 
@@ -501,6 +500,7 @@ def test_bigtable_table_data_client():
     table_data_client = client.table_data_client
     # [END bigtable_table_data_client]
 
+
 def test_bigtable_table_admin_client():
     # [START bigtable_table_admin_client]
     from google.cloud.bigtable import Client


Test patch:
diff --git a/bigtable/tests/system.py b/bigtable/tests/system.py
--- a/bigtable/tests/system.py
+++ b/bigtable/tests/system.py
@@ -15,10 +15,15 @@
 import datetime
 import operator
 import os
-
 import unittest
 
 from google.api_core.exceptions import TooManyRequests
+from google.cloud.environment_vars import BIGTABLE_EMULATOR
+from test_utils.retry import RetryErrors
+from test_utils.retry import RetryResult
+from test_utils.system import EmulatorCreds
+from test_utils.system import unique_resource_id
+
 from google.cloud._helpers import _datetime_from_microseconds
 from google.cloud._helpers import _microseconds_from_datetime
 from google.cloud._helpers import UTC
@@ -30,14 +35,11 @@
 from google.cloud.bigtable.row_filters import RowFilterUnion
 from google.cloud.bigtable.row_data import Cell
 from google.cloud.bigtable.row_data import PartialRowData
-from google.cloud.environment_vars import BIGTABLE_EMULATOR
 from google.cloud.bigtable.row_set import RowSet
 from google.cloud.bigtable.row_set import RowRange
-
-from test_utils.retry import RetryErrors
-from test_utils.retry import RetryResult
-from test_utils.system import EmulatorCreds
-from test_utils.system import unique_resource_id
+from google.cloud.bigtable_admin_v2.gapic import (
+    bigtable_table_admin_client_config as table_admin_config,
+)
 
 UNIQUE_SUFFIX = unique_resource_id("-")
 LOCATION_ID = "us-central1-c"
@@ -97,6 +99,13 @@ def setUpModule():
     from google.cloud.exceptions import GrpcRendezvous
     from google.cloud.bigtable.enums import Instance
 
+    # See: https://github.com/googleapis/google-cloud-python/issues/5928
+    interfaces = table_admin_config.config["interfaces"]
+    iface_config = interfaces["google.bigtable.admin.v2.BigtableTableAdmin"]
+    methods = iface_config["methods"]
+    create_table = methods["CreateTable"]
+    create_table["timeout_millis"] = 90000
+
     Config.IN_EMULATOR = os.getenv(BIGTABLE_EMULATOR) is not None
 
     if Config.IN_EMULATOR: