Repository: googleapis/google-cloud-python

Issue:
Bigtable: 'test_bigtable_create_table' snippet flakes with '504 Deadline Exceeded'.
From [this Kokoro failure](https://source.cloud.google.com/results/invocations/2322354e-a3c8-4e30-8cac-2dbf3a814cad/targets/cloud-devrel%2Fclient-libraries%2Fgoogle-cloud-python%2Fpresubmit%2Fbigtable/log):

```python
__________________________ test_bigtable_create_table __________________________
args = (parent: "projects/precise-truck-742/instances/snippet-tests-1561572579521"
table_id: "table_my"
table {
  column_families {
    key: "cf1"
    value {
      gc_rule {
        max_num_versions: 2
      }
    }
  }
}
,)
kwargs = {'metadata': [('x-goog-request-params', 'parent=projects/precise-truck-742/instances/snippet-tests-1561572579521'), ('x-goog-api-client', 'gl-python/3.7.0b3 grpc/1.21.1 gax/1.13.0 gapic/0.33.0 gccl/0.33.0')], 'timeout': 20.0}
    @six.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
>           return callable_(*args, **kwargs)
../api_core/google/api_core/grpc_helpers.py:57:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7f9df35a5828>
request = parent: "projects/precise-truck-742/instances/snippet-tests-1561572579521"
table_id: "table_my"
table {
  column_families {
    key: "cf1"
    value {
      gc_rule {
        max_num_versions: 2
      }
    }
  }
}
timeout = 20.0
metadata = [('x-goog-request-params', 'parent=projects/precise-truck-742/instances/snippet-tests-1561572579521'), ('x-goog-api-client', 'gl-python/3.7.0b3 grpc/1.21.1 gax/1.13.0 gapic/0.33.0 gccl/0.33.0')]
credentials = None, wait_for_ready = None, compression = None
    def __call__(self,
                 request,
                 timeout=None,
                 metadata=None,
                 credentials=None,
                 wait_for_ready=None,
                 compression=None):
        state, call, = self._blocking(request, timeout, metadata, credentials,
                                      wait_for_ready, compression)
>       return _end_unary_response_blocking(state, call, False, None)
.nox/snippets-3-7/lib/python3.7/site-packages/grpc/_channel.py:565:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
state = <grpc._channel._RPCState object at 0x7f9df35c54e0>
call = <grpc._cython.cygrpc.SegregatedCall object at 0x7f9df35b1e88>
with_call = False, deadline = None
    def _end_unary_response_blocking(state, call, with_call, deadline):
        if state.code is grpc.StatusCode.OK:
            if with_call:
                rendezvous = _Rendezvous(state, call, None, deadline)
                return state.response, rendezvous
            else:
                return state.response
        else:
>           raise _Rendezvous(state, None, None, deadline)
E           grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
E           	status = StatusCode.DEADLINE_EXCEEDED
E           	details = "Deadline Exceeded"
E           	debug_error_string = "{"created":"@1561572698.043685871","description":"Error received from peer ipv4:74.125.142.95:443","file":"src/core/lib/surface/call.cc","file_line":1046,"grpc_message":"Deadline Exceeded","grpc_status":4}"
E           >
.nox/snippets-3-7/lib/python3.7/site-packages/grpc/_channel.py:467: _Rendezvous
The above exception was the direct cause of the following exception:
    def test_bigtable_create_table():
        # [START bigtable_create_table]
        from google.cloud.bigtable import Client
        from google.cloud.bigtable import column_family
        client = Client(admin=True)
        instance = client.instance(INSTANCE_ID)
        table = instance.table("table_my")
        # Define the GC policy to retain only the most recent 2 versions.
        max_versions_rule = column_family.MaxVersionsGCRule(2)
>       table.create(column_families={"cf1": max_versions_rule})
docs/snippets.py:341:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
google/cloud/bigtable/table.py:251: in create
    initial_splits=splits,
google/cloud/bigtable_admin_v2/gapic/bigtable_table_admin_client.py:340: in create_table
    request, retry=retry, timeout=timeout, metadata=metadata
../api_core/google/api_core/gapic_v1/method.py:143: in __call__
    return wrapped_func(*args, **kwargs)
../api_core/google/api_core/retry.py:273: in retry_wrapped_func
    on_error=on_error,
../api_core/google/api_core/retry.py:182: in retry_target
    return target()
../api_core/google/api_core/timeout.py:214: in func_with_timeout
    return func(*args, **kwargs)
../api_core/google/api_core/grpc_helpers.py:59: in error_remapped_callable
    six.raise_from(exceptions.from_grpc_error(exc), exc)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
value = None
from_value = <_Rendezvous of RPC that terminated with:
	status = StatusCode.DEADLINE_EXCEEDED
	details = "Deadline Exceeded"
	debug...2.95:443","file":"src/core/lib/surface/call.cc","file_line":1046,"grpc_message":"Deadline Exceeded","grpc_status":4}"
>
>   ???
E google.api_core.exceptions.DeadlineExceeded: 504 Deadline Exceeded
``


Discussion:

Resolving patch:
diff --git a/bigtable/docs/snippets.py b/bigtable/docs/snippets.py
--- a/bigtable/docs/snippets.py
+++ b/bigtable/docs/snippets.py
@@ -395,6 +395,8 @@ def test_bigtable_update_cluster():
 
 def test_bigtable_create_table():
     # [START bigtable_create_table]
+    from google.api_core import exceptions
+    from google.api_core import retry
     from google.cloud.bigtable import Client
     from google.cloud.bigtable import column_family
 
@@ -403,7 +405,13 @@ def test_bigtable_create_table():
     table = instance.table("table_my")
     # Define the GC policy to retain only the most recent 2 versions.
     max_versions_rule = column_family.MaxVersionsGCRule(2)
-    table.create(column_families={"cf1": max_versions_rule})
+
+    # Could include other retriable exception types
+    # Could configure deadline, etc.
+    predicate_504 = retry.if_exception_type(exceptions.DeadlineExceeded)
+    retry_504 = retry.Retry(predicate_504)
+
+    retry_504(table.create)(column_families={"cf1": max_versions_rule})
     # [END bigtable_create_table]
 
     try:


Test patch: