Repository: googleapis/google-cloud-python

Issue:
pubsub frequently raises a gax DEADLINE_EXCEEDED error
Debian Jessie, Python 2.7 and 3.4

Stacktrace:
```python
tests/test_storage.py:44: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.tox/py34/lib/python3.4/site-packages/werkzeug/test.py:788: in post
    return self.open(*args, **kw)
.tox/py34/lib/python3.4/site-packages/flask/testing.py:113: in open
    follow_redirects=follow_redirects)
.tox/py34/lib/python3.4/site-packages/werkzeug/test.py:751: in open
    response = self.run_wsgi_app(environ, buffered=buffered)
.tox/py34/lib/python3.4/site-packages/werkzeug/test.py:668: in run_wsgi_app
    rv = run_wsgi_app(self.application, environ, buffered=buffered)
.tox/py34/lib/python3.4/site-packages/werkzeug/test.py:871: in run_wsgi_app
    app_rv = app(environ, start_response)
.tox/py34/lib/python3.4/site-packages/flask/app.py:2000: in __call__
    return self.wsgi_app(environ, start_response)
.tox/py34/lib/python3.4/site-packages/flask/app.py:1991: in wsgi_app
    response = self.make_response(self.handle_exception(e))
.tox/py34/lib/python3.4/site-packages/flask/app.py:1567: in handle_exception
    reraise(exc_type, exc_value, tb)
.tox/py34/lib/python3.4/site-packages/flask/_compat.py:33: in reraise
    raise value
.tox/py34/lib/python3.4/site-packages/flask/app.py:1988: in wsgi_app
    response = self.full_dispatch_request()
.tox/py34/lib/python3.4/site-packages/flask/app.py:1641: in full_dispatch_request
    rv = self.handle_user_exception(e)
.tox/py34/lib/python3.4/site-packages/flask/app.py:1544: in handle_user_exception
    reraise(exc_type, exc_value, tb)
.tox/py34/lib/python3.4/site-packages/flask/_compat.py:33: in reraise
    raise value
.tox/py34/lib/python3.4/site-packages/flask/app.py:1639: in full_dispatch_request
    rv = self.dispatch_request()
.tox/py34/lib/python3.4/site-packages/flask/app.py:1625: in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
bookshelf/crud.py:99: in add
    q = tasks.get_books_queue()
bookshelf/tasks.py:34: in get_books_queue
    client, 'books', extra_context=current_app.app_context)
.tox/py34/lib/python3.4/site-packages/psq/queue.py:41: in __init__
    self.topic = self._get_or_create_topic()
.tox/py34/lib/python3.4/site-packages/psq/queue.py:51: in _get_or_create_topic
    if not topic.exists():
.tox/py34/lib/python3.4/site-packages/google/cloud/pubsub/topic.py:179: in exists
    api.topic_get(topic_path=self.full_name)
.tox/py34/lib/python3.4/site-packages/google/cloud/pubsub/_gax.py:126: in topic_get
    topic_pb = self._gax_api.get_topic(topic_path)
.tox/py34/lib/python3.4/site-packages/google/cloud/gapic/pubsub/v1/publisher_client.py:315: in get_topic
    return self._get_topic(request, options)
.tox/py34/lib/python3.4/site-packages/google/gax/api_callable.py:481: in inner
    return api_caller(api_call, this_settings, request)
.tox/py34/lib/python3.4/site-packages/google/gax/api_callable.py:469: in base_caller
    return api_call(*args)
.tox/py34/lib/python3.4/site-packages/google/gax/api_callable.py:434: in inner
    errors.create_error('RPC failed', cause=exception))
.tox/py34/lib/python3.4/site-packages/future/utils/__init__.py:419: in raise_with_traceback
    raise exc.with_traceback(traceback)
.tox/py34/lib/python3.4/site-packages/google/gax/api_callable.py:430: in inner
    return a_func(*args, **kwargs)
.tox/py34/lib/python3.4/site-packages/google/gax/api_callable.py:64: in inner
    return a_func(*updated_args, **kwargs)
.tox/py34/lib/python3.4/site-packages/grpc/_channel.py:481: in __call__
    return _end_unary_response_blocking(state, False, deadline)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

state = <grpc._channel._RPCState object at 0x7fb3c4117c88>, with_call = False
deadline = 1486607618.4266632

    def _end_unary_response_blocking(state, with_call, deadline):
      if state.code is grpc.StatusCode.OK:
        if with_call:
          rendezvous = _Rendezvous(state, None, None, deadline)
          return state.response, rendezvous
        else:
          return state.response
      else:
>       raise _Rendezvous(state, None, None, deadline)
E       google.gax.errors.GaxError: GaxError(RPC failed, caused by <_Rendezvous of RPC that terminated with (StatusCode.DEADLINE_EXCEEDED, Deadline Exceeded)>)

.tox/py34/lib/python3.4/site-packages/grpc/_channel.py:432: GaxError
```

test code is [here](https://github.com/GoogleCloudPlatform/getting-started-python/blob/623f4fd00810925c1babf279d3d6f8e4d33bf696/6-pubsub/tests/test_storage.py)


Discussion:
similar to #2983 

Resolving patch:
diff --git a/pubsub/google/cloud/pubsub/_gax.py b/pubsub/google/cloud/pubsub/_gax.py
--- a/pubsub/google/cloud/pubsub/_gax.py
+++ b/pubsub/google/cloud/pubsub/_gax.py
@@ -148,7 +148,7 @@ def topic_delete(self, topic_path):
                 raise NotFound(topic_path)
             raise
 
-    def topic_publish(self, topic_path, messages):
+    def topic_publish(self, topic_path, messages, timeout=30):
         """API call:  publish one or more messages to a topic
 
         See:
@@ -161,12 +161,15 @@ def topic_publish(self, topic_path, messages):
         :type messages: list of dict
         :param messages: messages to be published.
 
+        :type timeout: int
+        :param timeout: (Optional) Timeout seconds.
+
         :rtype: list of string
         :returns: list of opaque IDs for published messages.
         :raises: :exc:`google.cloud.exceptions.NotFound` if the topic does not
                     exist
         """
-        options = CallOptions(is_bundling=False)
+        options = CallOptions(is_bundling=False, timeout=timeout)
         message_pbs = [_message_pb_from_mapping(message)
                        for message in messages]
         try:


Test patch:
diff --git a/pubsub/unit_tests/test__gax.py b/pubsub/unit_tests/test__gax.py
--- a/pubsub/unit_tests/test__gax.py
+++ b/pubsub/unit_tests/test__gax.py
@@ -274,12 +274,13 @@ def test_topic_publish_miss_w_attrs_w_bytes_payload(self):
         PAYLOAD = b'This is the message text'
         B64 = base64.b64encode(PAYLOAD)
         MESSAGE = {'data': B64, 'attributes': {'foo': 'bar'}}
+        timeout = 120  # 120 seconds or 2 minutes
         gax_api = _GAXPublisherAPI()
         client = _Client(self.PROJECT)
         api = self._make_one(gax_api, client)
 
         with self.assertRaises(NotFound):
-            api.topic_publish(self.TOPIC_PATH, [MESSAGE])
+            api.topic_publish(self.TOPIC_PATH, [MESSAGE], timeout=timeout)
 
         topic_path, message_pbs, options = gax_api._publish_called_with
         self.assertEqual(topic_path, self.TOPIC_PATH)
@@ -287,6 +288,7 @@ def test_topic_publish_miss_w_attrs_w_bytes_payload(self):
         self.assertEqual(message_pb.data, B64)
         self.assertEqual(message_pb.attributes, {'foo': 'bar'})
         self.assertEqual(options.is_bundling, False)
+        self.assertEqual(options.timeout, timeout)
 
     def test_topic_publish_error(self):
         import base64