Repository: googleapis/google-cloud-python

Issue:
BigQuery: 'test_extract_table' flakes with '429 Too Many Request' in bucket creation
From: https://circleci.com/gh/GoogleCloudPlatform/google-cloud-python/8007

```python
_______________________ TestBigQuery.test_extract_table ________________________

self = <tests.system.TestBigQuery testMethod=test_extract_table>

    def test_extract_table(self):
        from google.cloud.storage import Client as StorageClient
    
        storage_client = StorageClient()
        local_id = unique_resource_id()
        bucket_name = 'bq_extract_test' + local_id
        blob_name = 'person_ages.csv'
        dataset_id = _make_dataset_id('load_gcs_then_extract')
        table_id = 'test_table'
        table_ref = Config.CLIENT.dataset(dataset_id).table(table_id)
        table = Table(table_ref)
        self.to_delete.insert(0, table)
        self._load_table_for_extract_table(
>           storage_client, ROWS, bucket_name, blob_name, table_ref)

tests/system.py:808: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/system.py:768: in _load_table_for_extract_table
    bucket = storage_client.create_bucket(bucket_name)
../storage/google/cloud/storage/client.py:285: in create_bucket
    bucket.create(client=self, project=project)
../storage/google/cloud/storage/bucket.py:480: in create
    data=properties, _target_object=self)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <google.cloud.storage._http.Connection object at 0x7f16b74e6470>
method = 'POST', path = '/b', query_params = {'project': 'precise-truck-742'}
data = '{"name": "bq_extract_test_8007_1536697977"}'
content_type = 'application/json', headers = None, api_base_url = None
api_version = None, expect_json = True
_target_object = <Bucket: bq_extract_test_8007_1536697977>

    def api_request(self, method, path, query_params=None,
                    data=None, content_type=None, headers=None,
                    api_base_url=None, api_version=None,
                    expect_json=True, _target_object=None):
        """Make a request over the HTTP transport to the API.
    
            You shouldn't need to use this method, but if you plan to
            interact with the API using these primitives, this is the
            correct one to use.
    
            :type method: str
            :param method: The HTTP method name (ie, ``GET``, ``POST``, etc).
                           Required.
    
            :type path: str
            :param path: The path to the resource (ie, ``'/b/bucket-name'``).
                         Required.
    
            :type query_params: dict or list
            :param query_params: A dictionary of keys and values (or list of
                                 key-value pairs) to insert into the query
                                 string of the URL.
    
            :type data: str
            :param data: The data to send as the body of the request. Default is
                         the empty string.
    
            :type content_type: str
            :param content_type: The proper MIME type of the data provided. Default
                                 is None.
    
            :type headers: dict
            :param headers: extra HTTP headers to be sent with the request.
    
            :type api_base_url: str
            :param api_base_url: The base URL for the API endpoint.
                                 Typically you won't have to provide this.
                                 Default is the standard API base URL.
    
            :type api_version: str
            :param api_version: The version of the API to call.  Typically
                                you shouldn't provide this and instead use
                                the default for the library.  Default is the
                                latest API version supported by
                                google-cloud-python.
    
            :type expect_json: bool
            :param expect_json: If True, this method will try to parse the
                                response as JSON and raise an exception if
                                that cannot be done.  Default is True.
    
            :type _target_object: :class:`object`
            :param _target_object:
                (Optional) Protected argument to be used by library callers. This
                can allow custom behavior, for example, to defer an HTTP request
                and complete initialization of the object at a later time.
    
            :raises ~google.cloud.exceptions.GoogleCloudError: if the response code
                is not 200 OK.
            :raises ValueError: if the response content type is not JSON.
            :rtype: dict or str
            :returns: The API response payload, either as a raw string or
                      a dictionary if the response is valid JSON.
            """
        url = self.build_api_url(path=path, query_params=query_params,
                                 api_base_url=api_base_url,
                                 api_version=api_version)
    
        # Making the executive decision that any dictionary
        # data will be sent properly as JSON.
        if data and isinstance(data, dict):
            data = json.dumps(data)
            content_type = 'application/json'
    
        response = self._make_request(
            method=method, url=url, data=data, content_type=content_type,
            headers=headers, target_object=_target_object)
    
        if not 200 <= response.status_code < 300:
>           raise exceptions.from_http_response(response)
E           google.api_core.exceptions.TooManyRequests: 429 POST https://www.googleapis.com/storage/v1/b?project=precise-truck-742: The project exceeded the rate limit for creating and deleting buckets.

../core/google/cloud/_http.py:293: TooManyRequests
```

There is also a subsequent 404 in the tear-down, which I think we can ignore.


Discussion:

Resolving patch:
diff --git a/bigquery/docs/snippets.py b/bigquery/docs/snippets.py
--- a/bigquery/docs/snippets.py
+++ b/bigquery/docs/snippets.py
@@ -39,8 +39,11 @@
     pyarrow = None
 
 from google.api_core import datetime_helpers
+from google.api_core.exceptions import InternalServerError
+from google.api_core.exceptions import ServiceUnavailable
 from google.api_core.exceptions import TooManyRequests
 from google.cloud import bigquery
+from google.cloud import storage
 from test_utils.retry import RetryErrors
 
 ORIGINAL_FRIENDLY_NAME = 'Original friendly name'
@@ -68,6 +71,8 @@
 
 
 retry_429 = RetryErrors(TooManyRequests)
+retry_storage_errors = RetryErrors(
+    (TooManyRequests, InternalServerError, ServiceUnavailable))
 
 
 @pytest.fixture(scope='module')
@@ -82,6 +87,8 @@ def to_delete(client):
     for item in doomed:
         if isinstance(item, (bigquery.Dataset, bigquery.DatasetReference)):
             retry_429(client.delete_dataset)(item, delete_contents=True)
+        elif isinstance(item, storage.Bucket):
+            retry_storage_errors(item.delete)()
         else:
             retry_429(item.delete)()
 
@@ -1880,11 +1887,9 @@ def test_copy_table_cmek(client, to_delete):
 
 
 def test_extract_table(client, to_delete):
-    from google.cloud import storage
-
     bucket_name = 'extract_shakespeare_{}'.format(_millis())
     storage_client = storage.Client()
-    bucket = retry_429(storage_client.create_bucket)(bucket_name)
+    bucket = retry_storage_errors(storage_client.create_bucket)(bucket_name)
     to_delete.append(bucket)
 
     # [START bigquery_extract_table]
@@ -1910,18 +1915,16 @@ def test_extract_table(client, to_delete):
         project, dataset_id, table_id, destination_uri))
     # [END bigquery_extract_table]
 
-    blob = bucket.get_blob('shakespeare.csv')
+    blob = retry_storage_errors(bucket.get_blob)('shakespeare.csv')
     assert blob.exists
     assert blob.size > 0
     to_delete.insert(0, blob)
 
 
 def test_extract_table_json(client, to_delete):
-    from google.cloud import storage
-
     bucket_name = 'extract_shakespeare_json_{}'.format(_millis())
     storage_client = storage.Client()
-    bucket = retry_429(storage_client.create_bucket)(bucket_name)
+    bucket = retry_storage_errors(storage_client.create_bucket)(bucket_name)
     to_delete.append(bucket)
 
     # [START bigquery_extract_table_json]
@@ -1945,18 +1948,16 @@ def test_extract_table_json(client, to_delete):
     extract_job.result()  # Waits for job to complete.
     # [END bigquery_extract_table_json]
 
-    blob = bucket.get_blob('shakespeare.json')
+    blob = retry_storage_errors(bucket.get_blob)('shakespeare.json')
     assert blob.exists
     assert blob.size > 0
     to_delete.insert(0, blob)
 
 
 def test_extract_table_compressed(client, to_delete):
-    from google.cloud import storage
-
     bucket_name = 'extract_shakespeare_compress_{}'.format(_millis())
     storage_client = storage.Client()
-    bucket = retry_429(storage_client.create_bucket)(bucket_name)
+    bucket = retry_storage_errors(storage_client.create_bucket)(bucket_name)
     to_delete.append(bucket)
 
     # [START bigquery_extract_table_compressed]
@@ -1979,7 +1980,7 @@ def test_extract_table_compressed(client, to_delete):
     extract_job.result()  # Waits for job to complete.
     # [END bigquery_extract_table_compressed]
 
-    blob = bucket.get_blob('shakespeare.csv.gz')
+    blob = retry_storage_errors(bucket.get_blob)('shakespeare.csv.gz')
     assert blob.exists
     assert blob.size > 0
     to_delete.insert(0, blob)


Test patch: