Repository: googleapis/google-cloud-python

Issue:
BigQuery: 'test_undelete_table' fails with 500
It has failed for every run today. See [internal fusion link](https://fusion.corp.google.com/projectanalysis/summary/KOKORO/prod%3Acloud-devrel%2Fclient-libraries%2Fgoogle-cloud-python%2Fpresubmit%2Fbigquery?search_pattern=google-cloud-python%2Fpresubmit&include_inactive_projects=false).

I don't see any reported outages for BigQuery at https://status.cloud.google.com/
```
=================================== FAILURES ===================================
_____________________________ test_undelete_table ______________________________

client = <google.cloud.bigquery.client.Client object at 0x7fc1ddc76710>
to_delete = [Dataset(DatasetReference(u'precise-truck-742', u'undelete_table_dataset_1573146523039'))]

    def test_undelete_table(client, to_delete):
        dataset_id = "undelete_table_dataset_{}".format(_millis())
        table_id = "undelete_table_table_{}".format(_millis())
        dataset = bigquery.Dataset(client.dataset(dataset_id))
        dataset.location = "US"
        dataset = client.create_dataset(dataset)
        to_delete.append(dataset)
    
        table = bigquery.Table(dataset.table(table_id), schema=SCHEMA)
        client.create_table(table)
    
        # [START bigquery_undelete_table]
        # TODO(developer): Uncomment the lines below and replace with your values.
        # import time
        # from google.cloud import bigquery
        # client = bigquery.Client()
        # dataset_id = 'my_dataset'  # Replace with your dataset ID.
        # table_id = 'my_table'      # Replace with your table ID.
    
        table_ref = client.dataset(dataset_id).table(table_id)
    
        # TODO(developer): Choose an appropriate snapshot point as epoch
        # milliseconds. For this example, we choose the current time as we're about
        # to delete the table immediately afterwards.
        snapshot_epoch = int(time.time() * 1000)
        # [END bigquery_undelete_table]
    
        # Due to very short lifecycle of the table, ensure we're not picking a time
        # prior to the table creation due to time drift between backend and client.
        table = client.get_table(table_ref)
        created_epoch = datetime_helpers.to_microseconds(table.created)
        if created_epoch > snapshot_epoch:
            snapshot_epoch = created_epoch
    
        # [START bigquery_undelete_table]
    
        # "Accidentally" delete the table.
        client.delete_table(table_ref)  # API request
    
        # Construct the restore-from table ID using a snapshot decorator.
        snapshot_table_id = "{}@{}".format(table_id, snapshot_epoch)
        source_table_ref = client.dataset(dataset_id).table(snapshot_table_id)
    
        # Choose a new table ID for the recovered table data.
        recovered_table_id = "{}_recovered".format(table_id)
        dest_table_ref = client.dataset(dataset_id).table(recovered_table_id)
    
        # Construct and run a copy job.
        job = client.copy_table(
            source_table_ref,
            dest_table_ref,
            # Location must match that of the source and destination tables.
>           location="US",
        )  # API request

docs/snippets.py:1483: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
google/cloud/bigquery/client.py:1975: in copy_table
    copy_job._begin(retry=retry)
google/cloud/bigquery/job.py:631: in _begin
    retry, method="POST", path=path, data=self.to_api_repr()
google/cloud/bigquery/client.py:475: in _call_api
    return call()
../api_core/google/api_core/retry.py:277: in retry_wrapped_func
    on_error=on_error,
../api_core/google/api_core/retry.py:202: in retry_target
    last_exc,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

value = RetryError(u'Deadline of 120.0s exceeded while calling <functools.partial object at 0x7fc1dd52b260>',)
from_value = InternalServerError(u'POST https://bigquery.googleapis.com/bigquery/v2/project...742/jobs: An internal error occurred and the request could not be completed.',)

    def raise_from(value, from_value):
>       raise value
E       RetryError: Deadline of 120.0s exceeded while calling <functools.partial object at 0x7fc1dd52b260>, last exception: 500 POST https://bigquery.googleapis.com/bigquery/v2/projects/precise-truck-742/jobs: An internal error occurred and the request could not be completed.

.nox/snippets-2-7/lib/python2.7/site-packages/six.py:740: RetryError
```


Discussion:
I've filed internal issue 144114317 and am escalating to BigQuery on-call.
I'm able to reproduce when I run the test locally, but not when I try to perform the same actions with the BQ CLI or even with manually pasting similar code into IPython.

```python
import time
from google.cloud import bigquery

client = bigquery.Client()

dataset_id = "swast-scratch.test_dataset_b144114317"
table_id = "swast-scratch.test_dataset_b144114317.b144114317"
client.delete_dataset(dataset_id, delete_contents=True, not_found_ok=True)

dataset = client.create_dataset(dataset_id)
SCHEMA = [
    bigquery.SchemaField("full_name", "STRING"),
    bigquery.SchemaField("age", "INTEGER"),
]
table = bigquery.Table(table_id, schema=SCHEMA)
table = client.create_table(table)

# TODO(developer): Choose an appropriate snapshot point as epoch
# milliseconds. For this example, we choose the current time as we're about
# to delete the table immediately afterwards.
snapshot_epoch = int(time.time() * 1000)

# Due to very short lifecycle of the table, ensure we're not picking a time
# prior to the table creation due to time drift between backend and client.
created_epoch = int(table.created.timestamp() * 1000)
if created_epoch > snapshot_epoch:
    snapshot_epoch = created_epoch

# "Accidentally" delete the table.
client.delete_table(table_id)  # API request

# Construct the restore-from table ID using a snapshot decorator.
snapshot_table_id = "{}@{}".format(table_id, snapshot_epoch)

# Choose a new table ID for the recovered table data.
recovered_table_id = "{}_recovered".format(table_id)

# Construct and run a copy job.
job = client.copy_table(
    snapshot_table_id,
    recovered_table_id,
    # Location must match that of the source and destination tables.
    location="US",
)  # API request

job.result()  # Waits for job to complete.

print(
    "Copied data from deleted table {} to {}".format(table_id, recovered_table_id)
)
```
Oops. Not a backend issue. The difference between my manual code and the test is that the test is trying to use microsecond precision for the snapshot decorator, but only milliseconds are supported.

https://cloud.google.com/bigquery/table-decorators#snapshot_decorators

I'll need to update this code sample. I don't know why this is just started happening, but it's not a backend issue.

Resolving patch:
diff --git a/bigquery/docs/snippets.py b/bigquery/docs/snippets.py
--- a/bigquery/docs/snippets.py
+++ b/bigquery/docs/snippets.py
@@ -40,7 +40,6 @@
 except (ImportError, AttributeError):
     pyarrow = None
 
-from google.api_core import datetime_helpers
 from google.api_core.exceptions import InternalServerError
 from google.api_core.exceptions import ServiceUnavailable
 from google.api_core.exceptions import TooManyRequests
@@ -1428,69 +1427,6 @@ def test_extract_table_compressed(client, to_delete):
     to_delete.insert(0, blob)
 
 
-def test_undelete_table(client, to_delete):
-    dataset_id = "undelete_table_dataset_{}".format(_millis())
-    table_id = "undelete_table_table_{}".format(_millis())
-    dataset = bigquery.Dataset(client.dataset(dataset_id))
-    dataset.location = "US"
-    dataset = client.create_dataset(dataset)
-    to_delete.append(dataset)
-
-    table = bigquery.Table(dataset.table(table_id), schema=SCHEMA)
-    client.create_table(table)
-
-    # [START bigquery_undelete_table]
-    # TODO(developer): Uncomment the lines below and replace with your values.
-    # import time
-    # from google.cloud import bigquery
-    # client = bigquery.Client()
-    # dataset_id = 'my_dataset'  # Replace with your dataset ID.
-    # table_id = 'my_table'      # Replace with your table ID.
-
-    table_ref = client.dataset(dataset_id).table(table_id)
-
-    # TODO(developer): Choose an appropriate snapshot point as epoch
-    # milliseconds. For this example, we choose the current time as we're about
-    # to delete the table immediately afterwards.
-    snapshot_epoch = int(time.time() * 1000)
-    # [END bigquery_undelete_table]
-
-    # Due to very short lifecycle of the table, ensure we're not picking a time
-    # prior to the table creation due to time drift between backend and client.
-    table = client.get_table(table_ref)
-    created_epoch = datetime_helpers.to_microseconds(table.created)
-    if created_epoch > snapshot_epoch:
-        snapshot_epoch = created_epoch
-
-    # [START bigquery_undelete_table]
-
-    # "Accidentally" delete the table.
-    client.delete_table(table_ref)  # API request
-
-    # Construct the restore-from table ID using a snapshot decorator.
-    snapshot_table_id = "{}@{}".format(table_id, snapshot_epoch)
-    source_table_ref = client.dataset(dataset_id).table(snapshot_table_id)
-
-    # Choose a new table ID for the recovered table data.
-    recovered_table_id = "{}_recovered".format(table_id)
-    dest_table_ref = client.dataset(dataset_id).table(recovered_table_id)
-
-    # Construct and run a copy job.
-    job = client.copy_table(
-        source_table_ref,
-        dest_table_ref,
-        # Location must match that of the source and destination tables.
-        location="US",
-    )  # API request
-
-    job.result()  # Waits for job to complete.
-
-    print(
-        "Copied data from deleted table {} to {}".format(table_id, recovered_table_id)
-    )
-    # [END bigquery_undelete_table]
-
-
 def test_client_query_legacy_sql(client):
     """Run a query with Legacy SQL explicitly set"""
     # [START bigquery_query_legacy]
diff --git a/bigquery/samples/undelete_table.py b/bigquery/samples/undelete_table.py
new file mode 100644
--- /dev/null
+++ b/bigquery/samples/undelete_table.py
@@ -0,0 +1,67 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.api_core import datetime_helpers
+
+
+def undelete_table(client, table_id, recovered_table_id):
+    # [START bigquery_undelete_table]
+    import time
+
+    # TODO(developer): Import the client library.
+    # from google.cloud import bigquery
+
+    # TODO(developer): Construct a BigQuery client object.
+    # client = bigquery.Client()
+
+    # TODO(developer): Choose a table to recover.
+    # table_id = "your-project.your_dataset.your_table"
+
+    # TODO(developer): Choose a new table ID for the recovered table data.
+    # recovery_table_id = "your-project.your_dataset.your_table_recovered"
+
+    # TODO(developer): Choose an appropriate snapshot point as epoch
+    # milliseconds. For this example, we choose the current time as we're about
+    # to delete the table immediately afterwards.
+    snapshot_epoch = int(time.time() * 1000)
+
+    # [START_EXCLUDE]
+    # Due to very short lifecycle of the table, ensure we're not picking a time
+    # prior to the table creation due to time drift between backend and client.
+    table = client.get_table(table_id)
+    created_epoch = datetime_helpers.to_milliseconds(table.created)
+    if created_epoch > snapshot_epoch:
+        snapshot_epoch = created_epoch
+    # [END_EXCLUDE]
+
+    # "Accidentally" delete the table.
+    client.delete_table(table_id)  # API request
+
+    # Construct the restore-from table ID using a snapshot decorator.
+    snapshot_table_id = "{}@{}".format(table_id, snapshot_epoch)
+
+    # Construct and run a copy job.
+    job = client.copy_table(
+        snapshot_table_id,
+        recovered_table_id,
+        # Location must match that of the source and destination tables.
+        location="US",
+    )  # API request
+
+    job.result()  # Wait for job to complete.
+
+    print(
+        "Copied data from deleted table {} to {}".format(table_id, recovered_table_id)
+    )
+    # [END bigquery_undelete_table]


Test patch:
diff --git a/bigquery/samples/tests/conftest.py b/bigquery/samples/tests/conftest.py
--- a/bigquery/samples/tests/conftest.py
+++ b/bigquery/samples/tests/conftest.py
@@ -78,6 +78,22 @@ def table_id(client, dataset_id):
     client.delete_table(table, not_found_ok=True)
 
 
+@pytest.fixture
+def table_with_schema_id(client, dataset_id):
+    now = datetime.datetime.now()
+    table_id = "python_table_with_schema_{}_{}".format(
+        now.strftime("%Y%m%d%H%M%S"), uuid.uuid4().hex[:8]
+    )
+    schema = [
+        bigquery.SchemaField("full_name", "STRING"),
+        bigquery.SchemaField("age", "INTEGER"),
+    ]
+    table = bigquery.Table("{}.{}".format(dataset_id, table_id), schema=schema)
+    table = client.create_table(table)
+    yield "{}.{}.{}".format(table.project, table.dataset_id, table.table_id)
+    client.delete_table(table, not_found_ok=True)
+
+
 @pytest.fixture
 def table_with_data_id(client):
     return "bigquery-public-data.samples.shakespeare"
diff --git a/bigquery/samples/tests/test_undelete_table.py b/bigquery/samples/tests/test_undelete_table.py
new file mode 100644
--- /dev/null
+++ b/bigquery/samples/tests/test_undelete_table.py
@@ -0,0 +1,26 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .. import undelete_table
+
+
+def test_undelete_table(capsys, client, table_with_schema_id, random_table_id):
+    undelete_table.undelete_table(client, table_with_schema_id, random_table_id)
+    out, _ = capsys.readouterr()
+    assert (
+        "Copied data from deleted table {} to {}".format(
+            table_with_schema_id, random_table_id
+        )
+        in out
+    )