Repository: googleapis/python-bigquery

Issue:
TQDM progress bar causes ReadTimeout error
Using `2.6.1` and enabling the `tqdm` progress bar for some queries, when not using the bigquery-storage option, results in a timeout error:

```
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='bigquery.googleapis.com', port=443): Read timed out. (read timeout=0.5)
```

Maybe related to https://github.com/googleapis/python-bigquery/issues/403?

Disabling the `tqdm` progress bar and running the same query returns a data frame without any problem. 

For reproducible example see previous bug report https://github.com/googleapis/python-bigquery/issues/403 

#### Environment details

  - OS type and version: Debian Buster
  - Python version: 3.8
  - pip version:  20.3.1
  - `google-cloud-bigquery` version: 3.6.1

---
Trace: 
```
    df = query_job.to_dataframe(
  File "/usr/local/lib/python3.8/site-packages/google/cloud/bigquery/job/query.py", line 1329, in to_dataframe
    return query_result.to_dataframe(
  File "/usr/local/lib/python3.8/site-packages/google/cloud/bigquery/table.py", line 1769, in to_dataframe
    record_batch = self.to_arrow(
  File "/usr/local/lib/python3.8/site-packages/google/cloud/bigquery/table.py", line 1601, in to_arrow
    for record_batch in self._to_arrow_iterable(
  File "/usr/local/lib/python3.8/site-packages/google/cloud/bigquery/table.py", line 1500, in _to_page_iterable
    for item in tabledata_list_download():
  File "/usr/local/lib/python3.8/site-packages/google/cloud/bigquery/_pandas_helpers.py", line 512, in download_arrow_row_iterator
    for page in pages:
  File "/usr/local/lib/python3.8/site-packages/google/api_core/page_iterator.py", line 243, in _page_iter
    page = self._next_page()
  File "/usr/local/lib/python3.8/site-packages/google/api_core/page_iterator.py", line 369, in _next_page
    response = self._get_next_page_response()
  File "/usr/local/lib/python3.8/site-packages/google/cloud/bigquery/table.py", line 1474, in _get_next_page_response
    return self.api_request(
  File "/usr/local/lib/python3.8/site-packages/google/cloud/bigquery/client.py", line 637, in _call_api
    return call()
  File "/usr/local/lib/python3.8/site-packages/google/api_core/retry.py", line 281, in retry_wrapped_func
    return retry_target(
  File "/usr/local/lib/python3.8/site-packages/google/api_core/retry.py", line 184, in retry_target
    return target()
  File "/usr/local/lib/python3.8/site-packages/google/cloud/_http.py", line 427, in api_request
    response = self._make_request(
  File "/usr/local/lib/python3.8/site-packages/google/cloud/_http.py", line 291, in _make_request
    return self._do_request(
  File "/usr/local/lib/python3.8/site-packages/google/cloud/_http.py", line 329, in _do_request
    return self.http.request(
  File "/usr/local/lib/python3.8/site-packages/google/auth/transport/requests.py", line 464, in request
    response = super(AuthorizedSession, self).request(
  File "/usr/local/lib/python3.8/site-packages/requests/sessions.py", line 542, in request
    resp = self.send(prep, **send_kwargs)
  File "/usr/local/lib/python3.8/site-packages/requests/sessions.py", line 655, in send
    r = adapter.send(request, **kwargs)
  File "/usr/local/lib/python3.8/site-packages/requests/adapters.py", line 529, in send
    raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='bigquery.googleapis.com', port=443): Read timed out. (read timeout=0.5)
```



Discussion:
Dropping to version `2.3.1` also fixes, whilst still having the progress bar enabled  
I'm not actually seeing any timeouts in the relevant class https://github.com/googleapis/python-bigquery/blob/master/google/cloud/bigquery/table.py#L1346 so it could actually be a problem in the core library.

In my benchmarks `getQueryResults` calls can take 10+ seconds for wide rows, so if there is a default timeout being set, it should be quite high.
@tswast I think it's this https://github.com/googleapis/python-bigquery/blob/079b6a162f6929bf801366d92f8daeb3318426c4/google/cloud/bigquery/_tqdm_helpers.py#L31

Patching as follows fixes in 2.6.1
```
from google.cloud import bigquery

bigquery._tqdm_helpers._PROGRESS_BAR_UPDATE_INTERVAL = 10
```

Resolving patch:
diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py
--- a/google/cloud/bigquery/client.py
+++ b/google/cloud/bigquery/client.py
@@ -93,6 +93,14 @@
 )
 _LIST_ROWS_FROM_QUERY_RESULTS_FIELDS = "jobReference,totalRows,pageToken,rows"
 
+# In microbenchmarks, it's been shown that even in ideal conditions (query
+# finished, local data), requests to getQueryResults can take 10+ seconds.
+# In less-than-ideal situations, the response can take even longer, as it must
+# be able to download a full 100+ MB row in that time. Don't let the
+# connection timeout before data can be downloaded.
+# https://github.com/googleapis/python-bigquery/issues/438
+_MIN_GET_QUERY_RESULTS_TIMEOUT = 120
+
 
 class Project(object):
     """Wrapper for resource describing a BigQuery project.
@@ -1570,7 +1578,9 @@ def _get_query_results(
             location (Optional[str]): Location of the query job.
             timeout (Optional[float]):
                 The number of seconds to wait for the underlying HTTP transport
-                before using ``retry``.
+                before using ``retry``. If set, this connection timeout may be
+                increased to a minimum value. This prevents retries on what
+                would otherwise be a successful response.
 
         Returns:
             google.cloud.bigquery.query._QueryResults:
@@ -1579,6 +1589,9 @@ def _get_query_results(
 
         extra_params = {"maxResults": 0}
 
+        if timeout is not None:
+            timeout = max(timeout, _MIN_GET_QUERY_RESULTS_TIMEOUT)
+
         if project is None:
             project = self.project
 
@@ -3293,7 +3306,9 @@ def _list_rows_from_query_results(
                 How to retry the RPC.
             timeout (Optional[float]):
                 The number of seconds to wait for the underlying HTTP transport
-                before using ``retry``.
+                before using ``retry``. If set, this connection timeout may be
+                increased to a minimum value. This prevents retries on what
+                would otherwise be a successful response.
                 If multiple requests are made under the hood, ``timeout``
                 applies to each individual request.
         Returns:
@@ -3306,6 +3321,9 @@ def _list_rows_from_query_results(
             "location": location,
         }
 
+        if timeout is not None:
+            timeout = max(timeout, _MIN_GET_QUERY_RESULTS_TIMEOUT)
+
         if start_index is not None:
             params["startIndex"] = start_index
 


Test patch:
diff --git a/tests/system.py b/tests/system.py
--- a/tests/system.py
+++ b/tests/system.py
@@ -27,7 +27,6 @@
 import uuid
 import re
 
-import requests
 import psutil
 import pytest
 import pytz
@@ -1798,15 +1797,25 @@ def test_query_w_wrong_config(self):
             Config.CLIENT.query(good_query, job_config=bad_config).result()
 
     def test_query_w_timeout(self):
+        job_config = bigquery.QueryJobConfig()
+        job_config.use_query_cache = False
+
         query_job = Config.CLIENT.query(
             "SELECT * FROM `bigquery-public-data.github_repos.commits`;",
             job_id_prefix="test_query_w_timeout_",
+            location="US",
+            job_config=job_config,
         )
 
         with self.assertRaises(concurrent.futures.TimeoutError):
-            # 1 second is much too short for this query.
             query_job.result(timeout=1)
 
+        # Even though the query takes >1 second, the call to getQueryResults
+        # should succeed.
+        self.assertFalse(query_job.done(timeout=1))
+
+        Config.CLIENT.cancel_job(query_job.job_id, location=query_job.location)
+
     def test_query_w_page_size(self):
         page_size = 45
         query_job = Config.CLIENT.query(
@@ -2408,26 +2417,6 @@ def test_query_iter(self):
         row_tuples = [r.values() for r in query_job]
         self.assertEqual(row_tuples, [(1,)])
 
-    def test_querying_data_w_timeout(self):
-        job_config = bigquery.QueryJobConfig()
-        job_config.use_query_cache = False
-
-        query_job = Config.CLIENT.query(
-            """
-            SELECT COUNT(*)
-            FROM UNNEST(GENERATE_ARRAY(1,1000000)), UNNEST(GENERATE_ARRAY(1, 10000))
-            """,
-            location="US",
-            job_config=job_config,
-        )
-
-        # Specify a very tight deadline to demonstrate that the timeout
-        # actually has effect.
-        with self.assertRaises(requests.exceptions.Timeout):
-            query_job.done(timeout=0.1)
-
-        Config.CLIENT.cancel_job(query_job.job_id, location=query_job.location)
-
     @unittest.skipIf(pandas is None, "Requires `pandas`")
     def test_query_results_to_dataframe(self):
         QUERY = """
diff --git a/tests/unit/job/test_query.py b/tests/unit/job/test_query.py
--- a/tests/unit/job/test_query.py
+++ b/tests/unit/job/test_query.py
@@ -1046,6 +1046,8 @@ def test_result_invokes_begins(self):
         self.assertEqual(reload_request[1]["method"], "GET")
 
     def test_result_w_timeout(self):
+        import google.cloud.bigquery.client
+
         begun_resource = self._make_resource()
         query_resource = {
             "jobComplete": True,
@@ -1072,6 +1074,10 @@ def test_result_w_timeout(self):
             "/projects/{}/queries/{}".format(self.PROJECT, self.JOB_ID),
         )
         self.assertEqual(query_request[1]["query_params"]["timeoutMs"], 900)
+        self.assertEqual(
+            query_request[1]["timeout"],
+            google.cloud.bigquery.client._MIN_GET_QUERY_RESULTS_TIMEOUT,
+        )
         self.assertEqual(reload_request[1]["method"], "GET")
 
     def test_result_w_page_size(self):
diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py
--- a/tests/unit/test_client.py
+++ b/tests/unit/test_client.py
@@ -311,7 +311,7 @@ def test__get_query_results_miss_w_explicit_project_and_timeout(self):
                     project="other-project",
                     location=self.LOCATION,
                     timeout_ms=500,
-                    timeout=42,
+                    timeout=420,
                 )
 
         final_attributes.assert_called_once_with({"path": path}, client, None)
@@ -320,7 +320,32 @@ def test__get_query_results_miss_w_explicit_project_and_timeout(self):
             method="GET",
             path=path,
             query_params={"maxResults": 0, "timeoutMs": 500, "location": self.LOCATION},
-            timeout=42,
+            timeout=420,
+        )
+
+    def test__get_query_results_miss_w_short_timeout(self):
+        import google.cloud.bigquery.client
+        from google.cloud.exceptions import NotFound
+
+        creds = _make_credentials()
+        client = self._make_one(self.PROJECT, creds)
+        conn = client._connection = make_connection()
+        path = "/projects/other-project/queries/nothere"
+        with self.assertRaises(NotFound):
+            client._get_query_results(
+                "nothere",
+                None,
+                project="other-project",
+                location=self.LOCATION,
+                timeout_ms=500,
+                timeout=1,
+            )
+
+        conn.api_request.assert_called_once_with(
+            method="GET",
+            path=path,
+            query_params={"maxResults": 0, "timeoutMs": 500, "location": self.LOCATION},
+            timeout=google.cloud.bigquery.client._MIN_GET_QUERY_RESULTS_TIMEOUT,
         )
 
     def test__get_query_results_miss_w_client_location(self):