Repository: googleapis/google-cloud-python

Issue:
BigQuery: Print offending SQL on error with QueryJob.result()
OS: Linux dc32b7e8763a 4.9.0-6-amd64 #1 SMP Debian 4.9.82-1+deb9u3 (2018-03-02) x86_64 x86_64 x86_64 GNU/Linux
Python version: Python 2.7.6
google-cloud-bigquery: 1.1.0

It would be really helpful if the error message to QueryJob.result() printed out the query, along with line and column numbers. Here's our current workaround, which unfortunately does not allow us to use the built-in Retry due to a nested try/except.
```
RETRYABLE_EXCEPTIONS = (InternalServerError, TooManyRequests, ServiceUnavailable, BadGateway)

...

    def _execute_query_job(self, query_job, query, max_wait_secs=None):
        # Need to do retries manually, since the built-in retry doesn't work within a try/except.
        start_time = datetime.datetime.utcnow()
        deadline = start_time + datetime.timedelta(seconds=max_wait_secs or self.max_wait_secs)
        i = 1
        while datetime.datetime.utcnow() < deadline:
            try:
                return query_job.result()
            except RETRYABLE_EXCEPTIONS:
                time.sleep(i**2)
                i += 1
            except GoogleCloudError as e:
                msg = str(e)
                # Put line numbers next to the SQL.
                lines = query.split('\n')
                longest = max(len(l) for l in lines)
                # Print out a 'ruler' above and below the SQL so we can judge columns.
                ruler = ' ' * 4 + '|'  # Left pad for the line numbers (4 digits plus ':')
                for _ in range(longest / 10):
                    ruler += ' ' * 4 + '.' + ' ' * 4 + '|'
                header = '-----Offending Sql Follows-----'
                padding = ' ' * ((longest - len(header)) / 2)
                msg += '\n\n{}{}\n\n{}\n{}\n{}'.format(padding, header, ruler, '\n'.join(
                    '{:4}:{}'.format(n + 1, line) for n, line in enumerate(lines)), ruler)
                raise RuntimeError(msg)
        raise RuntimeError('Deadline of {} seconds exceeded.'.format(max_wait_secs))
```


Discussion:

Resolving patch:
diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py
--- a/bigquery/google/cloud/bigquery/job.py
+++ b/bigquery/google/cloud/bigquery/job.py
@@ -2846,6 +2846,36 @@ def _blocking_poll(self, timeout=None):
         self._done_timeout = timeout
         super(QueryJob, self)._blocking_poll(timeout=timeout)
 
+    @staticmethod
+    def _format_for_exception(query, job_id):
+        """Format a query for the output in exception message.
+
+        Args:
+            query (str): The SQL query to format.
+            job_id (str): The ID of the job that ran the query.
+
+        Returns: (str)
+            A formatted query text.
+        """
+        template = "\n\n(job ID: {job_id})\n\n{header}\n\n{ruler}\n{body}\n{ruler}"
+
+        lines = query.splitlines()
+        max_line_len = max(len(l) for l in lines)
+
+        header = "-----Query Job SQL Follows-----"
+        header = "{:^{total_width}}".format(header, total_width=max_line_len + 5)
+
+        # Print out a "ruler" above and below the SQL so we can judge columns.
+        # Left pad for the line numbers (4 digits plus ":").
+        ruler = "    |" + "    .    |" * (max_line_len // 10)
+
+        # Put line numbers next to the SQL.
+        body = "\n".join(
+            "{:4}:{}".format(n, line) for n, line in enumerate(lines, start=1)
+        )
+
+        return template.format(job_id=job_id, header=header, ruler=ruler, body=body)
+
     def result(self, timeout=None, page_size=None, retry=DEFAULT_RETRY):
         """Start the job and wait for it to complete and get the result.
 
@@ -2874,12 +2904,17 @@ def result(self, timeout=None, page_size=None, retry=DEFAULT_RETRY):
             concurrent.futures.TimeoutError:
                 If the job did not complete in the given timeout.
         """
-        super(QueryJob, self).result(timeout=timeout)
-        # Return an iterator instead of returning the job.
-        if not self._query_results:
-            self._query_results = self._client._get_query_results(
-                self.job_id, retry, project=self.project, location=self.location
-            )
+        try:
+            super(QueryJob, self).result(timeout=timeout)
+
+            # Return an iterator instead of returning the job.
+            if not self._query_results:
+                self._query_results = self._client._get_query_results(
+                    self.job_id, retry, project=self.project, location=self.location
+                )
+        except exceptions.GoogleCloudError as exc:
+            exc.message += self._format_for_exception(self.query, self.job_id)
+            raise
 
         # If the query job is complete but there are no query results, this was
         # special job, such as a DDL query. Return an empty result set to


Test patch:
diff --git a/bigquery/tests/unit/test_job.py b/bigquery/tests/unit/test_job.py
--- a/bigquery/tests/unit/test_job.py
+++ b/bigquery/tests/unit/test_job.py
@@ -14,6 +14,7 @@
 
 import copy
 import json
+import textwrap
 import unittest
 
 import mock
@@ -4256,8 +4257,15 @@ def test_result_w_page_size(self):
     def test_result_error(self):
         from google.cloud import exceptions
 
+        query = textwrap.dedent(
+            """
+            SELECT foo, bar
+            FROM table_baz
+            WHERE foo == bar"""
+        )
+
         client = _make_client(project=self.PROJECT)
-        job = self._make_one(self.JOB_ID, self.QUERY, client)
+        job = self._make_one(self.JOB_ID, query, client)
         error_result = {
             "debugInfo": "DEBUG",
             "location": "LOCATION",
@@ -4277,6 +4285,15 @@ def test_result_error(self):
         self.assertIsInstance(exc_info.exception, exceptions.GoogleCloudError)
         self.assertEqual(exc_info.exception.code, http_client.BAD_REQUEST)
 
+        full_text = str(exc_info.exception)
+
+        assert job.job_id in full_text
+        assert "Query Job SQL Follows" in full_text
+
+        for i, line in enumerate(query.splitlines(), start=1):
+            expected_line = "{}:{}".format(i, line)
+            assert expected_line in full_text
+
     def test_begin_w_bound_client(self):
         from google.cloud.bigquery.dataset import DatasetReference
         from google.cloud.bigquery.job import QueryJobConfig