Repository: googleapis/python-bigquery

Issue:
Job already exists
Python: 3.9 Cloud function runtime

Lib:
google-cloud-bigquery==2.15.0
google-auth==1.30.0
google-api-python-client==2.7.0

`
"/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/bigquery/client.py", line 3149, in query query_job._begin(retry=retry, timeout=timeout) File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/bigquery/job/query.py", line 1044, in _begin super(QueryJob, self)._begin(client=client, retry=retry, timeout=timeout) File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/bigquery/job/base.py", line 461, in _begin api_response = client._call_api( File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/bigquery/client.py", line 685, in _call_api return call() File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/api_core/retry.py", line 285, in retry_wrapped_func return retry_target( File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/api_core/retry.py", line 188, in retry_target return target() File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/_http.py", line 483, in api_request raise exceptions.from_http_response(response) google.api_core.exceptions.Conflict: 409 POST https://bigquery.googleapis.com/bigquery/v2/projects/<project_id>/jobs?prettyPrint=false: Already Exists: Job <project>:US.<Job_id>(job ID: <Job_id>) `


I got the following error in one of production environment, its first time i have encountered this error.


Discussion:
@nitishxp Does your application run queries (and create query jobs behind the scenes) by explicitly passing the `job_id` argument? Could it be that the same "create query job" request was sent twice in a short time span using the same `job_id`?


@plamut  

No I don't do any of those things

Below snippet code I use

```
job_config = bigquery.QueryJobConfig() 
job_config.destination = dest_tbl_ref
job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND
svtt_job = gbq.query(view_query, job_config=job_config).job_id

```
`
Traceback (most recent call last):   File "/workspace/common/functions.py", line 55, in wrapper     response = wrapped_func(*args, **kwargs)   File "/workspace/main.py", line 370, in bq_load job_id =  <function_name>)   File "/workspace/main.py", line 242,  in  <function_name>  svtt_job = gbq.query(view_query, job_config=job_config)   File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/bigquery/client.py", line 3149, in query     query_job._begin(retry=retry, timeout=timeout)   File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/bigquery/job/query.py", line 1044, in _begin     super(QueryJob, self)._begin(client=client, retry=retry, timeout=timeout)   File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/bigquery/job/base.py", line 461, in _begin     api_response = client._call_api(   File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/bigquery/client.py", line 685, in _call_api     return call()   File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/api_core/retry.py", line 285, in retry_wrapped_func     return retry_target(   File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/api_core/retry.py", line 188, in retry_target     return target()   File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/_http.py", line 483, in api_request     raise exceptions.from_http_response(response) google.api_core.exceptions.Conflict: 409 POST `
@nitishxp I see, interesting. There's indeed nothing special about the way the query is run.

@tswast Have you seen this before? Could it happen that the server receives the request successfully, but the client does not know that because the response gets lost (or happens just before the timeout expires), and thus the client retries the very same request, even though it already succeeded?
> @tswast Have you seen this before? Could it happen that the server receives the request successfully, but the client does not know that because the response gets lost (or happens just before the timeout expires), and thus the client retries the very same request, even though it already succeeded?

I haven't seen it before, but it is something we anticipate can happen. I suspect this issue was disguised before we started retrying connection/transport-layer errors.

We have implemented the fix for this in Java. I guess you just missed that part of the spec when we did the Python client implementation.

https://github.com/googleapis/java-bigquery/blob/e755470ad3699eee8816f55cdec4dd995cf85d52/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java#L385-L401

Resolving patch:
diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py
--- a/google/cloud/bigquery/client.py
+++ b/google/cloud/bigquery/client.py
@@ -3190,6 +3190,7 @@ def query(
                 If ``job_config`` is not an instance of :class:`~google.cloud.bigquery.job.QueryJobConfig`
                 class.
         """
+        job_id_given = job_id is not None
         job_id = _make_job_id(job_id, job_id_prefix)
 
         if project is None:
@@ -3221,9 +3222,30 @@ def query(
 
         job_ref = job._JobReference(job_id, project=project, location=location)
         query_job = job.QueryJob(job_ref, query, client=self, job_config=job_config)
-        query_job._begin(retry=retry, timeout=timeout)
 
-        return query_job
+        try:
+            query_job._begin(retry=retry, timeout=timeout)
+        except core_exceptions.Conflict as create_exc:
+            # The thought is if someone is providing their own job IDs and they get
+            # their job ID generation wrong, this could end up returning results for
+            # the wrong query. We thus only try to recover if job ID was not given.
+            if job_id_given:
+                raise create_exc
+
+            try:
+                query_job = self.get_job(
+                    job_id,
+                    project=project,
+                    location=location,
+                    retry=retry,
+                    timeout=timeout,
+                )
+            except core_exceptions.GoogleAPIError:  # (includes RetryError)
+                raise create_exc
+            else:
+                return query_job
+        else:
+            return query_job
 
     def insert_rows(
         self,


Test patch:
diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py
--- a/tests/unit/test_client.py
+++ b/tests/unit/test_client.py
@@ -4617,6 +4617,81 @@ def test_query_w_query_parameters(self):
             },
         )
 
+    def test_query_job_rpc_fail_w_random_error(self):
+        from google.api_core.exceptions import Unknown
+        from google.cloud.bigquery.job import QueryJob
+
+        creds = _make_credentials()
+        http = object()
+        client = self._make_one(project=self.PROJECT, credentials=creds, _http=http)
+
+        job_create_error = Unknown("Not sure what went wrong.")
+        job_begin_patcher = mock.patch.object(
+            QueryJob, "_begin", side_effect=job_create_error
+        )
+        with job_begin_patcher:
+            with pytest.raises(Unknown, match="Not sure what went wrong."):
+                client.query("SELECT 1;", job_id="123")
+
+    def test_query_job_rpc_fail_w_conflict_job_id_given(self):
+        from google.api_core.exceptions import Conflict
+        from google.cloud.bigquery.job import QueryJob
+
+        creds = _make_credentials()
+        http = object()
+        client = self._make_one(project=self.PROJECT, credentials=creds, _http=http)
+
+        job_create_error = Conflict("Job already exists.")
+        job_begin_patcher = mock.patch.object(
+            QueryJob, "_begin", side_effect=job_create_error
+        )
+        with job_begin_patcher:
+            with pytest.raises(Conflict, match="Job already exists."):
+                client.query("SELECT 1;", job_id="123")
+
+    def test_query_job_rpc_fail_w_conflict_random_id_job_fetch_fails(self):
+        from google.api_core.exceptions import Conflict
+        from google.api_core.exceptions import DataLoss
+        from google.cloud.bigquery.job import QueryJob
+
+        creds = _make_credentials()
+        http = object()
+        client = self._make_one(project=self.PROJECT, credentials=creds, _http=http)
+
+        job_create_error = Conflict("Job already exists.")
+        job_begin_patcher = mock.patch.object(
+            QueryJob, "_begin", side_effect=job_create_error
+        )
+        get_job_patcher = mock.patch.object(
+            client, "get_job", side_effect=DataLoss("we lost yor job, sorry")
+        )
+
+        with job_begin_patcher, get_job_patcher:
+            # If get job request fails, the original exception should be raised.
+            with pytest.raises(Conflict, match="Job already exists."):
+                client.query("SELECT 1;", job_id=None)
+
+    def test_query_job_rpc_fail_w_conflict_random_id_job_fetch_succeeds(self):
+        from google.api_core.exceptions import Conflict
+        from google.cloud.bigquery.job import QueryJob
+
+        creds = _make_credentials()
+        http = object()
+        client = self._make_one(project=self.PROJECT, credentials=creds, _http=http)
+
+        job_create_error = Conflict("Job already exists.")
+        job_begin_patcher = mock.patch.object(
+            QueryJob, "_begin", side_effect=job_create_error
+        )
+        get_job_patcher = mock.patch.object(
+            client, "get_job", return_value=mock.sentinel.query_job
+        )
+
+        with job_begin_patcher, get_job_patcher:
+            result = client.query("SELECT 1;", job_id=None)
+
+        assert result is mock.sentinel.query_job
+
     def test_insert_rows_w_timeout(self):
         from google.cloud.bigquery.schema import SchemaField
         from google.cloud.bigquery.table import Table