Repository: googleapis/python-bigquery

Issue:
Add BiEngineStatistics to query job statistics
Some new query statistics have been added, which are present if [BI engine is enabled](https://cloud.google.com/bi-engine/docs/introduction)

**Suggested implementation**

* Create two new classes, reflecting the API: `BiEngineStatistics` and `BiEngineReason`.
* Expose the [biEngineStatistics property from `JobStatistics2`](https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobStatistics2.FIELDS.bi_engine_statistics) on `QueryJob`, similar to DmlStats.

**API details**

Discovery representation for BiEngineStatistics:

```
"BiEngineStatistics": {
      "id": "BiEngineStatistics",
      "properties": {
        "biEngineMode": {
          "description": "[Output-only] Specifies which mode of BI Engine acceleration was performed (if any).",
          "type": "string",
          "default": "$(stats.bi_engine_mode)"
        },
        "biEngineReasons": {
          "type": "array",
          "items": {
            "$ref": "BiEngineReason"
          },
          "description": "In case of DISABLED or PARTIAL bi_engine_mode, these contain the explanatory reasons as to why BI Engine could not accelerate. In case the full query was accelerated, this field is not populated."
        }
      },
      "type": "object"
    },
```


The BiEngineReason discovery representation:

```
"BiEngineReason": {
      "type": "object",
      "id": "BiEngineReason",
      "properties": {
        "code": {
          "description": "[Output-only] High-level BI Engine reason for partial or disabled acceleration.",
          "default": "$(reason.code)",
          "type": "string"
        },
        "message": {
          "description": "[Output-only] Free form human-readable reason for partial or disabled acceleration.",
          "type": "string",
          "default": "$(reason.message)"
        }
      }
    },
```

**Testing notes**

Unit tests are sufficient for this work. Manually testing and/or system testing will be difficult as it requires manual steps to [set-up a long-term BI Engine reservation](https://cloud.google.com/bi-engine/docs/reserving-capacity) on the project.

**Related**

* [Go PR](https://github.com/googleapis/google-cloud-go/pull/5081)



Discussion:
DmlStats seems to be quite similar to this request: https://github.com/googleapis/python-bigquery/blob/66b3dd9f9aec3fda9610a3ceec8d8a477f2ab3b9/google/cloud/bigquery/job/query.py#L123

Though, the sub-BiEngineReason may mean it may be easier to use our `_properties` implementation than using named tuples.

Resolving patch:
diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py
--- a/google/cloud/bigquery/job/query.py
+++ b/google/cloud/bigquery/job/query.py
@@ -121,6 +121,44 @@ def _to_api_repr_table_defs(value):
     return {k: ExternalConfig.to_api_repr(v) for k, v in value.items()}
 
 
+class BiEngineReason(typing.NamedTuple):
+    """Reason for BI Engine acceleration failure
+
+    https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#bienginereason
+    """
+
+    code: str = "CODE_UNSPECIFIED"
+
+    reason: str = ""
+
+    @classmethod
+    def from_api_repr(cls, reason: Dict[str, str]) -> "BiEngineReason":
+        return cls(reason.get("code", "CODE_UNSPECIFIED"), reason.get("message", ""))
+
+
+class BiEngineStats(typing.NamedTuple):
+    """Statistics for a BI Engine query
+
+    https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#bienginestatistics
+    """
+
+    mode: str = "ACCELERATION_MODE_UNSPECIFIED"
+    """ Specifies which mode of BI Engine acceleration was performed (if any)
+    """
+
+    reasons: List[BiEngineReason] = []
+    """ Contains explanatory messages in case of DISABLED / PARTIAL acceleration
+    """
+
+    @classmethod
+    def from_api_repr(cls, stats: Dict[str, Any]) -> "BiEngineStats":
+        mode = stats.get("biEngineMode", "ACCELERATION_MODE_UNSPECIFIED")
+        reasons = [
+            BiEngineReason.from_api_repr(r) for r in stats.get("biEngineReasons", [])
+        ]
+        return cls(mode, reasons)
+
+
 class DmlStats(typing.NamedTuple):
     """Detailed statistics for DML statements.
 
@@ -1191,6 +1229,15 @@ def dml_stats(self) -> Optional[DmlStats]:
         else:
             return DmlStats.from_api_repr(stats)
 
+    @property
+    def bi_engine_stats(self) -> Optional[BiEngineStats]:
+        stats = self._job_statistics().get("biEngineStatistics")
+
+        if stats is None:
+            return None
+        else:
+            return BiEngineStats.from_api_repr(stats)
+
     def _blocking_poll(self, timeout=None, **kwargs):
         self._done_timeout = timeout
         self._transport_timeout = timeout


Test patch:
diff --git a/tests/unit/job/test_query.py b/tests/unit/job/test_query.py
--- a/tests/unit/job/test_query.py
+++ b/tests/unit/job/test_query.py
@@ -877,6 +877,23 @@ def test_estimated_bytes_processed(self):
         query_stats["estimatedBytesProcessed"] = str(est_bytes)
         self.assertEqual(job.estimated_bytes_processed, est_bytes)
 
+    def test_bi_engine_stats(self):
+        from google.cloud.bigquery.job.query import BiEngineStats
+
+        client = _make_client(project=self.PROJECT)
+        job = self._make_one(self.JOB_ID, self.QUERY, client)
+        assert job.bi_engine_stats is None
+
+        statistics = job._properties["statistics"] = {}
+        assert job.bi_engine_stats is None
+
+        query_stats = statistics["query"] = {}
+        assert job.bi_engine_stats is None
+
+        query_stats["biEngineStatistics"] = {"biEngineMode": "FULL"}
+        assert isinstance(job.bi_engine_stats, BiEngineStats)
+        assert job.bi_engine_stats.mode == "FULL"
+
     def test_dml_stats(self):
         from google.cloud.bigquery.job.query import DmlStats
 
diff --git a/tests/unit/job/test_query_stats.py b/tests/unit/job/test_query_stats.py
--- a/tests/unit/job/test_query_stats.py
+++ b/tests/unit/job/test_query_stats.py
@@ -15,6 +15,62 @@
 from .helpers import _Base
 
 
+class TestBiEngineStats:
+    @staticmethod
+    def _get_target_class():
+        from google.cloud.bigquery.job.query import BiEngineStats
+
+        return BiEngineStats
+
+    def _make_one(self, *args, **kw):
+        return self._get_target_class()(*args, **kw)
+
+    def test_ctor_defaults(self):
+        bi_engine_stats = self._make_one()
+        assert bi_engine_stats.mode == "ACCELERATION_MODE_UNSPECIFIED"
+        assert bi_engine_stats.reasons == []
+
+    def test_from_api_repr_unspecified(self):
+        klass = self._get_target_class()
+        result = klass.from_api_repr({"biEngineMode": "ACCELERATION_MODE_UNSPECIFIED"})
+
+        assert isinstance(result, klass)
+        assert result.mode == "ACCELERATION_MODE_UNSPECIFIED"
+        assert result.reasons == []
+
+    def test_from_api_repr_full(self):
+        klass = self._get_target_class()
+        result = klass.from_api_repr({"biEngineMode": "FULL"})
+
+        assert isinstance(result, klass)
+        assert result.mode == "FULL"
+        assert result.reasons == []
+
+    def test_from_api_repr_disabled(self):
+        klass = self._get_target_class()
+        result = klass.from_api_repr(
+            {
+                "biEngineMode": "DISABLED",
+                "biEngineReasons": [
+                    {
+                        "code": "OTHER_REASON",
+                        "message": "Unable to support input table xyz due to an internal error.",
+                    }
+                ],
+            }
+        )
+
+        assert isinstance(result, klass)
+        assert result.mode == "DISABLED"
+
+        reason = result.reasons[0]
+        assert reason.code == "OTHER_REASON"
+        assert (
+            reason.reason
+            == "Unable to support input table xyz due to an internal error."
+        )
+
+
 class TestDmlStats:
     @staticmethod
     def _get_target_class():