Repository: googleapis/python-bigquery

Issue:
minor: add integration test for column acls feature
Once the datacatalog v1 endpoint releases a version with support for the PolicyTagManager client, let's add an integration test to better exercise the column ACL feature.

[This commit](https://github.com/googleapis/googleapis/commit/91eee3d039fbdbadee008393504900287bbc6f43) to proto comments should unblock the datacatalog release.


Discussion:
@shollyman There exists a basic [system test](https://github.com/googleapis/python-bigquery/blob/6a48e80bc7d347f381b181f4cf81fef105d0ad0d/tests/system/test_client.py#L333-L382) for setting/updating policy tags on a field, but you mention a `PolicyTagManager` client - presumably this class will be generated?

The synthesis is currently blocked, though, it's failing because of outdated ariifacts in the v4 Bazel image (or something like that).
The system test uses an artificial policy tag.  The change is to create a real taxonomy and policy tag from datacatalog and use it in its place.

https://cloud.google.com/data-catalog/docs/reference/rpc/google.cloud.datacatalog.v1#google.cloud.datacatalog.v1.PolicyTagManager is the client in datacatalog for instantiating the necessary resources.


@shollyman Do you know when the API will support `PolicyTagManagerClient`? Soon-ish?

Trying to create a policy tag returns 404 MethodNotImplemented at the time of writing.

Resolving patch:
diff --git a/noxfile.py b/noxfile.py
--- a/noxfile.py
+++ b/noxfile.py
@@ -142,6 +142,9 @@ def system(session):
     else:
         session.install("google-cloud-storage", "-c", constraints_path)
 
+    # Data Catalog needed for the column ACL test with a real Policy Tag.
+    session.install("google-cloud-datacatalog", "-c", constraints_path)
+
     session.install("-e", ".[all]", "-c", constraints_path)
     session.install("ipython", "-c", constraints_path)
 
@@ -211,6 +214,7 @@ def prerelease_deps(session):
     session.install("--pre", "grpcio", "pandas")
     session.install(
         "freezegun",
+        "google-cloud-datacatalog",
         "google-cloud-storage",
         "google-cloud-testutils",
         "IPython",


Test patch:
diff --git a/tests/system/test_client.py b/tests/system/test_client.py
--- a/tests/system/test_client.py
+++ b/tests/system/test_client.py
@@ -68,6 +68,8 @@
 from google.cloud._helpers import UTC
 from google.cloud.bigquery import dbapi, enums
 from google.cloud import storage
+from google.cloud.datacatalog_v1 import types as datacatalog_types
+from google.cloud.datacatalog_v1 import PolicyTagManagerClient
 
 from test_utils.retry import RetryErrors
 from test_utils.retry import RetryInstanceState
@@ -167,6 +169,8 @@ def setUp(self):
         self.to_delete = [dataset]
 
     def tearDown(self):
+        policy_tag_client = PolicyTagManagerClient()
+
         def _still_in_use(bad_request):
             return any(
                 error["reason"] == "resourceInUse" for error in bad_request._errors
@@ -183,6 +187,8 @@ def _still_in_use(bad_request):
                 retry_in_use(Config.CLIENT.delete_dataset)(doomed, delete_contents=True)
             elif isinstance(doomed, (Table, bigquery.TableReference)):
                 retry_in_use(Config.CLIENT.delete_table)(doomed)
+            elif isinstance(doomed, datacatalog_types.Taxonomy):
+                policy_tag_client.delete_taxonomy(name=doomed.name)
             else:
                 doomed.delete()
 
@@ -381,6 +387,68 @@ def test_create_table_with_policy(self):
         table2 = Config.CLIENT.update_table(table, ["schema"])
         self.assertEqual(policy_2, table2.schema[1].policy_tags)
 
+    def test_create_table_with_real_custom_policy(self):
+        from google.cloud.bigquery.schema import PolicyTagList
+
+        policy_tag_client = PolicyTagManagerClient()
+        taxonomy_parent = f"projects/{Config.CLIENT.project}/locations/us"
+
+        new_taxonomy = datacatalog_types.Taxonomy(
+            display_name="Custom test taxonomy",
+            description="This taxonomy is ony used for a test.",
+            activated_policy_types=[
+                datacatalog_types.Taxonomy.PolicyType.FINE_GRAINED_ACCESS_CONTROL
+            ],
+        )
+
+        taxonomy = policy_tag_client.create_taxonomy(
+            parent=taxonomy_parent, taxonomy=new_taxonomy
+        )
+        self.to_delete.insert(0, taxonomy)
+
+        parent_policy_tag = policy_tag_client.create_policy_tag(
+            parent=taxonomy.name,
+            policy_tag=datacatalog_types.PolicyTag(
+                display_name="Parent policy tag", parent_policy_tag=None
+            ),
+        )
+        child_policy_tag = policy_tag_client.create_policy_tag(
+            parent=taxonomy.name,
+            policy_tag=datacatalog_types.PolicyTag(
+                display_name="Child policy tag",
+                parent_policy_tag=parent_policy_tag.name,
+            ),
+        )
+
+        dataset = self.temp_dataset(
+            _make_dataset_id("create_table_with_real_custom_policy")
+        )
+        table_id = "test_table"
+        policy_1 = PolicyTagList(names=[parent_policy_tag.name])
+        policy_2 = PolicyTagList(names=[child_policy_tag.name])
+
+        schema = [
+            bigquery.SchemaField(
+                "first_name", "STRING", mode="REQUIRED", policy_tags=policy_1
+            ),
+            bigquery.SchemaField(
+                "age", "INTEGER", mode="REQUIRED", policy_tags=policy_2
+            ),
+        ]
+        table_arg = Table(dataset.table(table_id), schema=schema)
+        self.assertFalse(_table_exists(table_arg))
+
+        table = helpers.retry_403(Config.CLIENT.create_table)(table_arg)
+        self.to_delete.insert(0, table)
+
+        self.assertTrue(_table_exists(table))
+        self.assertCountEqual(
+            list(table.schema[0].policy_tags.names), [parent_policy_tag.name]
+        )
+        self.assertCountEqual(
+            list(table.schema[1].policy_tags.names), [child_policy_tag.name]
+        )
+
     def test_create_table_w_time_partitioning_w_clustering_fields(self):
         from google.cloud.bigquery.table import TimePartitioning
         from google.cloud.bigquery.table import TimePartitioningType