Repository: googleapis/google-cloud-python

Issue:
BigTable Policy.etag is bytes (should be base64 str)
#### Environment details

1. API: BigTable
2. OS type and version: OSX
3. Python version and virtual environment information: 3.7.0
4. google-cloud-bigtable 0.32.1

#### Steps to reproduce

1. Try to convert the `to_api_repr()` of a `google.cloud.bigtable.policy.Policy` to json.

#### Code example

```python
from google.cloud import bigtable
import json
client = bigtable.Client(admin=True)
(instances_list, failed_locations) = client.list_instances()
policy = instances_list[0].get_iam_policy()
json.dumps(policy.to_api_repr())
```

#### Stack trace

Unlike other policies (such as the one returned from `google.cloud.storage`), `etag` is not a `str` (contrary to [doc](https://github.com/googleapis/google-cloud-python/blob/e47616a382f9b5e765db0c7c334b7cbf225f9387/api_core/google/api_core/iam.py#L65)), and `to_api_repr` cannot be converted to json (contrary to [doc](https://github.com/googleapis/google-cloud-python/blob/e47616a382f9b5e765db0c7c334b7cbf225f9387/api_core/google/api_core/iam.py#L251)), apparently because bigtable’s [`Policy` passes `_to_bytes(etag)` to the superclass constructor](https://github.com/googleapis/google-cloud-python/blob/e47616a382f9b5e765db0c7c334b7cbf225f9387/bigtable/google/cloud/bigtable/policy.py#L76).

```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/__init__.py", line 231, in dumps
    return _default_encoder.encode(obj)
  File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/encoder.py", line 199, in encode
    chunks = self.iterencode(o, _one_shot=True)
  File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/encoder.py", line 257, in iterencode
    return _iterencode(o, 0)
  File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/encoder.py", line 179, in default
    raise TypeError(f'Object of type {o.__class__.__name__} '
TypeError: Object of type bytes is not JSON serializable
```

The `to_api_repr()` of a bigtable `Policy` is a dict containing binary bytes like `{'etag': b'\x00 \x01', 'version': 0}`, whereas the `to_api_repr()` of a `Policy` from google cloud storage is a dict whose etag is base64-encoded such as `{'etag': 'CAU=', 'bindings': […]}`


Discussion:
The `google.api_core.iam.Policy` base class clearly expects `Policy.etag` is supposed to be stored as text (it copies values without conversion to / from the attribute in its `from_api_repr` / `to_api_repr`.
OK, digging further:   I think the patch in #7373 is actually wrong, because the backend APIs for `Instance.get_iam_policy` and `Instance.set_iam_policy` use a protobuf message whose etag is passed explicitly as bytes.

We should therefore override `Policy.to_api_repr` and `Policy.from_api_repr` to handle converting the bytes to / from a JSON-compatible representation, and leave the attribute stored as bytes.

Resolving patch:
diff --git a/bigtable/google/cloud/bigtable/instance.py b/bigtable/google/cloud/bigtable/instance.py
--- a/bigtable/google/cloud/bigtable/instance.py
+++ b/bigtable/google/cloud/bigtable/instance.py
@@ -414,7 +414,7 @@ def get_iam_policy(self):
         """
         instance_admin_client = self._client.instance_admin_client
         resp = instance_admin_client.get_iam_policy(resource=self.name)
-        return Policy.from_api_repr(self._to_dict_from_policy_pb(resp))
+        return Policy.from_pb(resp)
 
     def set_iam_policy(self, policy):
         """Sets the access control policy on an instance resource. Replaces any
@@ -438,9 +438,9 @@ class `google.cloud.bigtable.policy.Policy`
         """
         instance_admin_client = self._client.instance_admin_client
         resp = instance_admin_client.set_iam_policy(
-            resource=self.name, policy=policy.to_api_repr()
+            resource=self.name, policy=policy.to_pb()
         )
-        return Policy.from_api_repr(self._to_dict_from_policy_pb(resp))
+        return Policy.from_pb(resp)
 
     def test_iam_permissions(self, permissions):
         """Returns permissions that the caller has on the specified instance
@@ -470,21 +470,6 @@ def test_iam_permissions(self, permissions):
         )
         return list(resp.permissions)
 
-    def _to_dict_from_policy_pb(self, policy):
-        """Returns a dictionary representation of resource returned from
-        the getIamPolicy API to use as parameter for
-        :meth: google.api_core.iam.Policy.from_api_repr
-        """
-        pb_dict = {}
-        bindings = [
-            {"role": binding.role, "members": binding.members}
-            for binding in policy.bindings
-        ]
-        pb_dict["etag"] = policy.etag
-        pb_dict["version"] = policy.version
-        pb_dict["bindings"] = bindings
-        return pb_dict
-
     def cluster(
         self, cluster_id, location_id=None, serve_nodes=None, default_storage_type=None
     ):
diff --git a/bigtable/google/cloud/bigtable/policy.py b/bigtable/google/cloud/bigtable/policy.py
--- a/bigtable/google/cloud/bigtable/policy.py
+++ b/bigtable/google/cloud/bigtable/policy.py
@@ -12,8 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
+
 from google.api_core.iam import Policy as BasePolicy
 from google.cloud._helpers import _to_bytes
+from google.iam.v1 import policy_pb2
 
 """IAM roles supported by Bigtable Instance resource"""
 BIGTABLE_ADMIN_ROLE = "roles/bigtable.admin"
@@ -107,3 +110,76 @@ def bigtable_viewers(self):
         for member in self._bindings.get(BIGTABLE_VIEWER_ROLE, ()):
             result.add(member)
         return frozenset(result)
+
+    @classmethod
+    def from_pb(cls, policy_pb):
+        """Factory: create a policy from a protobuf message.
+
+        Args:
+            policy_pb (google.iam.policy_pb2.Policy): message returned by
+            ``get_iam_policy`` gRPC API.
+
+        Returns:
+            :class:`Policy`: the parsed policy
+        """
+        policy = cls(policy_pb.etag, policy_pb.version)
+
+        for binding in policy_pb.bindings:
+            policy[binding.role] = sorted(binding.members)
+
+        return policy
+
+    def to_pb(self):
+        """Render a protobuf message.
+
+        Returns:
+            google.iam.policy_pb2.Policy: a message to be passed to the
+            ``set_iam_policy`` gRPC API.
+        """
+
+        return policy_pb2.Policy(
+            etag=self.etag,
+            version=self.version or 0,
+            bindings=[
+                policy_pb2.Binding(role=role, members=sorted(self[role]))
+                for role in self
+            ],
+        )
+
+    @classmethod
+    def from_api_repr(cls, resource):
+        """Factory: create a policy from a JSON resource.
+
+        Overrides the base class version to store :attr:`etag` as bytes.
+
+        Args:
+            resource (dict): JSON policy resource returned by the
+            ``getIamPolicy`` REST API.
+
+        Returns:
+            :class:`Policy`: the parsed policy
+        """
+        etag = resource.get("etag")
+
+        if etag is not None:
+            resource = resource.copy()
+            resource["etag"] = base64.b64decode(etag.encode("ascii"))
+
+        return super(Policy, cls).from_api_repr(resource)
+
+    def to_api_repr(self):
+        """Render a JSON policy resource.
+
+        Overrides the base class version to convert :attr:`etag` from bytes
+        to JSON-compatible base64-encoded text.
+
+        Returns:
+            dict: a JSON resource to be passed to the
+            ``setIamPolicy`` REST API.
+        """
+        resource = super(Policy, self).to_api_repr()
+
+        if self.etag is not None:
+            resource["etag"] = base64.b64encode(self.etag).decode("ascii")
+
+        return resource


Test patch:
diff --git a/bigtable/tests/unit/test_instance.py b/bigtable/tests/unit/test_instance.py
--- a/bigtable/tests/unit/test_instance.py
+++ b/bigtable/tests/unit/test_instance.py
@@ -641,7 +641,7 @@ def test_set_iam_policy(self):
         version = 1
         etag = b"etag_v1"
         members = ["serviceAccount:service_acc1@test.com", "user:user1@test.com"]
-        bindings = [{"role": BIGTABLE_ADMIN_ROLE, "members": members}]
+        bindings = [{"role": BIGTABLE_ADMIN_ROLE, "members": sorted(members)}]
         iam_policy_pb = policy_pb2.Policy(version=version, etag=etag, bindings=bindings)
 
         # Patch the stub used by the API method.
@@ -661,8 +661,7 @@ def test_set_iam_policy(self):
         result = instance.set_iam_policy(iam_policy)
 
         instance_api.set_iam_policy.assert_called_once_with(
-            resource=instance.name,
-            policy={"version": version, "etag": etag, "bindings": bindings},
+            resource=instance.name, policy=iam_policy_pb
         )
         self.assertEqual(result.version, version)
         self.assertEqual(result.etag, etag)
diff --git a/bigtable/tests/unit/test_policy.py b/bigtable/tests/unit/test_policy.py
--- a/bigtable/tests/unit/test_policy.py
+++ b/bigtable/tests/unit/test_policy.py
@@ -86,3 +86,116 @@ def test_bigtable_viewers_getter(self):
         policy = self._make_one()
         policy[BIGTABLE_VIEWER_ROLE] = [MEMBER]
         self.assertEqual(policy.bigtable_viewers, expected)
+
+    def test_from_pb_empty(self):
+        from google.iam.v1 import policy_pb2
+
+        empty = frozenset()
+        message = policy_pb2.Policy()
+        klass = self._get_target_class()
+        policy = klass.from_pb(message)
+        self.assertEqual(policy.etag, b"")
+        self.assertEqual(policy.version, 0)
+        self.assertEqual(policy.bigtable_admins, empty)
+        self.assertEqual(policy.bigtable_readers, empty)
+        self.assertEqual(policy.bigtable_users, empty)
+        self.assertEqual(policy.bigtable_viewers, empty)
+        self.assertEqual(len(policy), 0)
+        self.assertEqual(dict(policy), {})
+
+    def test_from_pb_non_empty(self):
+        from google.iam.v1 import policy_pb2
+        from google.cloud.bigtable.policy import BIGTABLE_ADMIN_ROLE
+
+        ETAG = b"ETAG"
+        VERSION = 17
+        members = ["serviceAccount:service_acc1@test.com", "user:user1@test.com"]
+        empty = frozenset()
+        message = policy_pb2.Policy(
+            etag=ETAG,
+            version=VERSION,
+            bindings=[{"role": BIGTABLE_ADMIN_ROLE, "members": members}],
+        )
+        klass = self._get_target_class()
+        policy = klass.from_pb(message)
+        self.assertEqual(policy.etag, ETAG)
+        self.assertEqual(policy.version, VERSION)
+        self.assertEqual(policy.bigtable_admins, set(members))
+        self.assertEqual(policy.bigtable_readers, empty)
+        self.assertEqual(policy.bigtable_users, empty)
+        self.assertEqual(policy.bigtable_viewers, empty)
+        self.assertEqual(len(policy), 1)
+        self.assertEqual(dict(policy), {BIGTABLE_ADMIN_ROLE: set(members)})
+
+    def test_to_pb_empty(self):
+        from google.iam.v1 import policy_pb2
+
+        policy = self._make_one()
+        expected = policy_pb2.Policy()
+
+        self.assertEqual(policy.to_pb(), expected)
+
+    def test_to_pb_explicit(self):
+        from google.iam.v1 import policy_pb2
+        from google.cloud.bigtable.policy import BIGTABLE_ADMIN_ROLE
+
+        VERSION = 17
+        ETAG = b"ETAG"
+        members = ["serviceAccount:service_acc1@test.com", "user:user1@test.com"]
+        policy = self._make_one(ETAG, VERSION)
+        policy[BIGTABLE_ADMIN_ROLE] = members
+        expected = policy_pb2.Policy(
+            etag=ETAG,
+            version=VERSION,
+            bindings=[
+                policy_pb2.Binding(role=BIGTABLE_ADMIN_ROLE, members=sorted(members))
+            ],
+        )
+
+        self.assertEqual(policy.to_pb(), expected)
+
+    def test_from_api_repr_wo_etag(self):
+        VERSION = 17
+        empty = frozenset()
+        resource = {"version": VERSION}
+        klass = self._get_target_class()
+        policy = klass.from_api_repr(resource)
+        self.assertIsNone(policy.etag)
+        self.assertEqual(policy.version, VERSION)
+        self.assertEqual(policy.bigtable_admins, empty)
+        self.assertEqual(policy.bigtable_readers, empty)
+        self.assertEqual(policy.bigtable_users, empty)
+        self.assertEqual(policy.bigtable_viewers, empty)
+        self.assertEqual(len(policy), 0)
+        self.assertEqual(dict(policy), {})
+
+    def test_from_api_repr_w_etag(self):
+        import base64
+
+        ETAG = b"ETAG"
+        empty = frozenset()
+        resource = {"etag": base64.b64encode(ETAG).decode("ascii")}
+        klass = self._get_target_class()
+        policy = klass.from_api_repr(resource)
+        self.assertEqual(policy.etag, ETAG)
+        self.assertIsNone(policy.version)
+        self.assertEqual(policy.bigtable_admins, empty)
+        self.assertEqual(policy.bigtable_readers, empty)
+        self.assertEqual(policy.bigtable_users, empty)
+        self.assertEqual(policy.bigtable_viewers, empty)
+        self.assertEqual(len(policy), 0)
+        self.assertEqual(dict(policy), {})
+
+    def test_to_api_repr_wo_etag(self):
+        VERSION = 17
+        resource = {"version": VERSION}
+        policy = self._make_one(version=VERSION)
+        self.assertEqual(policy.to_api_repr(), resource)
+
+    def test_to_api_repr_w_etag(self):
+        import base64
+
+        ETAG = b"ETAG"
+        policy = self._make_one(etag=ETAG)
+        resource = {"etag": base64.b64encode(ETAG).decode("ascii")}
+        self.assertEqual(policy.to_api_repr(), resource)