Repository: googleapis/google-cloud-python

Issue:
Spanner: TypeError: '' has type str, but expected one of: bytes
OS type and version: ubuntu xenial

Python version and virtual environment information `python --version`: python 3.6

google-cloud-python version: https://github.com/GoogleCloudPlatform/google-cloud-python/releases/tag/spanner-1.3.0

Stacktrace if available:
```
  File "/tmp/par__22_www_server.par/__main__/third_party/google_cloud/current/xenial/google/cloud/spanner_v1/streamed.py", line 140, in __iter__
    self._consume_next()  # raises StopIteration
  File "/tmp/par__22_www_server.par/__main__/third_party/google_cloud/current/xenial/google/cloud/spanner_v1/streamed.py", line 114, in _consume_next
    response = six.next(self._response_iterator)
  File "/tmp/par__22_www_server.par/__main__/third_party/google_cloud/current/xenial/google/cloud/spanner_v1/snapshot.py", line 51, in _restart_on_unavailable
    iterator = restart(resume_token=resume_token)
  File "/tmp/par__22_www_server.par/__main__/third_party/google_cloud/current/xenial/google/cloud/spanner_v1/gapic/spanner_client.py", line 662, in execute_streaming_sql
    partition_token=partition_token,
TypeError: '' has type str, but expected one of: bytes
```



Discussion:
Are you iterating over your own data?
Could you convert `''` to `b''`?

which param for that method i should convert `''` to `b''`?

the only thing changed recently is the `partition_token`, whose type is `byte` but i don't use that one. https://github.com/GoogleCloudPlatform/google-cloud-python/blame/spanner-1.3.0/spanner/google/cloud/spanner_v1/gapic/spanner_client.py#L662
@jsimonweb FYI
@yixizhang We need enough information to be able to reproduce the issue.  Can you post a snippet / gist of your code, with any sensitive data redacted / replaced?
@yixizhang can you confirm if you are still experiencing this issue? Thanks!
I can't reproduce it and I haven't experienced the same error ever since.  But by reading the code the other I think this line could be buggy.

https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/spanner/google/cloud/spanner_v1/snapshot.py#L39

That the default value should not be a str, but rather `b''`.  

Hope that helps.
@yixizhang Thanks very much for the follow-up:  I can confirm that the initial `resume_token` should be bytes, rather than text.

Resolving patch:
diff --git a/spanner/google/cloud/spanner_v1/snapshot.py b/spanner/google/cloud/spanner_v1/snapshot.py
--- a/spanner/google/cloud/spanner_v1/snapshot.py
+++ b/spanner/google/cloud/spanner_v1/snapshot.py
@@ -36,7 +36,7 @@ def _restart_on_unavailable(restart):
     :type restart: callable
     :param restart: curried function returning iterator
     """
-    resume_token = ''
+    resume_token = b''
     item_buffer = []
     iterator = restart()
     while True:


Test patch:
diff --git a/spanner/tests/unit/test_snapshot.py b/spanner/tests/unit/test_snapshot.py
--- a/spanner/tests/unit/test_snapshot.py
+++ b/spanner/tests/unit/test_snapshot.py
@@ -28,7 +28,9 @@
 PARAM_TYPES = {'max_age': 'INT64'}
 SQL_QUERY_WITH_BYTES_PARAM = """\
 SELECT image_name FROM images WHERE @bytes IN image_data"""
-PARAMS_WITH_BYTES = {'bytes': b'DEADBEEF'}
+PARAMS_WITH_BYTES = {'bytes': b'FACEDACE'}
+RESUME_TOKEN = b'DEADBEEF'
+TXN_ID = b'DEAFBEAD'
 
 
 class Test_restart_on_unavailable(unittest.TestCase):
@@ -38,7 +40,7 @@ def _call_fut(self, restart):
 
         return _restart_on_unavailable(restart)
 
-    def _make_item(self, value, resume_token=''):
+    def _make_item(self, value, resume_token=b''):
         return mock.Mock(
             value=value, resume_token=resume_token,
             spec=['value', 'resume_token'])
@@ -60,7 +62,7 @@ def test_iteration_w_non_empty_raw(self):
     def test_iteration_w_raw_w_resume_tken(self):
         ITEMS = (
             self._make_item(0),
-            self._make_item(1, resume_token='DEADBEEF'),
+            self._make_item(1, resume_token=RESUME_TOKEN),
             self._make_item(2),
             self._make_item(3),
         )
@@ -70,10 +72,25 @@ def test_iteration_w_raw_w_resume_tken(self):
         self.assertEqual(list(resumable), list(ITEMS))
         restart.assert_called_once_with()
 
+    def test_iteration_w_raw_raising_unavailable_no_token(self):
+        ITEMS = (
+            self._make_item(0),
+            self._make_item(1, resume_token=RESUME_TOKEN),
+            self._make_item(2),
+        )
+        before = _MockIterator(fail_after=True)
+        after = _MockIterator(*ITEMS)
+        restart = mock.Mock(spec=[], side_effect=[before, after])
+        resumable = self._call_fut(restart)
+        self.assertEqual(list(resumable), list(ITEMS))
+        self.assertEqual(
+            restart.mock_calls,
+            [mock.call(), mock.call(resume_token=b'')])
+
     def test_iteration_w_raw_raising_unavailable(self):
         FIRST = (
             self._make_item(0),
-            self._make_item(1, resume_token='DEADBEEF'),
+            self._make_item(1, resume_token=RESUME_TOKEN),
         )
         SECOND = (  # discarded after 503
             self._make_item(2),
@@ -88,12 +105,12 @@ def test_iteration_w_raw_raising_unavailable(self):
         self.assertEqual(list(resumable), list(FIRST + LAST))
         self.assertEqual(
             restart.mock_calls,
-            [mock.call(), mock.call(resume_token='DEADBEEF')])
+            [mock.call(), mock.call(resume_token=RESUME_TOKEN)])
 
     def test_iteration_w_raw_raising_unavailable_after_token(self):
         FIRST = (
             self._make_item(0),
-            self._make_item(1, resume_token='DEADBEEF'),
+            self._make_item(1, resume_token=RESUME_TOKEN),
         )
         SECOND = (
             self._make_item(2),
@@ -106,7 +123,7 @@ def test_iteration_w_raw_raising_unavailable_after_token(self):
         self.assertEqual(list(resumable), list(FIRST + SECOND))
         self.assertEqual(
             restart.mock_calls,
-            [mock.call(), mock.call(resume_token='DEADBEEF')])
+            [mock.call(), mock.call(resume_token=RESUME_TOKEN)])
 
 
 class Test_SnapshotBase(unittest.TestCase):
@@ -190,7 +207,6 @@ def _read_helper(self, multi_use, first=True, count=0, partition=None):
         from google.cloud.spanner_v1.keyset import KeySet
         from google.cloud.spanner_v1._helpers import _make_value_pb
 
-        txn_id = b'DEADBEEF'
         VALUES = [
             [u'bharney', 31],
             [u'phred', 32],
@@ -224,7 +240,7 @@ def _read_helper(self, multi_use, first=True, count=0, partition=None):
         derived._multi_use = multi_use
         derived._read_request_count = count
         if not first:
-            derived._transaction_id = txn_id
+            derived._transaction_id = TXN_ID
 
         if partition is not None:  # 'limit' and 'partition' incompatible
             result_set = derived.read(
@@ -258,7 +274,7 @@ def _read_helper(self, multi_use, first=True, count=0, partition=None):
             if first:
                 self.assertTrue(transaction.begin.read_only.strong)
             else:
-                self.assertEqual(transaction.id, txn_id)
+                self.assertEqual(transaction.id, TXN_ID)
         else:
             self.assertTrue(transaction.single_use.read_only.strong)
         self.assertEqual(index, INDEX)
@@ -286,7 +302,7 @@ def test_read_w_multi_use_wo_first_w_count_gt_0(self):
         self._read_helper(multi_use=True, first=False, count=1)
 
     def test_read_w_multi_use_w_first_w_partition(self):
-        PARTITION = b'DEADBEEF'
+        PARTITION = b'FADEABED'
         self._read_helper(multi_use=True, first=True, partition=PARTITION)
 
     def test_read_w_multi_use_w_first_w_count_gt_0(self):
@@ -322,7 +338,6 @@ def _execute_sql_helper(
         from google.cloud.spanner_v1.proto.type_pb2 import STRING, INT64
         from google.cloud.spanner_v1._helpers import _make_value_pb
 
-        txn_id = b'DEADBEEF'
         VALUES = [
             [u'bharney', u'rhubbyl', 31],
             [u'phred', u'phlyntstone', 32],
@@ -355,7 +370,7 @@ def _execute_sql_helper(
         derived._multi_use = multi_use
         derived._read_request_count = count
         if not first:
-            derived._transaction_id = txn_id
+            derived._transaction_id = TXN_ID
 
         result_set = derived.execute_sql(
             SQL_QUERY_WITH_PARAM, PARAMS, PARAM_TYPES,
@@ -383,7 +398,7 @@ def _execute_sql_helper(
             if first:
                 self.assertTrue(transaction.begin.read_only.strong)
             else:
-                self.assertEqual(transaction.id, txn_id)
+                self.assertEqual(transaction.id, TXN_ID)
         else:
             self.assertTrue(transaction.single_use.read_only.strong)
         expected_params = Struct(fields={
@@ -428,7 +443,6 @@ def _partition_read_helper(
             TransactionSelector)
 
         keyset = KeySet(all_=True)
-        txn_id = b'DEADBEEF'
         new_txn_id = b'ABECAB91'
         token_1 = b'FACE0FFF'
         token_2 = b'BADE8CAF'
@@ -446,7 +460,7 @@ def _partition_read_helper(
         derived = self._makeDerived(session)
         derived._multi_use = multi_use
         if w_txn:
-            derived._transaction_id = txn_id
+            derived._transaction_id = TXN_ID
 
         tokens = list(derived.partition_read(
             TABLE_NAME, COLUMNS, keyset,
@@ -464,7 +478,7 @@ def _partition_read_helper(
         self.assertEqual(table, TABLE_NAME)
         self.assertEqual(key_set, keyset._to_pb())
         self.assertIsInstance(transaction, TransactionSelector)
-        self.assertEqual(transaction.id, txn_id)
+        self.assertEqual(transaction.id, TXN_ID)
         self.assertFalse(transaction.HasField('begin'))
         self.assertEqual(r_index, index)
         self.assertEqual(columns, COLUMNS)
@@ -493,7 +507,7 @@ def test_partition_read_other_error(self):
         session = _Session(database)
         derived = self._makeDerived(session)
         derived._multi_use = True
-        derived._transaction_id = b'DEADBEEF'
+        derived._transaction_id = TXN_ID
 
         with self.assertRaises(RuntimeError):
             list(derived.partition_read(TABLE_NAME, COLUMNS, keyset))
@@ -519,7 +533,6 @@ def _partition_query_helper(
             TransactionSelector)
         from google.cloud.spanner_v1._helpers import _make_value_pb
 
-        txn_id = b'DEADBEEF'
         new_txn_id = b'ABECAB91'
         token_1 = b'FACE0FFF'
         token_2 = b'BADE8CAF'
@@ -537,7 +550,7 @@ def _partition_query_helper(
         derived = self._makeDerived(session)
         derived._multi_use = multi_use
         if w_txn:
-            derived._transaction_id = txn_id
+            derived._transaction_id = TXN_ID
 
         tokens = list(derived.partition_query(
             SQL_QUERY_WITH_PARAM, PARAMS, PARAM_TYPES,
@@ -553,7 +566,7 @@ def _partition_query_helper(
         self.assertEqual(r_session, self.SESSION_NAME)
         self.assertEqual(sql, SQL_QUERY_WITH_PARAM)
         self.assertIsInstance(transaction, TransactionSelector)
-        self.assertEqual(transaction.id, txn_id)
+        self.assertEqual(transaction.id, TXN_ID)
         self.assertFalse(transaction.HasField('begin'))
         expected_params = Struct(fields={
             key: _make_value_pb(value) for (key, value) in PARAMS.items()})
@@ -573,7 +586,7 @@ def test_partition_query_other_error(self):
         session = _Session(database)
         derived = self._makeDerived(session)
         derived._multi_use = True
-        derived._transaction_id = b'DEADBEEF'
+        derived._transaction_id = TXN_ID
 
         with self.assertRaises(RuntimeError):
             list(derived.partition_query(SQL_QUERY))
@@ -583,7 +596,7 @@ def test_partition_query_w_params_wo_param_types(self):
         session = _Session(database)
         derived = self._makeDerived(session)
         derived._multi_use = True
-        derived._transaction_id = b'DEADBEEF'
+        derived._transaction_id = TXN_ID
 
         with self.assertRaises(ValueError):
             list(derived.partition_query(SQL_QUERY_WITH_PARAM, PARAMS))
@@ -616,7 +629,6 @@ class TestSnapshot(unittest.TestCase):
     DATABASE_NAME = INSTANCE_NAME + '/databases/' + DATABASE_ID
     SESSION_ID = 'session-id'
     SESSION_NAME = DATABASE_NAME + '/sessions/' + SESSION_ID
-    TRANSACTION_ID = b'DEADBEEF'
 
     def _getTargetClass(self):
         from google.cloud.spanner_v1.snapshot import Snapshot
@@ -766,9 +778,9 @@ def test_ctor_w_multi_use_and_exact_staleness(self):
     def test__make_txn_selector_w_transaction_id(self):
         session = _Session()
         snapshot = self._make_one(session)
-        snapshot._transaction_id = self.TRANSACTION_ID
+        snapshot._transaction_id = TXN_ID
         selector = snapshot._make_txn_selector()
-        self.assertEqual(selector.id, self.TRANSACTION_ID)
+        self.assertEqual(selector.id, TXN_ID)
 
     def test__make_txn_selector_strong(self):
         session = _Session()
@@ -865,7 +877,7 @@ def test_begin_w_read_request_count_gt_0(self):
     def test_begin_w_existing_txn_id(self):
         session = _Session()
         snapshot = self._make_one(session, multi_use=True)
-        snapshot._transaction_id = self.TRANSACTION_ID
+        snapshot._transaction_id = TXN_ID
         with self.assertRaises(ValueError):
             snapshot.begin()
 
@@ -885,7 +897,7 @@ def test_begin_ok_exact_staleness(self):
         from google.cloud.spanner_v1.proto.transaction_pb2 import (
             Transaction as TransactionPB)
 
-        transaction_pb = TransactionPB(id=self.TRANSACTION_ID)
+        transaction_pb = TransactionPB(id=TXN_ID)
         database = _Database()
         api = database.spanner_api = _FauxSpannerAPI(
             _begin_transaction_response=transaction_pb)
@@ -896,8 +908,8 @@ def test_begin_ok_exact_staleness(self):
 
         txn_id = snapshot.begin()
 
-        self.assertEqual(txn_id, self.TRANSACTION_ID)
-        self.assertEqual(snapshot._transaction_id, self.TRANSACTION_ID)
+        self.assertEqual(txn_id, TXN_ID)
+        self.assertEqual(snapshot._transaction_id, TXN_ID)
 
         session_id, txn_options, metadata = api._begun
         self.assertEqual(session_id, session.name)
@@ -911,7 +923,7 @@ def test_begin_ok_exact_strong(self):
         from google.cloud.spanner_v1.proto.transaction_pb2 import (
             Transaction as TransactionPB)
 
-        transaction_pb = TransactionPB(id=self.TRANSACTION_ID)
+        transaction_pb = TransactionPB(id=TXN_ID)
         database = _Database()
         api = database.spanner_api = _FauxSpannerAPI(
             _begin_transaction_response=transaction_pb)
@@ -920,8 +932,8 @@ def test_begin_ok_exact_strong(self):
 
         txn_id = snapshot.begin()
 
-        self.assertEqual(txn_id, self.TRANSACTION_ID)
-        self.assertEqual(snapshot._transaction_id, self.TRANSACTION_ID)
+        self.assertEqual(txn_id, TXN_ID)
+        self.assertEqual(snapshot._transaction_id, TXN_ID)
 
         session_id, txn_options, metadata = api._begun
         self.assertEqual(session_id, session.name)