Repository: googleapis/google-cloud-python

Issue:
Add system tests for spanner datatypes
Show inserting / reading all the datatypes supported by spanner, particularly:

- Special-case floats (`nan`, `+inf`, `-inf`).
- Arrays of all scalar types.


Discussion:
Working on this right now.

Resolving patch:
diff --git a/spanner/google/cloud/spanner/_helpers.py b/spanner/google/cloud/spanner/_helpers.py
--- a/spanner/google/cloud/spanner/_helpers.py
+++ b/spanner/google/cloud/spanner/_helpers.py
@@ -133,7 +133,10 @@ def _make_value_pb(value):
         if math.isnan(value):
             return Value(string_value='NaN')
         if math.isinf(value):
-            return Value(string_value=str(value))
+            if value > 0:
+                return Value(string_value='Infinity')
+            else:
+                return Value(string_value='-Infinity')
         return Value(number_value=value)
     if isinstance(value, TimestampWithNanoseconds):
         return Value(string_value=value.rfc3339())


Test patch:
diff --git a/spanner/google/cloud/spanner/_fixtures.py b/spanner/tests/_fixtures.py
similarity index 81%
rename from spanner/google/cloud/spanner/_fixtures.py
rename to spanner/tests/_fixtures.py
--- a/spanner/google/cloud/spanner/_fixtures.py
+++ b/spanner/tests/_fixtures.py
@@ -28,6 +28,16 @@
     phone_number STRING(1024) )
     PRIMARY KEY (contact_id, phone_type),
     INTERLEAVE IN PARENT contacts ON DELETE CASCADE;
+CREATE TABLE all_types (
+    list_goes_on ARRAY<INT64>,
+    are_you_sure BOOL,
+    raw_data BYTES(16),
+    hwhen DATE,
+    approx_value FLOAT64,
+    eye_d INT64,
+    description STRING(16),
+    exactly_hwhen TIMESTAMP)
+    PRIMARY KEY (eye_d);
 """
 
 DDL_STATEMENTS = [stmt.strip() for stmt in DDL.split(';') if stmt.strip()]
diff --git a/spanner/tests/system.py b/spanner/tests/system.py
--- a/spanner/tests/system.py
+++ b/spanner/tests/system.py
@@ -12,23 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import datetime
+import math
 import operator
 import os
+import struct
 import unittest
 
 from google.cloud.proto.spanner.v1.type_pb2 import STRING
 from google.cloud.proto.spanner.v1.type_pb2 import Type
+
+from google.cloud._helpers import UTC
+from google.cloud.exceptions import GrpcRendezvous
+from google.cloud.spanner._helpers import TimestampWithNanoseconds
 from google.cloud.spanner.client import Client
 from google.cloud.spanner.keyset import KeySet
 from google.cloud.spanner.pool import BurstyPool
-from google.cloud.spanner._fixtures import DDL_STATEMENTS
-
-from grpc._channel import _Rendezvous
 
 from test_utils.retry import RetryErrors
 from test_utils.retry import RetryInstanceState
 from test_utils.retry import RetryResult
 from test_utils.system import unique_resource_id
+from tests._fixtures import DDL_STATEMENTS
+
 
 IS_CIRCLE = os.getenv('CIRCLECI') == 'true'
 CREATE_INSTANCE = IS_CIRCLE or os.getenv(
@@ -64,9 +70,13 @@ def _has_all_ddl(database):
     return len(database.ddl_statements) == len(DDL_STATEMENTS)
 
 
+def _list_instances():
+    return list(Config.CLIENT.list_instances())
+
+
 def setUpModule():
     Config.CLIENT = Client()
-    retry = RetryErrors(_Rendezvous, error_predicate=_retry_on_unavailable)
+    retry = RetryErrors(GrpcRendezvous, error_predicate=_retry_on_unavailable)
 
     configs = list(retry(Config.CLIENT.list_instance_configs)())
 
@@ -76,9 +86,6 @@ def setUpModule():
     Config.INSTANCE_CONFIG = configs[0]
     config_name = configs[0].name
 
-    def _list_instances():
-        return list(Config.CLIENT.list_instances())
-
     instances = retry(_list_instances)()
     EXISTING_INSTANCES[:] = instances
 
@@ -179,12 +186,37 @@ class _TestData(object):
     ALL = KeySet(all_=True)
     SQL = 'SELECT * FROM contacts ORDER BY contact_id'
 
-    def _check_row_data(self, row_data):
-        self.assertEqual(len(row_data), len(self.ROW_DATA))
-        for found, expected in zip(row_data, self.ROW_DATA):
+    def _assert_timestamp(self, value, nano_value):
+        self.assertIsInstance(value, datetime.datetime)
+        self.assertIsNone(value.tzinfo)
+        self.assertIs(nano_value.tzinfo, UTC)
+
+        self.assertEqual(value.year, nano_value.year)
+        self.assertEqual(value.month, nano_value.month)
+        self.assertEqual(value.day, nano_value.day)
+        self.assertEqual(value.hour, nano_value.hour)
+        self.assertEqual(value.minute, nano_value.minute)
+        self.assertEqual(value.second, nano_value.second)
+        self.assertEqual(value.microsecond, nano_value.microsecond)
+        if isinstance(value, TimestampWithNanoseconds):
+            self.assertEqual(value.nanosecond, nano_value.nanosecond)
+        else:
+            self.assertEqual(value.microsecond * 1000, nano_value.nanosecond)
+
+    def _check_row_data(self, row_data, expected=None):
+        if expected is None:
+            expected = self.ROW_DATA
+
+        self.assertEqual(len(row_data), len(expected))
+        for found, expected in zip(row_data, expected):
             self.assertEqual(len(found), len(expected))
-            for f_cell, e_cell in zip(found, expected):
-                self.assertEqual(f_cell, e_cell)
+            for found_cell, expected_cell in zip(found, expected):
+                if isinstance(found_cell, TimestampWithNanoseconds):
+                    self._assert_timestamp(expected_cell, found_cell)
+                elif isinstance(found_cell, float) and math.isnan(found_cell):
+                    self.assertTrue(math.isnan(expected_cell))
+                else:
+                    self.assertEqual(found_cell, expected_cell)
 
 
 class TestDatabaseAPI(unittest.TestCase, _TestData):
@@ -333,8 +365,49 @@ def test_batch_insert_then_read(self):
         rows = list(snapshot.read(self.TABLE, self.COLUMNS, self.ALL))
         self._check_row_data(rows)
 
-    def test_batch_insert_or_update_then_query(self):
+    def test_batch_insert_then_read_all_datatypes(self):
+        from google.cloud.spanner import KeySet
 
+        keyset = KeySet(all_=True)
+        retry = RetryInstanceState(_has_all_ddl)
+        retry(self._db.reload)()
+
+        session = self._db.session()
+        session.create()
+        self.to_delete.append(session)
+
+        table = 'all_types'
+        columns = (
+            'list_goes_on',
+            'are_you_sure',
+            'raw_data',
+            'hwhen',
+            'approx_value',
+            'eye_d',
+            'description',
+            'exactly_hwhen',
+        )
+        some_date = datetime.date(2011, 1, 17)
+        some_time = datetime.datetime(1989, 1, 17, 17, 59, 12, 345612)
+        nano_time = TimestampWithNanoseconds(1995, 8, 31, nanosecond=987654321)
+        other_nan, = struct.unpack('<d', b'\x01\x00\x01\x00\x00\x00\xf8\xff')
+        row_data = (
+            ([1], True, b'Ymlu', some_date, 0.0, 19, u'dog', some_time),
+            ([5, 10], True, b'Ymlu', None, 1.25, 99, u'cat', None),
+            ([], False, b'Ym9vdHM=', None, float('inf'), 107, u'frog', None),
+            ([], False, None, None, float('-inf'), 207, None, None),
+            ([], False, None, None, float('nan'), 1207, None, None),
+            ([], False, None, None, other_nan, 2000, None, nano_time),
+        )
+        with session.batch() as batch:
+            batch.delete(table, keyset)
+            batch.insert(table, columns, row_data)
+
+        snapshot = session.snapshot(read_timestamp=batch.committed)
+        rows = list(snapshot.read(table, columns, keyset))
+        self._check_row_data(rows, expected=row_data)
+
+    def test_batch_insert_or_update_then_query(self):
         retry = RetryInstanceState(_has_all_ddl)
         retry(self._db.reload)()
 
@@ -349,7 +422,7 @@ def test_batch_insert_or_update_then_query(self):
         rows = list(snapshot.execute_sql(self.SQL))
         self._check_row_data(rows)
 
-    @RetryErrors(exception=_Rendezvous)
+    @RetryErrors(exception=GrpcRendezvous)
     def test_transaction_read_and_insert_then_rollback(self):
         retry = RetryInstanceState(_has_all_ddl)
         retry(self._db.reload)()
@@ -377,7 +450,7 @@ def test_transaction_read_and_insert_then_rollback(self):
         rows = list(session.read(self.TABLE, self.COLUMNS, self.ALL))
         self.assertEqual(rows, [])
 
-    @RetryErrors(exception=_Rendezvous)
+    @RetryErrors(exception=GrpcRendezvous)
     def test_transaction_read_and_insert_or_update_then_commit(self):
         retry = RetryInstanceState(_has_all_ddl)
         retry(self._db.reload)()
@@ -403,12 +476,17 @@ def test_transaction_read_and_insert_or_update_then_commit(self):
         rows = list(session.read(self.TABLE, self.COLUMNS, self.ALL))
         self._check_row_data(rows)
 
-    def _set_up_table(self, row_count):
-        def _row_data(max_index):
-            for index in range(max_index):
-                yield [index, 'First%09d' % (index,), 'Last09%d' % (index),
-                       'test-%09d@example.com' % (index,)]
+    @staticmethod
+    def _row_data(max_index):
+        for index in range(max_index):
+            yield [
+                index,
+                'First%09d' % (index,),
+                'Last09%d' % (index),
+                'test-%09d@example.com' % (index,),
+            ]
 
+    def _set_up_table(self, row_count):
         retry = RetryInstanceState(_has_all_ddl)
         retry(self._db.reload)()
 
@@ -418,7 +496,8 @@ def _row_data(max_index):
 
         with session.transaction() as transaction:
             transaction.delete(self.TABLE, self.ALL)
-            transaction.insert(self.TABLE, self.COLUMNS, _row_data(row_count))
+            transaction.insert(
+                self.TABLE, self.COLUMNS, self._row_data(row_count))
 
         return session, transaction.committed
 
diff --git a/spanner/tests/unit/test__helpers.py b/spanner/tests/unit/test__helpers.py
--- a/spanner/tests/unit/test__helpers.py
+++ b/spanner/tests/unit/test__helpers.py
@@ -190,14 +190,14 @@ def test_w_float_neg_inf(self):
 
         value_pb = self._callFUT(float('-inf'))
         self.assertIsInstance(value_pb, Value)
-        self.assertEqual(value_pb.string_value, '-inf')
+        self.assertEqual(value_pb.string_value, '-Infinity')
 
     def test_w_float_pos_inf(self):
         from google.protobuf.struct_pb2 import Value
 
         value_pb = self._callFUT(float('inf'))
         self.assertIsInstance(value_pb, Value)
-        self.assertEqual(value_pb.string_value, 'inf')
+        self.assertEqual(value_pb.string_value, 'Infinity')
 
     def test_w_date(self):
         import datetime
diff --git a/spanner/tests/unit/test_database.py b/spanner/tests/unit/test_database.py
--- a/spanner/tests/unit/test_database.py
+++ b/spanner/tests/unit/test_database.py
@@ -81,7 +81,7 @@ def test_ctor_w_ddl_statements_w_create_database(self):
                 ddl_statements=['CREATE DATABASE foo'])
 
     def test_ctor_w_ddl_statements_ok(self):
-        from google.cloud.spanner._fixtures import DDL_STATEMENTS
+        from tests._fixtures import DDL_STATEMENTS
 
         instance = _Instance(self.INSTANCE_NAME)
         pool = _Pool()
@@ -297,7 +297,7 @@ def test_create_instance_not_found(self):
                          [('google-cloud-resource-prefix', database.name)])
 
     def test_create_success(self):
-        from google.cloud.spanner._fixtures import DDL_STATEMENTS
+        from tests._fixtures import DDL_STATEMENTS
 
         op_future = _FauxOperationFuture()
         client = _Client()
@@ -360,7 +360,7 @@ def test_exists_not_found(self):
     def test_exists_success(self):
         from google.cloud.proto.spanner.admin.database.v1 import (
             spanner_database_admin_pb2 as admin_v1_pb2)
-        from google.cloud.spanner._fixtures import DDL_STATEMENTS
+        from tests._fixtures import DDL_STATEMENTS
 
         client = _Client()
         ddl_pb = admin_v1_pb2.GetDatabaseDdlResponse(
@@ -417,7 +417,7 @@ def test_reload_not_found(self):
     def test_reload_success(self):
         from google.cloud.proto.spanner.admin.database.v1 import (
             spanner_database_admin_pb2 as admin_v1_pb2)
-        from google.cloud.spanner._fixtures import DDL_STATEMENTS
+        from tests._fixtures import DDL_STATEMENTS
 
         client = _Client()
         ddl_pb = admin_v1_pb2.GetDatabaseDdlResponse(
@@ -439,7 +439,7 @@ def test_reload_success(self):
 
     def test_update_ddl_grpc_error(self):
         from google.gax.errors import GaxError
-        from google.cloud.spanner._fixtures import DDL_STATEMENTS
+        from tests._fixtures import DDL_STATEMENTS
 
         client = _Client()
         api = client.database_admin_api = _FauxDatabaseAdminAPI(
@@ -460,7 +460,7 @@ def test_update_ddl_grpc_error(self):
 
     def test_update_ddl_not_found(self):
         from google.cloud.exceptions import NotFound
-        from google.cloud.spanner._fixtures import DDL_STATEMENTS
+        from tests._fixtures import DDL_STATEMENTS
 
         client = _Client()
         api = client.database_admin_api = _FauxDatabaseAdminAPI(
@@ -480,7 +480,7 @@ def test_update_ddl_not_found(self):
                          [('google-cloud-resource-prefix', database.name)])
 
     def test_update_ddl(self):
-        from google.cloud.spanner._fixtures import DDL_STATEMENTS
+        from tests._fixtures import DDL_STATEMENTS
 
         op_future = _FauxOperationFuture()
         client = _Client()
diff --git a/spanner/tests/unit/test_instance.py b/spanner/tests/unit/test_instance.py
--- a/spanner/tests/unit/test_instance.py
+++ b/spanner/tests/unit/test_instance.py
@@ -503,8 +503,8 @@ def test_database_factory_defaults(self):
         self.assertIs(pool._database, database)
 
     def test_database_factory_explicit(self):
-        from google.cloud.spanner._fixtures import DDL_STATEMENTS
         from google.cloud.spanner.database import Database
+        from tests._fixtures import DDL_STATEMENTS
 
         client = _Client(self.PROJECT)
         instance = self._make_one(self.INSTANCE_ID, client, self.CONFIG_NAME)