Repository: snowflakedb/gosnowflake
Pull request #1088 (2024-04-02 05:34:35)

SNOW-1256908 Handle int64 correctly in structured types
### Description

SNOW-1256908 When JSON is being deserialized, by default all numbers are deserialized into into float64 and for int64 precision is lost. This change deserializes to string.

### Checklist
- [x] Created tests which fail without the change (if possible)
- [x] Extended the README / documentation, if necessary


codecov[bot]: ## [Codecov](https://app.codecov.io/gh/snowflakedb/gosnowflake/pull/1088?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=snowflakedb) Report
Attention: Patch coverage is `0%` with `13 lines` in your changes are missing coverage. Please review.
> Project coverage is 82.28%. Comparing base [(`cec541e`)](https://app.codecov.io/gh/snowflakedb/gosnowflake/commit/cec541e1da5475f8ff065195a93c0bca8e09cff7?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=snowflakedb) to head [(`5323dfc`)](https://app.codecov.io/gh/snowflakedb/gosnowflake/pull/1088?dropdown=coverage&src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=snowflakedb).

| [Files](https://app.codecov.io/gh/snowflakedb/gosnowflake/pull/1088?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=snowflakedb) | Patch % | Lines |
|---|---|---|
| [structured\_type.go](https://app.codecov.io/gh/snowflakedb/gosnowflake/pull/1088?src=pr&el=tree&filepath=structured_type.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=snowflakedb#diff-c3RydWN0dXJlZF90eXBlLmdv) | 0.00% | [7 Missing :warning: ](https://app.codecov.io/gh/snowflakedb/gosnowflake/pull/1088?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=snowflakedb) |
| [converter.go](https://app.codecov.io/gh/snowflakedb/gosnowflake/pull/1088?src=pr&el=tree&filepath=converter.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=snowflakedb#diff-Y29udmVydGVyLmdv) | 0.00% | [6 Missing :warning: ](https://app.codecov.io/gh/snowflakedb/gosnowflake/pull/1088?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=snowflakedb) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #1088      +/-   ##
==========================================
- Coverage   82.33%   82.28%   -0.05%     
==========================================
  Files          52       52              
  Lines       10195    10201       +6     
==========================================
  Hits         8394     8394              
- Misses       1801     1807       +6     
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/snowflakedb/gosnowflake/pull/1088?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=snowflakedb).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=snowflakedb).


Patch:
diff --git a/converter.go b/converter.go
index 019587e..cd292a8 100644
--- a/converter.go
+++ b/converter.go
@@ -3,6 +3,7 @@
 package gosnowflake
 
 import (
+	"bytes"
 	"context"
 	"database/sql"
 	"database/sql/driver"
@@ -280,8 +281,9 @@ func stringToValue(dest *driver.Value, srcColumnMeta execResponseRowType, srcVal
 			return nil
 		}
 		m := make(map[string]any)
-		err := json.Unmarshal([]byte(*srcValue), &m)
-		if err != nil {
+		decoder := json.NewDecoder(bytes.NewBufferString(*srcValue))
+		decoder.UseNumber()
+		if err := decoder.Decode(&m); err != nil {
 			return err
 		}
 		*dest = buildStructuredTypeRecursive(m, srcColumnMeta.Fields, params)
@@ -539,8 +541,9 @@ func arrowToValue(rowIdx int, srcColumnMeta execResponseRowType, srcValue arrow.
 			// structured objects as json
 			if !srcValue.IsNull(rowIdx) {
 				m := make(map[string]any)
-				err := json.Unmarshal([]byte(strings.Value(rowIdx)), &m)
-				if err != nil {
+				decoder := json.NewDecoder(bytes.NewBufferString(strings.Value(rowIdx)))
+				decoder.UseNumber()
+				if err := decoder.Decode(&m); err != nil {
 					return nil, err
 				}
 				return buildStructuredTypeRecursive(m, srcColumnMeta.Fields, params), nil
diff --git a/structured_type.go b/structured_type.go
index 247ccc1..ba437f3 100644
--- a/structured_type.go
+++ b/structured_type.go
@@ -3,6 +3,7 @@ package gosnowflake
 import (
 	"database/sql"
 	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"math/big"
@@ -81,10 +82,11 @@ func (st *structuredType) GetInt64(fieldName string) (int64, error) {
 	if b, err := getType[int64](st, fieldName, 0); err == nil {
 		return b, err
 	} else if b, err := getType[string](st, fieldName, ""); err == nil {
-		res, err := strconv.ParseInt(b, 10, 64)
-		return int64(res), err
+		return strconv.ParseInt(b, 10, 64)
 	} else if b, err := getType[float64](st, fieldName, 0); err == nil {
 		return int64(b), nil
+	} else if b, err := getType[json.Number](st, fieldName, ""); err == nil {
+		return strconv.ParseInt(string(b), 10, 64)
 	} else {
 		return 0, fmt.Errorf("cannot cast column %v to byte", fieldName)
 	}
@@ -95,7 +97,7 @@ func (st *structuredType) GetBigInt(fieldName string) (*big.Int, error) {
 }
 
 func (st *structuredType) GetFloat32(fieldName string) (float32, error) {
-	f32, err := getType[float64](st, fieldName, 0)
+	f32, err := st.GetFloat64(fieldName)
 	if err != nil {
 		return 0, err
 	}
@@ -103,6 +105,9 @@ func (st *structuredType) GetFloat32(fieldName string) (float32, error) {
 }
 
 func (st *structuredType) GetFloat64(fieldName string) (float64, error) {
+	if b, err := getType[json.Number](st, fieldName, ""); err == nil {
+		return strconv.ParseFloat(string(b), 64)
+	}
 	return getType[float64](st, fieldName, 0)
 }
 


Test patch:
diff --git a/structured_type_test.go b/structured_type_test.go
index 78946b6..c02b4b8 100644
--- a/structured_type_test.go
+++ b/structured_type_test.go
@@ -105,7 +105,7 @@ func TestStructuredTypeWithAllTypes(t *testing.T) {
 	runDBTest(t, func(dbt *DBTest) {
 		forAllStructureTypeFormats(dbt, func(t *testing.T, format string) {
 			dbt.mustExec("ALTER SESSION SET TIMEZONE = 'Europe/Warsaw'")
-			rows := dbt.mustQuery("SELECT 1, {'s': 'some string', 'b': 1, 'i16': 2, 'i': 3, 'i64': 4, 'f32': '1.1', 'f64': 2.2, 'bo': true, 'bi': TO_BINARY('616263', 'HEX'), 'date': '2024-03-21'::DATE, 'time': '13:03:02'::TIME, 'ltz': '2021-07-21 11:22:33'::TIMESTAMP_LTZ, 'tz': '2022-08-31 13:43:22 +0200'::TIMESTAMP_TZ, 'ntz': '2023-05-22 01:17:19'::TIMESTAMP_NTZ, 'so': {'s': 'child'}}::OBJECT(s VARCHAR, b TINYINT, i16 SMALLINT, i INTEGER, i64 BIGINT, f32 FLOAT, f64 DOUBLE, bo BOOLEAN, bi BINARY, date DATE, time TIME, ltz TIMESTAMP_LTZ, tz TIMESTAMP_TZ, ntz TIMESTAMP_NTZ, so OBJECT(s VARCHAR))")
+			rows := dbt.mustQuery("SELECT 1, {'s': 'some string', 'b': 1, 'i16': 2, 'i': 3, 'i64': 9223372036854775807, 'f32': '1.1', 'f64': 2.2, 'bo': true, 'bi': TO_BINARY('616263', 'HEX'), 'date': '2024-03-21'::DATE, 'time': '13:03:02'::TIME, 'ltz': '2021-07-21 11:22:33'::TIMESTAMP_LTZ, 'tz': '2022-08-31 13:43:22 +0200'::TIMESTAMP_TZ, 'ntz': '2023-05-22 01:17:19'::TIMESTAMP_NTZ, 'so': {'s': 'child'}}::OBJECT(s VARCHAR, b TINYINT, i16 SMALLINT, i INTEGER, i64 BIGINT, f32 FLOAT, f64 DOUBLE, bo BOOLEAN, bi BINARY, date DATE, time TIME, ltz TIMESTAMP_LTZ, tz TIMESTAMP_TZ, ntz TIMESTAMP_NTZ, so OBJECT(s VARCHAR))")
 			defer rows.Close()
 			rows.Next()
 			var ignore int
@@ -116,7 +116,7 @@ func TestStructuredTypeWithAllTypes(t *testing.T) {
 			assertEqualE(t, res.b, byte(1))
 			assertEqualE(t, res.i16, int16(2))
 			assertEqualE(t, res.i, 3)
-			assertEqualE(t, res.i64, int64(4))
+			assertEqualE(t, res.i64, int64(9223372036854775807))
 			assertEqualE(t, res.f32, float32(1.1))
 			assertEqualE(t, res.f64, 2.2)
 			assertEqualE(t, res.bo, true)