Repository: duckdb/duckdb (language: python)

Issue:
uhugeints are truncated when imported from json data
### What happens?

Very large numbers in json files are truncated/corrupted when read from json (see the reproduction below).

I'd say it's not really a bug since RFC 7159 allows implementations to set limits on the range and precision of numbers, but it'd be nice not to have to use workarounds such as representing the number as a string since duckdb supports large ints.

### To Reproduce

When reading a large integer as a uhugeint, it gets corrupted (col ends with 82 in the json, with 36 in duckdb):

```sh
echo '{"col": 277447099861456945273576150847928801582 }' | duckdb -c "select col,hex(col) from read_json('/dev/stdin', columns={col: 'uhugeint'})"
┌─────────────────────────────────────────┬──────────────────────────────────┐
│                   col                   │             hex(col)             │
│                 uint128                 │             varchar              │
├─────────────────────────────────────────┼──────────────────────────────────┤
│ 277447099861456950424513728867138011136 │ D0BA5E258FFD00000000000000000000 │
└─────────────────────────────────────────┴──────────────────────────────────┘
```

### OS:

macOS aarm64

### DuckDB Version:

- v0.10.0 20b1486d11
- v0.10.1-dev414 372c514af8 (nightly as of Feb. 27)

### DuckDB Client:

N/A

### Full Name:

Nicolas Even

### Affiliation:

https://chessdata.com

### Have you tried this on the latest [nightly build](https://duckdb.org/docs/installation/?version=main)?

I have tested with a nightly build

### Have you tried the steps to reproduce? Do they include all relevant data and configuration? Does the issue you report still appear there?

- [X] Yes, I have


Discussion / hints:
There is a similar issue with the DuckDB CLI : 

```sql
D select 277447099861456945273576150847928801582,277447099861456945273576150847928801582::uhugeint, '277447099861456945273576150847928801582'::uhugeint;
┌────────────────────────┬──────────────────────────────────────────┬─────────────────────────────────────────────────────────────┐
│ 2.7744709986145695e+38 │ CAST(2.7744709986145695e+38 AS UHUGEINT) │ CAST('277447099861456945273576150847928801582' AS UHUGEINT) │
│         double         │                 uint128                  │                           uint128                           │
├────────────────────────┼──────────────────────────────────────────┼─────────────────────────────────────────────────────────────┤
│ 2.7744709986145695e+38 │  277447099861456950424513728867138011136 │                     277447099861456945273576150847928801582 │
└────────────────────────┴──────────────────────────────────────────┴─────────────────────────────────────────────────────────────┘
```
Thanks, we'll take a look.
This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 30 days.
This issue was closed because it has been stale for 30 days with no activity.

Fix patch:
diff --git a/extension/json/include/json.json b/extension/json/include/json.json
index 4bababcb0699..85fda6852ed5 100644
--- a/extension/json/include/json.json
+++ b/extension/json/include/json.json
@@ -158,7 +158,7 @@
         "id": 116,
         "name": "map_inference_threshold",
         "type": "idx_t",
-        "default": 25
+        "default": 200
       }
     ],
     "constructor": ["$ClientContext", "files", "date_format", "timestamp_format"]
diff --git a/extension/json/include/json_common.hpp b/extension/json/include/json_common.hpp
index ca29cde9a567..1387bf4f5d96 100644
--- a/extension/json/include/json_common.hpp
+++ b/extension/json/include/json_common.hpp
@@ -89,7 +89,8 @@ using json_key_set_t = unordered_set<JSONKey, JSONKeyHash, JSONKeyEquality>;
 struct JSONCommon {
 public:
 	//! Read/Write flags
-	static constexpr auto READ_FLAG = YYJSON_READ_ALLOW_INF_AND_NAN | YYJSON_READ_ALLOW_TRAILING_COMMAS;
+	static constexpr auto READ_FLAG =
+	    YYJSON_READ_ALLOW_INF_AND_NAN | YYJSON_READ_ALLOW_TRAILING_COMMAS | YYJSON_READ_BIGNUM_AS_RAW;
 	static constexpr auto READ_STOP_FLAG = READ_FLAG | YYJSON_READ_STOP_WHEN_DONE;
 	static constexpr auto READ_INSITU_FLAG = READ_STOP_FLAG | YYJSON_READ_INSITU;
 	static constexpr auto WRITE_FLAG = YYJSON_WRITE_ALLOW_INF_AND_NAN;
@@ -102,6 +103,7 @@ struct JSONCommon {
 	static constexpr char const *TYPE_STRING_BIGINT = "BIGINT";
 	static constexpr char const *TYPE_STRING_UBIGINT = "UBIGINT";
 	static constexpr char const *TYPE_STRING_DOUBLE = "DOUBLE";
+	static constexpr char const *TYPE_STRING_HUGEINT = "HUGEINT";
 	static constexpr char const *TYPE_STRING_VARCHAR = "VARCHAR";
 	static constexpr char const *TYPE_STRING_ARRAY = "ARRAY";
 	static constexpr char const *TYPE_STRING_OBJECT = "OBJECT";
@@ -109,23 +111,24 @@ struct JSONCommon {
 	static inline const char *ValTypeToString(yyjson_val *val) {
 		switch (yyjson_get_tag(val)) {
 		case YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE:
-			return JSONCommon::TYPE_STRING_NULL;
+			return TYPE_STRING_NULL;
 		case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NOESC:
 		case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE:
-			return JSONCommon::TYPE_STRING_VARCHAR;
+			return TYPE_STRING_VARCHAR;
 		case YYJSON_TYPE_ARR | YYJSON_SUBTYPE_NONE:
-			return JSONCommon::TYPE_STRING_ARRAY;
+			return TYPE_STRING_ARRAY;
 		case YYJSON_TYPE_OBJ | YYJSON_SUBTYPE_NONE:
-			return JSONCommon::TYPE_STRING_OBJECT;
+			return TYPE_STRING_OBJECT;
 		case YYJSON_TYPE_BOOL | YYJSON_SUBTYPE_TRUE:
 		case YYJSON_TYPE_BOOL | YYJSON_SUBTYPE_FALSE:
-			return JSONCommon::TYPE_STRING_BOOLEAN;
+			return TYPE_STRING_BOOLEAN;
 		case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_UINT:
-			return JSONCommon::TYPE_STRING_UBIGINT;
+			return TYPE_STRING_UBIGINT;
 		case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_SINT:
-			return JSONCommon::TYPE_STRING_BIGINT;
+			return TYPE_STRING_BIGINT;
 		case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_REAL:
-			return JSONCommon::TYPE_STRING_DOUBLE;
+		case YYJSON_TYPE_RAW | YYJSON_SUBTYPE_NONE:
+			return TYPE_STRING_DOUBLE;
 		default:
 			throw InternalException("Unexpected yyjson tag in ValTypeToString");
 		}
@@ -154,6 +157,7 @@ struct JSONCommon {
 		case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_SINT:
 			return LogicalTypeId::BIGINT;
 		case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_REAL:
+		case YYJSON_TYPE_RAW | YYJSON_SUBTYPE_NONE:
 			return LogicalTypeId::DOUBLE;
 		default:
 			throw InternalException("Unexpected yyjson tag in ValTypeToLogicalTypeId");
diff --git a/extension/json/include/json_scan.hpp b/extension/json/include/json_scan.hpp
index 88ad7d8ab053..9c5be6239a48 100644
--- a/extension/json/include/json_scan.hpp
+++ b/extension/json/include/json_scan.hpp
@@ -132,7 +132,7 @@ struct JSONScanData : public TableFunctionData {
 	bool convert_strings_to_integers = false;
 	//! If a struct contains more fields than this threshold with at least 80% similar types,
 	//! we infer it as MAP type
-	idx_t map_inference_threshold = 25;
+	idx_t map_inference_threshold = 200;
 
 	//! All column names (in order)
 	vector<string> names;
diff --git a/extension/json/json_functions/json_transform.cpp b/extension/json/json_functions/json_transform.cpp
index d1f1caa1116f..c37fff3644ad 100644
--- a/extension/json/json_functions/json_transform.cpp
+++ b/extension/json/json_functions/json_transform.cpp
@@ -102,6 +102,7 @@ static inline bool GetValueNumerical(yyjson_val *val, T &result, JSONTransformOp
 	switch (unsafe_yyjson_get_tag(val)) {
 	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NOESC:
 	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE:
+	case YYJSON_TYPE_RAW | YYJSON_SUBTYPE_NONE:
 		success = OP::template Operation<string_t, T>(GetString(val), result, options.strict_cast);
 		break;
 	case YYJSON_TYPE_ARR | YYJSON_SUBTYPE_NONE:
@@ -138,6 +139,7 @@ static inline bool GetValueDecimal(yyjson_val *val, T &result, uint8_t w, uint8_
 	switch (unsafe_yyjson_get_tag(val)) {
 	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NOESC:
 	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE:
+	case YYJSON_TYPE_RAW | YYJSON_SUBTYPE_NONE:
 		success = OP::template Operation<string_t, T>(GetString(val), result, options.parameters, w, s);
 		break;
 	case YYJSON_TYPE_ARR | YYJSON_SUBTYPE_NONE:
@@ -172,6 +174,7 @@ static inline bool GetValueString(yyjson_val *val, yyjson_alc *alc, string_t &re
 	switch (unsafe_yyjson_get_tag(val)) {
 	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NOESC:
 	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE:
+	case YYJSON_TYPE_RAW | YYJSON_SUBTYPE_NONE:
 		result = string_t(unsafe_yyjson_get_str(val), unsafe_yyjson_get_len(val));
 		return true;
 	case YYJSON_TYPE_ARR | YYJSON_SUBTYPE_NONE:
diff --git a/extension/json/serialize_json.cpp b/extension/json/serialize_json.cpp
index ee8b436861fc..b5724e77ed12 100644
--- a/extension/json/serialize_json.cpp
+++ b/extension/json/serialize_json.cpp
@@ -44,7 +44,7 @@ void JSONScanData::Serialize(Serializer &serializer) const {
 	serializer.WritePropertyWithDefault<double>(113, "field_appearance_threshold", field_appearance_threshold, 0.1);
 	serializer.WritePropertyWithDefault<idx_t>(114, "maximum_sample_files", maximum_sample_files, 32);
 	serializer.WritePropertyWithDefault<bool>(115, "convert_strings_to_integers", convert_strings_to_integers, false);
-	serializer.WritePropertyWithDefault<idx_t>(116, "map_inference_threshold", map_inference_threshold, 25);
+	serializer.WritePropertyWithDefault<idx_t>(116, "map_inference_threshold", map_inference_threshold, 200);
 }
 
 unique_ptr<JSONScanData> JSONScanData::Deserialize(Deserializer &deserializer) {
@@ -75,7 +75,7 @@ unique_ptr<JSONScanData> JSONScanData::Deserialize(Deserializer &deserializer) {
 	deserializer.ReadPropertyWithExplicitDefault<double>(113, "field_appearance_threshold", result->field_appearance_threshold, 0.1);
 	deserializer.ReadPropertyWithExplicitDefault<idx_t>(114, "maximum_sample_files", result->maximum_sample_files, 32);
 	deserializer.ReadPropertyWithExplicitDefault<bool>(115, "convert_strings_to_integers", result->convert_strings_to_integers, false);
-	deserializer.ReadPropertyWithExplicitDefault<idx_t>(116, "map_inference_threshold", result->map_inference_threshold, 25);
+	deserializer.ReadPropertyWithExplicitDefault<idx_t>(116, "map_inference_threshold", result->map_inference_threshold, 200);
 	return result;
 }
 


Test patch:
diff --git a/test/sql/json/issues/issue10866.test b/test/sql/json/issues/issue10866.test
new file mode 100644
index 000000000000..6f784ff73f15
--- /dev/null
+++ b/test/sql/json/issues/issue10866.test
@@ -0,0 +1,13 @@
+# name: test/sql/json/issues/issue10866.test
+# description: Test issue 1010866 - uhugeints are truncated when imported from json data
+# group: [issues]
+
+require json
+
+statement ok
+copy (select '{"col": 277447099861456945273576150847928801582}') to '__TEST_DIR__/10866.json' (format csv, quote '', header 0)
+
+query II
+select col, hex(col) from read_json('__TEST_DIR__/10866.json', columns={col: 'uhugeint'})
+----
+277447099861456945273576150847928801582	D0BA5E258FFCFEE8C4619BA0E21A192E
diff --git a/test/sql/json/table/read_json_auto.test_slow b/test/sql/json/table/read_json_auto.test_slow
index e4e404dd0e79..8c127039b80c 100644
--- a/test/sql/json/table/read_json_auto.test_slow
+++ b/test/sql/json/table/read_json_auto.test_slow
@@ -288,13 +288,13 @@ MAP(VARCHAR, STRUCT(a JSON[]))
 
 # Candidate types are properly handled for map inference
 query I
-SELECT distinct typeof(a) FROM read_json_auto('data/json/map_of_dates.jsonl')
+SELECT distinct typeof(a) FROM read_json_auto('data/json/map_of_dates.jsonl', map_inference_threshold=25)
 ----
 MAP(VARCHAR, DATE)
 
 # Mixed candidate types are also handled
 query I
-SELECT distinct typeof(a) FROM read_json_auto('data/json/map_of_mixed_date_timestamps.jsonl')
+SELECT distinct typeof(a) FROM read_json_auto('data/json/map_of_mixed_date_timestamps.jsonl', map_inference_threshold=25)
 ----
 MAP(VARCHAR, VARCHAR)