Repository: pingcap/tispark (language: python)

Issue:
Fix TiFlash overflow
### What problem does this PR solve? <!--add issue link with summary if exists-->

Reproduce:

create and insert
```
CREATE TABLE test.t(`a` bigint(20) UNSIGNED NOT NULL)
insert into t values (16717361816800086255)
```

query from tispark with tiflash returns an error result（negative number）
```
 spark.conf
        .set("spark.tispark.isolation_read_engines","tiflash")
spark.sql("select * from tidb_catalog.test.t").show()
```
### What is changed and how it works?

TiSpark requests TiFlash using `TypeCHBlock` encode, but forgets to handle the UNSIGNED scenes when decoded. 

this pr shows how to  decode it corretly


### Check List <!--REMOVE the items that are not applicable-->

Tests <!-- At least one of them must be included. -->

 - Unit test
 - Integration test
 - Manual test (add detailed scripts or steps below)
 - No code

Code changes

 - Has exported function/method change
 - Has exported variable/fields change
 - Has interface methods change
 - Has persistent data change

Side effects

 - Possible performance regression
 - Increased code complexity
 - Breaking backward compatibility

Related changes

 - Need to cherry-pick to the release branch
 - Need to update the documentation
 - Need to update the `tidb-ansible` repository
 - Need to be included in the release note



Discussion / hints:

Fix patch:
diff --git a/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java b/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java
index 12486e2728..6c60683fbb 100644
--- a/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java
+++ b/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java
@@ -18,6 +18,7 @@
 
 import static com.pingcap.tikv.util.MemoryUtil.EMPTY_BYTE_BUFFER_DIRECT;
 
+import com.google.common.primitives.UnsignedLong;
 import com.pingcap.tikv.codec.Codec.DateCodec;
 import com.pingcap.tikv.codec.Codec.DateTimeCodec;
 import com.pingcap.tikv.codec.ExtendedDateTime;
@@ -25,6 +26,7 @@
 import com.pingcap.tikv.types.AbstractDateTimeType;
 import com.pingcap.tikv.types.BytesType;
 import com.pingcap.tikv.types.DateType;
+import com.pingcap.tikv.types.DecimalType;
 import com.pingcap.tikv.util.MemoryUtil;
 import java.math.BigDecimal;
 import java.nio.ByteBuffer;
@@ -240,6 +242,10 @@ public double getDouble(int rowId) {
   @Override
   public BigDecimal getDecimal(int rowId, int precision, int scale) {
     long rowIdAddr = (long) rowId * fixedLength + dataAddr;
+    // avoid unsigned long overflow here
+    if (type == DecimalType.BIG_INT_DECIMAL) {
+      return new BigDecimal(UnsignedLong.fromLongBits(this.getLong(rowId)).bigIntegerValue());
+    }
     if (fixedLength == 4) {
       return MemoryUtil.getDecimal32(rowIdAddr, scale);
     } else if (fixedLength == 8) {


Test patch:
diff --git a/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala b/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala
index 38bac47e56..2b6eb22adb 100644
--- a/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala
+++ b/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala
@@ -22,6 +22,33 @@ import org.apache.spark.sql.functions.{col, sum}
 
 class IssueTestSuite extends BaseTiSparkTest {
 
+  test("test tiflash overflow in unsigned bigint") {
+    if (!enableTiFlashTest) {
+      cancel("tiflash test not enabled")
+    }
+    val dbTable = "tispark_test.tiflash_overflow"
+    tidbStmt.execute(s"drop table if exists $dbTable")
+    tidbStmt.execute(
+      s"CREATE TABLE $dbTable (`a` bigint(20) UNSIGNED NOT NULL,`b` bigint(20) UNSIGNED NOT NULL)")
+    tidbStmt.execute(s"insert into $dbTable values(16717361816800086255, 16717361816800086255)")
+    tidbStmt.execute(s"ALTER TABLE $dbTable SET TIFLASH REPLICA 1")
+
+    Thread.sleep(5 * 1000)
+
+    val prev = spark.conf.getOption(TiConfigConst.ISOLATION_READ_ENGINES)
+    try {
+      spark.conf
+        .set(TiConfigConst.ISOLATION_READ_ENGINES, TiConfigConst.TIFLASH_STORAGE_ENGINE)
+      val df = spark.sql(s"select * from $dbTable")
+      val row = Row(BigDecimal("16717361816800086255"), BigDecimal("16717361816800086255"))
+      checkAnswer(df, Seq(row))
+    } finally {
+      spark.conf.set(
+        TiConfigConst.ISOLATION_READ_ENGINES,
+        prev.getOrElse(TiConfigConst.DEFAULT_STORAGE_ENGINES))
+    }
+  }
+
   test("test issue 2649") {
     val dbTable = "tispark_test.mutil_uniq"
     tidbStmt.execute(s"drop table if exists $dbTable")