Repository: pingcap/tispark (language: python)

Issue:
Compatible with TiDB mater (v6.5.0)
### What problem does this PR solve? <!--add issue link with summary if exists-->

This pr fix three errors:
- SHARD_ROW_ID_BITS can't work with cluster index
- Write (replace=true) with date type commonhandle may fail to do upsert. because we don't handle the date type correctly when build commonhandle
- random tests need to specify non-cluster index for TiDB mater default to cluster index now.

### What is changed and how it works?


### Check List <!--REMOVE the items that are not applicable-->

Tests <!-- At least one of them must be included. -->

 - Unit test
 - Integration test
 - Manual test (add detailed scripts or steps below)
 - No code

Code changes

 - Has exported function/method change
 - Has exported variable/fields change
 - Has interface methods change
 - Has persistent data change

Side effects

 - Possible performance regression
 - Increased code complexity
 - Breaking backward compatibility

Related changes

 - Need to cherry-pick to the release branch
 - Need to update the documentation
 - Need to update the `tidb-ansible` repository
 - Need to be included in the release note



Discussion / hints:
[REVIEW NOTIFICATION]

This pull request has been approved by:

- xuanyu66
- zhangyangyu




To complete the [pull request process](https://book.prow.tidb.net/#/en/workflows/pr), please ask the reviewers in the [list](https://prow.tidb.net/tichi/repos/pingcap/tispark/pulls/2598/owners) to review by filling `/cc @reviewer` in the comment.
After your PR has acquired the required number of LGTMs, you can assign this pull request to the committer in the [list](https://prow.tidb.net/tichi/repos/pingcap/tispark/pulls/2598/owners) by filling  `/assign @committer` in the comment to help you merge this pull request.

The full list of commands accepted by this bot can be found [here](https://prow.tidb.net/command-help?repo=pingcap%2Ftispark).

<details>

Reviewer can indicate their review by submitting an approval review.
Reviewer can cancel approval by submitting a request changes review.
</details>

<!--Review Notification Identifier-->
/run-all-test tidb=v6.1.0 tikv=v6.1.0


/run-all-tests tidb=v6.1.0 tikv=v6.1.0


/run-all-tests
/run-all-tests

/run-all-tests



/run-all-tests



/run-all-tests



/run-all-tests


/run-all-tests

/run-all-tests



/run-all-tests


/merge
This pull request has been accepted and is ready to merge. <details>Commit hash: c2072e2dc705b486d1842445de4fa196d9bfd38c</details>

Fix patch:
diff --git a/tikv-client/src/main/java/com/pingcap/tikv/key/CommonHandle.java b/tikv-client/src/main/java/com/pingcap/tikv/key/CommonHandle.java
index 7769d93085..c77943a41e 100644
--- a/tikv-client/src/main/java/com/pingcap/tikv/key/CommonHandle.java
+++ b/tikv-client/src/main/java/com/pingcap/tikv/key/CommonHandle.java
@@ -19,15 +19,16 @@
 import com.pingcap.tikv.codec.CodecDataInput;
 import com.pingcap.tikv.codec.CodecDataOutput;
 import com.pingcap.tikv.exception.CodecException;
-import com.pingcap.tikv.types.Converter;
 import com.pingcap.tikv.types.DataType;
 import com.pingcap.tikv.types.MySQLType;
 import com.pingcap.tikv.util.FastByteComparisons;
 import java.sql.Date;
 import java.sql.Timestamp;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.TimeZone;
 import java.util.stream.Collectors;
 import org.joda.time.Days;
 import org.joda.time.LocalDate;
@@ -69,16 +70,16 @@ public static CommonHandle newCommonHandle(
         // When indexScan or tableScan, it will pass `Long` object.
         // It's a compromise here since we don't have a good way to make them consistent.
         if (data[i] instanceof Date) {
-          days = Days.daysBetween(new LocalDate(0), new LocalDate(data[i])).getDays();
+          days = Days.daysBetween(new LocalDate(1970, 1, 1), new LocalDate(data[i])).getDays();
         } else {
           days = (long) data[i];
         }
 
-        // Convert to UTC days for row key.
-        if (Converter.getLocalTimezone().getOffset(0) < 0) {
-          days += 1;
-        }
-        dataTypes[i].encode(cdo, DataType.EncodeType.KEY, new Date((days) * MS_OF_ONE_DAY));
+        SimpleDateFormat utcFmt = new SimpleDateFormat("yyyy-MM-dd");
+        utcFmt.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+        dataTypes[i].encode(
+            cdo, DataType.EncodeType.KEY, Date.valueOf(utcFmt.format(days * MS_OF_ONE_DAY)));
       } else {
         if (prefixLengthes[i] > 0 && data[i] instanceof String) {
           String source = (String) data[i];
diff --git a/tikv-client/src/main/java/com/pingcap/tikv/types/DateType.java b/tikv-client/src/main/java/com/pingcap/tikv/types/DateType.java
index 1330badc7a..ebc7626576 100644
--- a/tikv-client/src/main/java/com/pingcap/tikv/types/DateType.java
+++ b/tikv-client/src/main/java/com/pingcap/tikv/types/DateType.java
@@ -29,7 +29,7 @@
 import org.joda.time.LocalDate;
 
 public class DateType extends AbstractDateTimeType {
-  private static final LocalDate EPOCH = new LocalDate(0);
+  private static final LocalDate EPOCH = new LocalDate(1970, 1, 1);
   public static final DateType DATE = new DateType(MySQLType.TypeDate);
   public static final MySQLType[] subTypes = new MySQLType[] {MySQLType.TypeDate};
 
@@ -93,13 +93,8 @@ public String getName() {
   }
 
   public int getDays(LocalDate d) {
-    // count how many days from EPOCH
-    int days = Days.daysBetween(EPOCH, d).getDays();
-    // if the timezone has negative offset, minus one day.
-    if (getTimezone().getOffset(0) < 0) {
-      days -= 1;
-    }
-    return days;
+    // count how many days from EPOCH (UTC)
+    return Days.daysBetween(EPOCH, d).getDays();
   }
 
   /** {@inheritDoc} */


Test patch:
diff --git a/core/src/test/scala/com/pingcap/tispark/datasource/AutoIncrementSuite.scala b/core/src/test/scala/com/pingcap/tispark/datasource/AutoIncrementSuite.scala
index df05a7ac28..38967595c3 100644
--- a/core/src/test/scala/com/pingcap/tispark/datasource/AutoIncrementSuite.scala
+++ b/core/src/test/scala/com/pingcap/tispark/datasource/AutoIncrementSuite.scala
@@ -29,7 +29,7 @@ class AutoIncrementSuite extends BaseBatchWriteTest("test_datasource_auto_increm
     val schema = StructType(List(StructField("j", LongType)))
 
     jdbcUpdate(
-      s"create table $dbtable(i int NOT NULL AUTO_INCREMENT, j int NOT NULL, primary key (i)) SHARD_ROW_ID_BITS=4")
+      s"create table $dbtable(i int NOT NULL AUTO_INCREMENT, j int NOT NULL, primary key (i)/*T![clustered_index] NONCLUSTERED */) SHARD_ROW_ID_BITS=4")
 
     val tiTableInfo = ti.tiSession.getCatalog.getTable(dbPrefix + database, table)
     assert(!tiTableInfo.isPkHandle)
diff --git a/db-random-test/src/main/scala/com/pingcap/tispark/test/generator/IndexColumn.scala b/db-random-test/src/main/scala/com/pingcap/tispark/test/generator/IndexColumn.scala
index 4d64e5e1f7..b419044952 100644
--- a/db-random-test/src/main/scala/com/pingcap/tispark/test/generator/IndexColumn.scala
+++ b/db-random-test/src/main/scala/com/pingcap/tispark/test/generator/IndexColumn.scala
@@ -110,7 +110,9 @@ case class IndexInfo(
     isUnique: Boolean) {
 
   def toString(isClusteredIndex: Boolean): String = {
-    val clusteredIndexStr = if (isClusteredIndex) " /*T![clustered_index] CLUSTERED */" else ""
+    val clusteredIndexStr =
+      if (isClusteredIndex) " /*T![clustered_index] CLUSTERED */"
+      else "/*T![clustered_index] NONCLUSTERED */"
     val indexColumnString = indexColumns.mkString("(", ",", ")")
     if (isPrimary) {
       s"PRIMARY KEY $indexColumnString$clusteredIndexStr"