Repository: pingcap/tiflow
Pull request #11953 (2024-12-30 10:22:23)

OWNERS: Auto Sync OWNERS files from community membership (#11943)
This is an automated cherry-pick of #11943


### Check List

Tests <!-- At least one of them must be included. -->

- [x] No need to test
  > - [x] I checked and no code files have been changed.
  > <!-- Or your custom  "No need to test" reasons -->



ti-chi-bot[bot]: This cherry pick PR is for a release branch and has not yet been approved by triage owners.
Adding the `do-not-merge/cherry-pick-not-approved` label.

To merge this cherry pick:
1. It must be approved by the approvers firstly.
2. **AFTER** it has been approved by approvers, please wait for the cherry-pick merging approval from triage owners.


<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
ti-chi-bot[bot]: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by:
**Once this PR has been reviewed and has the lgtm label**, please assign [lance6716](https://github.com/lance6716) for approval. For more information see [the Code Review Process](https://book.prow.tidb.net/#/workflows/pr).

The full list of commands accepted by this bot can be found [here](https://prow.tidb.net/command-help?repo=pingcap%2Ftiflow).

<details open>
Needs approval from an approver in each of these files:

- **[OWNERS](https://github.com/pingcap/tiflow/blob/release-6.5/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":["lance6716"]} -->
ti-chi-bot[bot]: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/pingcap/tiflow/pull/11953#pullrequestreview-2525422188" title="Approved">asddongmen</a>*

The full list of commands accepted by this bot can be found [here](https://prow.tidb.net/command-help?repo=pingcap%2Ftiflow).

The pull request process is described [here](https://book.prow.tidb.net/#/workflows/pr)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/pingcap/tiflow/blob/release-6.5/OWNERS)~~ [asddongmen]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->

Patch:
diff --git a/OWNERS b/OWNERS
index 6dfab384e..f12d5ff1a 100644
--- a/OWNERS
+++ b/OWNERS
@@ -29,6 +29,7 @@ approvers:
   - sdojjy
   - suzaku
   - wk989898
+  - wlwilliamx
   - YuJuncen
 reviewers:
   # - migration-reviewers
diff --git a/cdc/sink/codec/avro/avro.go b/cdc/sink/codec/avro/avro.go
index ca1ddc612..f396719bc 100644
--- a/cdc/sink/codec/avro/avro.go
+++ b/cdc/sink/codec/avro/avro.go
@@ -401,8 +401,18 @@ func rowToAvroSchema(
 			}
 		} else {
 			if col.Flag.IsNullable() {
+				// the string literal "null" must be coerced to a `nil`
+				// see https://github.com/linkedin/goavro/blob/5ec5a5ee7ec82e16e6e2b438d610e1cab2588393/record.go#L109-L114
 				// https://stackoverflow.com/questions/22938124/avro-field-default-values
+				defaultFirst := false
 				if defaultValue == nil {
+					defaultFirst = true
+				} else if s, ok := defaultValue.(string); ok && s == "null" {
+					defaultFirst = true
+				} else if b, ok := defaultValue.([]byte); ok && string(b) == "null" {
+					defaultFirst = true
+				}
+				if defaultFirst {
 					field["type"] = []interface{}{"null", avroType}
 				} else {
 					field["type"] = []interface{}{avroType, "null"}
diff --git a/pkg/etcd/client.go b/pkg/etcd/client.go
index e63f77579..d00521793 100644
--- a/pkg/etcd/client.go
+++ b/pkg/etcd/client.go
@@ -358,6 +358,7 @@ func newClient(tlsConfig *tls.Config, grpcDialOption grpc.DialOption, endpoints
 		TLS:                  tlsConfig,
 		LogConfig:            &logConfig,
 		DialTimeout:          defaultEtcdClientTimeout,
+		AutoSyncInterval:     30 * time.Second,
 		DialKeepAliveTime:    defaultDialKeepAliveTime,
 		DialKeepAliveTimeout: defaultDialKeepAliveTimeout,
 		DialOptions: []grpc.DialOption{
diff --git a/pkg/filter/sql_event_filter.go b/pkg/filter/sql_event_filter.go
index 39235eb05..7e929878d 100644
--- a/pkg/filter/sql_event_filter.go
+++ b/pkg/filter/sql_event_filter.go
@@ -166,6 +166,18 @@ func (f *sqlEventFilter) shouldSkipDDL(ddl *model.DDLEvent) (bool, error) {
 	if err != nil {
 		return false, err
 	}
+	if evenType == bf.RenameTable {
+		if ddl.PreTableInfo == nil {
+			log.Warn("sql event filter doesn't find old table info when the event type is `rename table`",
+				zap.String("schema", schema),
+				zap.String("table", table),
+				zap.Stringer("type", ddl.Type),
+				zap.String("query", ddl.Query))
+			return true, cerror.ErrTableIneligible.GenWithStackByArgs(ddl.TableInfo.TableName)
+		}
+		schema = ddl.PreTableInfo.TableName.Schema
+		table = ddl.PreTableInfo.TableName.Table
+	}
 	if evenType == bf.NullEvent {
 		log.Warn("sql event filter unsupported ddl type, do nothing",
 			zap.String("type", ddl.Type.String()),


Test patch:
diff --git a/pkg/filter/sql_event_filter_test.go b/pkg/filter/sql_event_filter_test.go
index a564e29d7..ec5d59521 100644
--- a/pkg/filter/sql_event_filter_test.go
+++ b/pkg/filter/sql_event_filter_test.go
@@ -18,6 +18,7 @@ import (
 
 	"github.com/pingcap/errors"
 	bf "github.com/pingcap/tidb-tools/pkg/binlog-filter"
+	timodel "github.com/pingcap/tidb/parser/model"
 	"github.com/pingcap/tiflow/cdc/model"
 	"github.com/pingcap/tiflow/pkg/config"
 	cerror "github.com/pingcap/tiflow/pkg/errors"
@@ -27,10 +28,13 @@ import (
 func TestShouldSkipDDL(t *testing.T) {
 	t.Parallel()
 	type innerCase struct {
-		schema string
-		table  string
-		query  string
-		skip   bool
+		schema    string
+		table     string
+		preSchema string
+		preTable  string
+		query     string
+		ddlType   timodel.ActionType
+		skip      bool
 	}
 
 	type testCase struct {
@@ -51,28 +55,32 @@ func TestShouldSkipDDL(t *testing.T) {
 			},
 			cases: []innerCase{
 				{
-					schema: "test",
-					table:  "t1",
-					query:  "alter table t1 modify column age int",
-					skip:   true,
+					schema:  "test",
+					table:   "t1",
+					query:   "alter table t1 modify column age int",
+					ddlType: timodel.ActionModifyColumn,
+					skip:    true,
 				},
 				{
-					schema: "test",
-					table:  "t1",
-					query:  "create table t1(id int primary key)",
-					skip:   true,
+					schema:  "test",
+					table:   "t1",
+					query:   "create table t1(id int primary key)",
+					ddlType: timodel.ActionCreateTable,
+					skip:    true,
 				},
 				{
-					schema: "test",
-					table:  "t2", // table name not match
-					query:  "alter table t2 modify column age int",
-					skip:   false,
+					schema:  "test",
+					table:   "t2", // table name not match
+					query:   "alter table t2 modify column age int",
+					ddlType: timodel.ActionModifyColumn,
+					skip:    false,
 				},
 				{
-					schema: "test2", // schema name not match
-					table:  "t1",
-					query:  "alter table t1 modify column age int",
-					skip:   false,
+					schema:  "test2", // schema name not match
+					table:   "t1",
+					query:   "alter table t1 modify column age int",
+					ddlType: timodel.ActionModifyColumn,
+					skip:    false,
 				},
 			},
 		},
@@ -87,28 +95,32 @@ func TestShouldSkipDDL(t *testing.T) {
 			},
 			cases: []innerCase{
 				{
-					schema: "test",
-					table:  "t1",
-					query:  "alter table t1 modify column age int",
-					skip:   false,
+					schema:  "test",
+					table:   "t1",
+					query:   "alter table t1 modify column age int",
+					ddlType: timodel.ActionModifyColumn,
+					skip:    false,
 				},
 				{
-					schema: "test",
-					table:  "t1",
-					query:  "alter table t1 drop column age",
-					skip:   false,
+					schema:  "test",
+					table:   "t1",
+					query:   "alter table t1 drop column age",
+					ddlType: timodel.ActionModifyColumn,
+					skip:    false,
 				},
 				{
-					schema: "test2",
-					table:  "t1",
-					query:  "drop database test2",
-					skip:   true,
+					schema:  "test2",
+					table:   "t1",
+					query:   "drop database test2",
+					ddlType: timodel.ActionDropSchema,
+					skip:    true,
 				},
 				{
-					schema: "test3",
-					table:  "t1",
-					query:  "drop index i3 on t1",
-					skip:   false,
+					schema:  "test3",
+					table:   "t1",
+					query:   "drop index i3 on t1",
+					ddlType: timodel.ActionDropIndex,
+					skip:    false,
 				},
 			},
 		},
@@ -123,27 +135,31 @@ func TestShouldSkipDDL(t *testing.T) {
 			},
 			cases: []innerCase{
 				{
-					schema: "test",
-					table:  "t1",
-					query:  "ALTER TABLE t1 MODIFY COLUMN age int(11) NOT NULL",
-					skip:   true,
+					schema:  "test",
+					table:   "t1",
+					query:   "ALTER TABLE t1 MODIFY COLUMN age int(11) NOT NULL",
+					ddlType: timodel.ActionModifyColumn,
+					skip:    true,
 				},
 				{
-					schema: "test",
-					table:  "t1",
-					query:  "ALTER TABLE t1 DROP COLUMN age",
-					skip:   true,
+					schema:  "test",
+					table:   "t1",
+					query:   "ALTER TABLE t1 DROP COLUMN age",
+					ddlType: timodel.ActionDropColumn,
+					skip:    true,
 				},
 				{ // no table name
-					schema: "test2",
-					query:  "DROP DATABASE test",
-					skip:   true,
+					schema:  "test2",
+					query:   "DROP DATABASE test",
+					ddlType: timodel.ActionDropSchema,
+					skip:    true,
 				},
 				{
-					schema: "test3",
-					table:  "t1",
-					query:  "Drop Index i1 on test3.t1",
-					skip:   false,
+					schema:  "test3",
+					table:   "t1",
+					query:   "Drop Index i1 on test3.t1",
+					ddlType: timodel.ActionDropIndex,
+					skip:    false,
 				},
 			},
 		},
@@ -168,6 +184,47 @@ func TestShouldSkipDDL(t *testing.T) {
 				},
 			},
 		},
+		// rename table
+		{
+			cfg: &config.FilterConfig{
+				Rules: []string{"*.t1", "*.t2"},
+				EventFilters: []*config.EventFilterRule{
+					{
+						Matcher:     []string{"*.t1"},
+						IgnoreEvent: []bf.EventType{bf.AllDDL},
+					},
+				},
+			},
+			cases: []innerCase{
+				{
+					schema:    "test",
+					table:     "t2",
+					preSchema: "test",
+					preTable:  "t1",
+					query:     "RENAME TABLE t1 to t2",
+					ddlType:   timodel.ActionRenameTable,
+					skip:      true,
+				},
+				{
+					schema:    "test",
+					table:     "t3",
+					preSchema: "test",
+					preTable:  "t1",
+					query:     "RENAME TABLE t1 to t3",
+					ddlType:   timodel.ActionRenameTable,
+					skip:      true,
+				},
+				{
+					schema:    "test",
+					table:     "t1",
+					preSchema: "test",
+					preTable:  "t2",
+					query:     "RENAME TABLE t2 to t1",
+					ddlType:   timodel.ActionRenameTable,
+					skip:      false,
+				},
+			},
+		},
 	}
 
 	for _, tc := range testCases {
@@ -181,7 +238,14 @@ func TestShouldSkipDDL(t *testing.T) {
 						Table:  c.table,
 					},
 				},
+				PreTableInfo: &model.TableInfo{
+					TableName: model.TableName{
+						Schema: c.preSchema,
+						Table:  c.preTable,
+					},
+				},
 				Query: c.query,
+				Type:  c.ddlType,
 			}
 			skip, err := f.shouldSkipDDL(ddl)
 			require.NoError(t, err)
diff --git a/tests/integration_tests/event_filter/conf/cf.toml b/tests/integration_tests/event_filter/conf/cf.toml
index d3a36188b..4597fe3e4 100644
--- a/tests/integration_tests/event_filter/conf/cf.toml
+++ b/tests/integration_tests/event_filter/conf/cf.toml
@@ -13,3 +13,7 @@ ignore-event = ["truncate table"]
 [[filter.event-filters]]
 matcher = ["event_filter.t_alter"]
 ignore-event = ["alter table"]
+
+[[filter.event-filters]]
+matcher = ["event_filter.t_name*"]
+ignore-event = ["rename table"]
\ No newline at end of file
diff --git a/tests/integration_tests/event_filter/conf/diff_config.toml b/tests/integration_tests/event_filter/conf/diff_config.toml
index fff11e2ee..ae516cbae 100644
--- a/tests/integration_tests/event_filter/conf/diff_config.toml
+++ b/tests/integration_tests/event_filter/conf/diff_config.toml
@@ -13,7 +13,7 @@ source-instances = ["tidb0"]
 
 target-instance = "mysql1"
 
-target-check-tables = ["event_filter.t_normal", "event_filter.t_truncate", "event_filter.t_alter"]
+target-check-tables = ["event_filter.t_normal", "event_filter.t_truncate", "event_filter.t_alter", "event_filter.t_name*", "event_filter.t_rename*"]
 
 [data-sources]
 [data-sources.tidb0]
diff --git a/tests/integration_tests/event_filter/data/test.sql b/tests/integration_tests/event_filter/data/test.sql
index 6af0efed2..5ffd396ae 100644
--- a/tests/integration_tests/event_filter/data/test.sql
+++ b/tests/integration_tests/event_filter/data/test.sql
@@ -89,4 +89,25 @@ CREATE TABLE t_alter
 ) ENGINE = InnoDB
   DEFAULT CHARSET = utf8
   COLLATE = utf8_bin;
-  
\ No newline at end of file
+  
+
+CREATE TABLE t_name (
+                    id INT,
+                    name varchar(128),
+                    PRIMARY KEY (id)
+);
+CREATE TABLE t_name1 (
+                    id INT,
+                    name varchar(128),
+                    PRIMARY KEY (id)
+);
+CREATE TABLE t_name2 (
+                    id INT,
+                    name varchar(128),
+                    PRIMARY KEY (id)
+);
+CREATE TABLE t_name3 (
+                    id INT,
+                    name varchar(128),
+                    PRIMARY KEY (id)
+);
\ No newline at end of file
diff --git a/tests/integration_tests/event_filter/data/test_rename.sql b/tests/integration_tests/event_filter/data/test_rename.sql
new file mode 100644
index 000000000..01df4e517
--- /dev/null
+++ b/tests/integration_tests/event_filter/data/test_rename.sql
@@ -0,0 +1,27 @@
+USE `event_filter`;
+
+RENAME TABLE t_name TO t_rename;
+
+INSERT INTO t_rename
+VALUES (1, 'guagua');
+
+INSERT INTO t_rename
+VALUES (2, 'huahua');
+
+INSERT INTO t_rename
+VALUES (3, 'xigua');
+
+INSERT INTO t_rename
+VALUES (4, 'yuko');
+
+-- rename tables
+RENAME TABLE t_name1 TO t_rename1, t_name2 TO t_rename2, t_name3 TO t_rename3;
+
+INSERT INTO t_rename1
+VALUES (1, 'guagua');
+
+INSERT INTO t_rename2
+VALUES (2, 'huahua');
+
+INSERT INTO t_rename3
+VALUES (3, 'xigua');
diff --git a/tests/integration_tests/event_filter/run.sh b/tests/integration_tests/event_filter/run.sh
index 6e9385052..ff4dbce92 100644
--- a/tests/integration_tests/event_filter/run.sh
+++ b/tests/integration_tests/event_filter/run.sh
@@ -37,6 +37,10 @@ function run() {
 	check_table_exists "event_filter.t_normal" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
 	check_table_exists "event_filter.t_truncate" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
 	check_table_exists "event_filter.t_alter" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
+	check_table_exists "event_filter.t_name" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
+	check_table_exists "event_filter.t_name1" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
+	check_table_exists "event_filter.t_name2" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
+	check_table_exists "event_filter.t_name3" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
 
 	# check those rows that are not filtered are synced to downstream
 	run_sql "select count(1) from event_filter.t1;" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
@@ -54,6 +58,11 @@ function run() {
 	run_sql_file $CUR/data/test_truncate.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT}
 	run_sql "ALTER TABLE event_filter.t_alter MODIFY t_bigint BIGINT;" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
 	run_sql_file $CUR/data/test_alter.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT}
+	run_sql "RENAME TABLE event_filter.t_name TO event_filter.t_rename;" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
+	run_sql "RENAME TABLE event_filter.t_name1 TO event_filter.t_rename1;" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
+	run_sql "RENAME TABLE event_filter.t_name2 TO event_filter.t_rename2;" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
+	run_sql "RENAME TABLE event_filter.t_name3 TO event_filter.t_rename3;" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
+	run_sql_file $CUR/data/test_rename.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT}
 	run_sql "create table event_filter.finish_mark(id int primary key);" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
 	check_table_exists "event_filter.finish_mark" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}