Repository: AfterShip/clickhouse-sql-parser
Pull request #73 (2024-07-09 22:50:33)

Handling Codec(Delta, ZSTD(1))
This PR is handling the case when `codec` has `Delta` argument.

https://github.com/AfterShip/clickhouse-sql-parser/issues/72


Patch:
diff --git a/parser/ast.go b/parser/ast.go
index 399e931..fbec71c 100644
--- a/parser/ast.go
+++ b/parser/ast.go
@@ -567,7 +567,7 @@ func (a *AlterTableAddIndex) AlterType() string {
 
 func (a *AlterTableAddIndex) String(level int) string {
 	var builder strings.Builder
-	builder.WriteString("ADD INDEX ")
+	builder.WriteString("ADD ")
 	builder.WriteString(a.Index.String(level))
 	if a.IfNotExists {
 		builder.WriteString("IF NOT EXISTS ")
@@ -1271,8 +1271,11 @@ func (a *TableIndex) End() Pos {
 
 func (a *TableIndex) String(level int) string {
 	var builder strings.Builder
+	builder.WriteString("INDEX")
+	builder.WriteByte(' ')
 	builder.WriteString(a.Name.String(0))
 	builder.WriteString(a.ColumnExpr.String(level))
+	builder.WriteByte(' ')
 	builder.WriteString("TYPE")
 	builder.WriteByte(' ')
 	builder.WriteString(a.ColumnType.String(level))
@@ -3265,6 +3268,7 @@ func (n *NestedTypeExpr) Accept(visitor ASTVisitor) error {
 type CompressionCodec struct {
 	CodecPos      Pos
 	RightParenPos Pos
+	Type          *Ident
 	Name          *Ident
 	Level         *NumberLiteral // compression level
 }
@@ -3280,6 +3284,11 @@ func (c *CompressionCodec) End() Pos {
 func (c *CompressionCodec) String(level int) string {
 	var builder strings.Builder
 	builder.WriteString("CODEC(")
+	if c.Type != nil {
+		builder.WriteString(c.Type.String(level))
+		builder.WriteByte(',')
+		builder.WriteByte(' ')
+	}
 	builder.WriteString(c.Name.String(level))
 	if c.Level != nil {
 		builder.WriteByte('(')
diff --git a/parser/parser_column.go b/parser/parser_column.go
index 38d226d..d1b801d 100644
--- a/parser/parser_column.go
+++ b/parser/parser_column.go
@@ -801,6 +801,19 @@ func (p *Parser) tryParseCompressionCodecs(pos Pos) (*CompressionCodec, error) {
 	if err != nil {
 		return nil, err
 	}
+	// parse DELTA if  CODEC(Delta, ZSTD(1))
+	var codecType *Ident
+	if strings.ToUpper(name.Name) == "DELTA" {
+		codecType = name
+		if _, err := p.consumeTokenKind(","); err != nil {
+			return nil, err
+		}
+		name, err = p.parseIdent()
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	var level *NumberLiteral
 	// TODO: check if the codec name is valid
 	switch strings.ToUpper(name.Name) {
@@ -819,6 +832,7 @@ func (p *Parser) tryParseCompressionCodecs(pos Pos) (*CompressionCodec, error) {
 	return &CompressionCodec{
 		CodecPos:      pos,
 		RightParenPos: rightParenPos,
+		Type:          codecType,
 		Name:          name,
 		Level:         level,
 	}, nil


Test patch:
diff --git a/parser/testdata/ddl/create_table_with_codec_delta.sql b/parser/testdata/ddl/create_table_with_codec_delta.sql
new file mode 100644
index 0000000..322e9e4
--- /dev/null
+++ b/parser/testdata/ddl/create_table_with_codec_delta.sql
@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS test_local
+(
+ `id` UInt64 CODEC(Delta, ZSTD(1)),
+ `api_id` UInt64 CODEC(ZSTD(1)),
+ `timestamp` DateTime64(9) CODEC(ZSTD(1)),
+ INDEX timestamp_index(timestamp) TYPE minmax GRANULARITY 4
+)
+ENGINE = ReplicatedMergeTree('/root/test_local', '{replica}')
+PARTITION BY toStartOfHour(`timestamp`)
+ORDER BY (toUnixTimestamp64Nano(`timestamp`), `api_id`)
+TTL toStartOfHour(`timestamp`) + INTERVAL 7 DAY,toStartOfHour(`timestamp`) + INTERVAL 2 DAY
+SETTINGS execute_merges_on_single_replica_time_threshold=1200, index_granularity=16384, max_bytes_to_merge_at_max_space_in_pool=64424509440, storage_policy='main', ttl_only_drop_parts=1;
diff --git a/parser/testdata/ddl/format/alter_table_add_index.sql b/parser/testdata/ddl/format/alter_table_add_index.sql
index 6c58e32..7648deb 100644
--- a/parser/testdata/ddl/format/alter_table_add_index.sql
+++ b/parser/testdata/ddl/format/alter_table_add_index.sql
@@ -5,4 +5,4 @@ ALTER TABLE test.events_local ON CLUSTER 'default_cluster' ADD INDEX my_index(f0
 -- Format SQL:
 ALTER TABLE test.events_local
 ON CLUSTER 'default_cluster'
-ADD INDEX my_index(f0)TYPE minmax GRANULARITY 1024;
+ADD INDEX my_index(f0) TYPE minmax GRANULARITY 1024;
diff --git a/parser/testdata/ddl/format/create_table_with_codec_delta.sql b/parser/testdata/ddl/format/create_table_with_codec_delta.sql
new file mode 100644
index 0000000..a458de5
--- /dev/null
+++ b/parser/testdata/ddl/format/create_table_with_codec_delta.sql
@@ -0,0 +1,28 @@
+-- Origin SQL:
+CREATE TABLE IF NOT EXISTS test_local
+(
+ `id` UInt64 CODEC(Delta, ZSTD(1)),
+ `api_id` UInt64 CODEC(ZSTD(1)),
+ `timestamp` DateTime64(9) CODEC(ZSTD(1)),
+ INDEX timestamp_index(timestamp) TYPE minmax GRANULARITY 4
+)
+ENGINE = ReplicatedMergeTree('/root/test_local', '{replica}')
+PARTITION BY toStartOfHour(`timestamp`)
+ORDER BY (toUnixTimestamp64Nano(`timestamp`), `api_id`)
+TTL toStartOfHour(`timestamp`) + INTERVAL 7 DAY,toStartOfHour(`timestamp`) + INTERVAL 2 DAY
+SETTINGS execute_merges_on_single_replica_time_threshold=1200, index_granularity=16384, max_bytes_to_merge_at_max_space_in_pool=64424509440, storage_policy='main', ttl_only_drop_parts=1;
+
+
+-- Format SQL:
+CREATE TABLE IF NOT EXISTS test_local
+(
+  `id` UInt64 CODEC(Delta, ZSTD(1)),
+  `api_id` UInt64 CODEC(ZSTD(1)),
+  `timestamp` DateTime64(9) CODEC(ZSTD(1)),
+  INDEX timestamp_index(timestamp) TYPE minmax GRANULARITY 4
+)
+ENGINE = ReplicatedMergeTree('/root/test_local', '{replica}')
+PARTITION BY toStartOfHour(`timestamp`)
+TTL toStartOfHour(`timestamp`) + INTERVAL 7 DAY,toStartOfHour(`timestamp`) + INTERVAL 2 DAY
+SETTINGS execute_merges_on_single_replica_time_threshold=1200, index_granularity=16384, max_bytes_to_merge_at_max_space_in_pool=64424509440, storage_policy='main', ttl_only_drop_parts=1
+ORDER BY (toUnixTimestamp64Nano(`timestamp`), `api_id`);
diff --git a/parser/testdata/ddl/output/create_table_basic.sql.golden.json b/parser/testdata/ddl/output/create_table_basic.sql.golden.json
index f73c460..d928845 100644
--- a/parser/testdata/ddl/output/create_table_basic.sql.golden.json
+++ b/parser/testdata/ddl/output/create_table_basic.sql.golden.json
@@ -77,6 +77,7 @@
           "Codec": {
             "CodecPos": 198,
             "RightParenPos": 212,
+            "Type": null,
             "Name": {
               "Name": "ZSTD",
               "QuoteType": 1,
diff --git a/parser/testdata/ddl/output/create_table_with_codec_delta.sql.golden.json b/parser/testdata/ddl/output/create_table_with_codec_delta.sql.golden.json
new file mode 100644
index 0000000..1bced6d
--- /dev/null
+++ b/parser/testdata/ddl/output/create_table_with_codec_delta.sql.golden.json
@@ -0,0 +1,522 @@
+[
+  {
+    "CreatePos": 0,
+    "StatementEnd": 649,
+    "Name": {
+      "Database": null,
+      "Table": {
+        "Name": "test_local",
+        "QuoteType": 1,
+        "NamePos": 27,
+        "NameEnd": 37
+      }
+    },
+    "IfNotExists": true,
+    "UUID": null,
+    "OnCluster": null,
+    "TableSchema": {
+      "SchemaPos": 38,
+      "SchemaEnd": 212,
+      "Columns": [
+        {
+          "NamePos": 42,
+          "ColumnEnd": 74,
+          "Name": {
+            "Ident": {
+              "Name": "id",
+              "QuoteType": 3,
+              "NamePos": 42,
+              "NameEnd": 44
+            },
+            "DotIdent": null
+          },
+          "Type": {
+            "Name": {
+              "Name": "UInt64",
+              "QuoteType": 1,
+              "NamePos": 46,
+              "NameEnd": 52
+            }
+          },
+          "NotNull": null,
+          "Nullable": null,
+          "Property": null,
+          "Codec": {
+            "CodecPos": 53,
+            "RightParenPos": 74,
+            "Type": {
+              "Name": "Delta",
+              "QuoteType": 1,
+              "NamePos": 59,
+              "NameEnd": 64
+            },
+            "Name": {
+              "Name": "ZSTD",
+              "QuoteType": 1,
+              "NamePos": 66,
+              "NameEnd": 70
+            },
+            "Level": {
+              "NumPos": 70,
+              "NumEnd": 72,
+              "Literal": "1",
+              "Base": 10
+            }
+          },
+          "TTL": null,
+          "Comment": null,
+          "CompressionCodec": null
+        },
+        {
+          "NamePos": 78,
+          "ColumnEnd": 107,
+          "Name": {
+            "Ident": {
+              "Name": "api_id",
+              "QuoteType": 3,
+              "NamePos": 78,
+              "NameEnd": 84
+            },
+            "DotIdent": null
+          },
+          "Type": {
+            "Name": {
+              "Name": "UInt64",
+              "QuoteType": 1,
+              "NamePos": 86,
+              "NameEnd": 92
+            }
+          },
+          "NotNull": null,
+          "Nullable": null,
+          "Property": null,
+          "Codec": {
+            "CodecPos": 93,
+            "RightParenPos": 107,
+            "Type": null,
+            "Name": {
+              "Name": "ZSTD",
+              "QuoteType": 1,
+              "NamePos": 99,
+              "NameEnd": 103
+            },
+            "Level": {
+              "NumPos": 103,
+              "NumEnd": 105,
+              "Literal": "1",
+              "Base": 10
+            }
+          },
+          "TTL": null,
+          "Comment": null,
+          "CompressionCodec": null
+        },
+        {
+          "NamePos": 111,
+          "ColumnEnd": 150,
+          "Name": {
+            "Ident": {
+              "Name": "timestamp",
+              "QuoteType": 3,
+              "NamePos": 111,
+              "NameEnd": 120
+            },
+            "DotIdent": null
+          },
+          "Type": {
+            "LeftParenPos": 133,
+            "RightParenPos": 134,
+            "Name": {
+              "Name": "DateTime64",
+              "QuoteType": 1,
+              "NamePos": 122,
+              "NameEnd": 132
+            },
+            "Params": [
+              {
+                "NumPos": 133,
+                "NumEnd": 134,
+                "Literal": "9",
+                "Base": 10
+              }
+            ]
+          },
+          "NotNull": null,
+          "Nullable": null,
+          "Property": null,
+          "Codec": {
+            "CodecPos": 136,
+            "RightParenPos": 150,
+            "Type": null,
+            "Name": {
+              "Name": "ZSTD",
+              "QuoteType": 1,
+              "NamePos": 142,
+              "NameEnd": 146
+            },
+            "Level": {
+              "NumPos": 146,
+              "NumEnd": 148,
+              "Literal": "1",
+              "Base": 10
+            }
+          },
+          "TTL": null,
+          "Comment": null,
+          "CompressionCodec": null
+        },
+        {
+          "IndexPos": 153,
+          "Name": {
+            "Ident": {
+              "Name": "timestamp_index",
+              "QuoteType": 1,
+              "NamePos": 159,
+              "NameEnd": 174
+            },
+            "DotIdent": null
+          },
+          "ColumnExpr": {
+            "LeftParenPos": 174,
+            "RightParenPos": 184,
+            "Items": {
+              "ListPos": 175,
+              "ListEnd": 184,
+              "HasDistinct": false,
+              "Items": [
+                {
+                  "Name": "timestamp",
+                  "QuoteType": 1,
+                  "NamePos": 175,
+                  "NameEnd": 184
+                }
+              ]
+            },
+            "ColumnArgList": null
+          },
+          "ColumnType": {
+            "Name": {
+              "Name": "minmax",
+              "QuoteType": 1,
+              "NamePos": 191,
+              "NameEnd": 197
+            }
+          },
+          "Granularity": {
+            "NumPos": 210,
+            "NumEnd": 211,
+            "Literal": "4",
+            "Base": 10
+          }
+        }
+      ],
+      "AliasTable": null,
+      "TableFunction": null
+    },
+    "Engine": {
+      "EnginePos": 214,
+      "EngineEnd": 649,
+      "Name": "ReplicatedMergeTree",
+      "Params": {
+        "LeftParenPos": 242,
+        "RightParenPos": 274,
+        "Items": {
+          "ListPos": 244,
+          "ListEnd": 273,
+          "HasDistinct": false,
+          "Items": [
+            {
+              "LiteralPos": 244,
+              "LiteralEnd": 260,
+              "Literal": "/root/test_local"
+            },
+            {
+              "LiteralPos": 264,
+              "LiteralEnd": 273,
+              "Literal": "{replica}"
+            }
+          ]
+        },
+        "ColumnArgList": null
+      },
+      "PrimaryKey": null,
+      "PartitionBy": {
+        "PartitionPos": 276,
+        "Expr": {
+          "ListPos": 289,
+          "ListEnd": 314,
+          "HasDistinct": false,
+          "Items": [
+            {
+              "Name": {
+                "Name": "toStartOfHour",
+                "QuoteType": 1,
+                "NamePos": 289,
+                "NameEnd": 302
+              },
+              "Params": {
+                "LeftParenPos": 302,
+                "RightParenPos": 314,
+                "Items": {
+                  "ListPos": 304,
+                  "ListEnd": 313,
+                  "HasDistinct": false,
+                  "Items": [
+                    {
+                      "Name": "timestamp",
+                      "QuoteType": 3,
+                      "NamePos": 304,
+                      "NameEnd": 313
+                    }
+                  ]
+                },
+                "ColumnArgList": null
+              }
+            }
+          ]
+        }
+      },
+      "SampleBy": null,
+      "TTL": {
+        "TTLPos": 372,
+        "ListEnd": 463,
+        "Items": [
+          {
+            "TTLPos": 372,
+            "Expr": {
+              "LeftExpr": {
+                "Name": {
+                  "Name": "toStartOfHour",
+                  "QuoteType": 1,
+                  "NamePos": 376,
+                  "NameEnd": 389
+                },
+                "Params": {
+                  "LeftParenPos": 389,
+                  "RightParenPos": 401,
+                  "Items": {
+                    "ListPos": 391,
+                    "ListEnd": 400,
+                    "HasDistinct": false,
+                    "Items": [
+                      {
+                        "Name": "timestamp",
+                        "QuoteType": 3,
+                        "NamePos": 391,
+                        "NameEnd": 400
+                      }
+                    ]
+                  },
+                  "ColumnArgList": null
+                }
+              },
+              "Operation": "+",
+              "RightExpr": {
+                "IntervalPos": 405,
+                "Expr": {
+                  "NumPos": 414,
+                  "NumEnd": 415,
+                  "Literal": "7",
+                  "Base": 10
+                },
+                "Unit": {
+                  "Name": "DAY",
+                  "QuoteType": 1,
+                  "NamePos": 416,
+                  "NameEnd": 419
+                }
+              },
+              "HasGlobal": false,
+              "HasNot": false
+            }
+          },
+          {
+            "TTLPos": 372,
+            "Expr": {
+              "LeftExpr": {
+                "Name": {
+                  "Name": "toStartOfHour",
+                  "QuoteType": 1,
+                  "NamePos": 420,
+                  "NameEnd": 433
+                },
+                "Params": {
+                  "LeftParenPos": 433,
+                  "RightParenPos": 445,
+                  "Items": {
+                    "ListPos": 435,
+                    "ListEnd": 444,
+                    "HasDistinct": false,
+                    "Items": [
+                      {
+                        "Name": "timestamp",
+                        "QuoteType": 3,
+                        "NamePos": 435,
+                        "NameEnd": 444
+                      }
+                    ]
+                  },
+                  "ColumnArgList": null
+                }
+              },
+              "Operation": "+",
+              "RightExpr": {
+                "IntervalPos": 449,
+                "Expr": {
+                  "NumPos": 458,
+                  "NumEnd": 459,
+                  "Literal": "2",
+                  "Base": 10
+                },
+                "Unit": {
+                  "Name": "DAY",
+                  "QuoteType": 1,
+                  "NamePos": 460,
+                  "NameEnd": 463
+                }
+              },
+              "HasGlobal": false,
+              "HasNot": false
+            }
+          }
+        ]
+      },
+      "Settings": {
+        "SettingsPos": 464,
+        "ListEnd": 649,
+        "Items": [
+          {
+            "SettingsPos": 473,
+            "Name": {
+              "Name": "execute_merges_on_single_replica_time_threshold",
+              "QuoteType": 1,
+              "NamePos": 473,
+              "NameEnd": 520
+            },
+            "Expr": {
+              "NumPos": 521,
+              "NumEnd": 525,
+              "Literal": "1200",
+              "Base": 10
+            }
+          },
+          {
+            "SettingsPos": 527,
+            "Name": {
+              "Name": "index_granularity",
+              "QuoteType": 1,
+              "NamePos": 527,
+              "NameEnd": 544
+            },
+            "Expr": {
+              "NumPos": 545,
+              "NumEnd": 550,
+              "Literal": "16384",
+              "Base": 10
+            }
+          },
+          {
+            "SettingsPos": 552,
+            "Name": {
+              "Name": "max_bytes_to_merge_at_max_space_in_pool",
+              "QuoteType": 1,
+              "NamePos": 552,
+              "NameEnd": 591
+            },
+            "Expr": {
+              "NumPos": 592,
+              "NumEnd": 603,
+              "Literal": "64424509440",
+              "Base": 10
+            }
+          },
+          {
+            "SettingsPos": 605,
+            "Name": {
+              "Name": "storage_policy",
+              "QuoteType": 1,
+              "NamePos": 605,
+              "NameEnd": 619
+            },
+            "Expr": {
+              "LiteralPos": 621,
+              "LiteralEnd": 625,
+              "Literal": "main"
+            }
+          },
+          {
+            "SettingsPos": 628,
+            "Name": {
+              "Name": "ttl_only_drop_parts",
+              "QuoteType": 1,
+              "NamePos": 628,
+              "NameEnd": 647
+            },
+            "Expr": {
+              "NumPos": 648,
+              "NumEnd": 649,
+              "Literal": "1",
+              "Base": 10
+            }
+          }
+        ]
+      },
+      "OrderBy": {
+        "OrderPos": 316,
+        "ListEnd": 370,
+        "Items": [
+          {
+            "OrderPos": 316,
+            "Expr": {
+              "LeftParenPos": 325,
+              "RightParenPos": 370,
+              "Items": {
+                "ListPos": 326,
+                "ListEnd": 369,
+                "HasDistinct": false,
+                "Items": [
+                  {
+                    "Name": {
+                      "Name": "toUnixTimestamp64Nano",
+                      "QuoteType": 1,
+                      "NamePos": 326,
+                      "NameEnd": 347
+                    },
+                    "Params": {
+                      "LeftParenPos": 347,
+                      "RightParenPos": 359,
+                      "Items": {
+                        "ListPos": 349,
+                        "ListEnd": 358,
+                        "HasDistinct": false,
+                        "Items": [
+                          {
+                            "Name": "timestamp",
+                            "QuoteType": 3,
+                            "NamePos": 349,
+                            "NameEnd": 358
+                          }
+                        ]
+                      },
+                      "ColumnArgList": null
+                    }
+                  },
+                  {
+                    "Name": "api_id",
+                    "QuoteType": 3,
+                    "NamePos": 363,
+                    "NameEnd": 369
+                  }
+                ]
+              },
+              "ColumnArgList": null
+            },
+            "Direction": "None"
+          }
+        ]
+      }
+    },
+    "SubQuery": null,
+    "HasTemporary": false
+  }
+]
\ No newline at end of file