Repository: oom-ai/oomstore
Pull request #1140 (2022-02-08 08:00:06)

Refactor/csv source
modify the type of delimiter from `string` to `rune`


Patch:
diff --git a/internal/database/dbutil/data_loader.go b/internal/database/dbutil/data_loader.go
index 1d48757d..1b92d131 100644
--- a/internal/database/dbutil/data_loader.go
+++ b/internal/database/dbutil/data_loader.go
@@ -73,6 +73,8 @@ func ReadLine(opt ReadLineOpt) ([]interface{}, error) {
 	// so here we use csv.Reader to read data directly.
 	// If there are other data formats in the future, we need to choose different decoders according to the different formats.
 	reader := csv.NewReader(opt.Source.Reader)
+	reader.Comma = opt.Source.Delimiter
+
 	row, err := reader.Read()
 	if err != nil {
 		return nil, err
diff --git a/internal/database/offline/types.go b/internal/database/offline/types.go
index 9b04c439..1f199023 100644
--- a/internal/database/offline/types.go
+++ b/internal/database/offline/types.go
@@ -51,7 +51,7 @@ type PushOpt struct {
 
 type CSVSource struct {
 	Reader    *bufio.Reader
-	Delimiter string
+	Delimiter rune
 }
 
 type SnapshotOpt struct {
diff --git a/oomagent/server.go b/oomagent/server.go
index 0c5a0dba..d93b4ca0 100644
--- a/oomagent/server.go
+++ b/oomagent/server.go
@@ -97,11 +97,12 @@ func (s *server) ChannelImport(stream codegen.OomAgent_ChannelImportServer) erro
 	if firstReq.Group == nil {
 		return status.Errorf(codes.InvalidArgument, "group is required in first request")
 	}
-	var description, delimiter string
+	var description string
+	var delimiter rune
 	if firstReq.Description != nil {
 		description = *firstReq.Description
 	}
-	delimiter = ","
+	delimiter = ','
 	reader, writer := io.Pipe()
 
 	go func() {
@@ -165,14 +166,16 @@ func (s *server) Snapshot(ctx context.Context, re *codegen.SnapshotRequest) (*co
 }
 
 func (s *server) Import(ctx context.Context, req *codegen.ImportRequest) (*codegen.ImportResponse, error) {
-	var description, delimiter string
+	var description string
+	var delimiter rune
+
 	if req.Description != nil {
 		description = *req.Description
 	}
 	if req.Delimiter != nil {
-		delimiter = *req.Delimiter
+		delimiter = []rune(*req.Delimiter)[0]
 	} else {
-		delimiter = ","
+		delimiter = ','
 	}
 	revisionID, err := s.oomstore.Import(ctx, types.ImportOpt{
 		GroupName:      req.Group,
diff --git a/oomcli/cmd/import.go b/oomcli/cmd/import.go
index c500d3de..bd9d19c1 100644
--- a/oomcli/cmd/import.go
+++ b/oomcli/cmd/import.go
@@ -12,11 +12,14 @@ import (
 var importOpt types.ImportOpt
 var importCSVFileDataSource types.CsvFileDataSource
 var importTableLinkDataSource types.TableLinkDataSource
+var delimitre string
 
 var importCmd = &cobra.Command{
 	Use:   "import",
 	Short: "Import feature data from a data source",
 	PreRunE: func(cmd *cobra.Command, args []string) error {
+		importCSVFileDataSource.Delimiter = []rune(delimitre)[0]
+
 		if !cmd.Flags().Changed("revision") {
 			importOpt.Revision = nil
 		}
@@ -61,6 +64,6 @@ func init() {
 	flags.StringVar(&importCSVFileDataSource.InputFilePath, "input-file", "", "input csv file")
 	flags.StringVar(&importTableLinkDataSource.TableName, "table-link", "", "link to a existing data table")
 
-	flags.StringVar(&importCSVFileDataSource.Delimiter, "delimiter", ",", "specify field delimiter")
+	flags.StringVar(&delimitre, "delimiter", ",", "specify field delimiter")
 	importOpt.Revision = flags.Int64P("revision", "r", 0, "user-defined revision")
 }
diff --git a/pkg/oomstore/types/import.go b/pkg/oomstore/types/import.go
index c0f2f43b..003fd1b1 100644
--- a/pkg/oomstore/types/import.go
+++ b/pkg/oomstore/types/import.go
@@ -23,12 +23,12 @@ type ImportOpt struct {
 
 type CsvReaderDataSource struct {
 	Reader    io.Reader
-	Delimiter string
+	Delimiter rune
 }
 
 type CsvFileDataSource struct {
 	InputFilePath string
-	Delimiter     string
+	Delimiter     rune
 }
 
 type TableLinkDataSource struct {


Test patch:
diff --git a/internal/database/offline/test_impl/export.go b/internal/database/offline/test_impl/export.go
index 8c8437a8..561c4f4a 100644
--- a/internal/database/offline/test_impl/export.go
+++ b/internal/database/offline/test_impl/export.go
@@ -29,7 +29,7 @@ func TestExport(t *testing.T, prepareStore PrepareStoreFn, destroyStore DestroyS
 1236,huawei,300
 1237,oneplus,240
 `)),
-		Delimiter: ",",
+		Delimiter: ',',
 	})
 	buildTestSnapshotTable(ctx, t, store, streamFeatures, 1, streamSnapshotTable, &offline.CSVSource{
 		Reader: bufio.NewReader(strings.NewReader(`1234,1000,true
@@ -37,7 +37,7 @@ func TestExport(t *testing.T, prepareStore PrepareStoreFn, destroyStore DestroyS
 1236,1560,true
 1237,4000,false
 `)),
-		Delimiter: ",",
+		Delimiter: ',',
 	})
 	buildTestSnapshotTable(ctx, t, store, append(streamFeatures, unixMilli), 1, streamCdcTable, &offline.CSVSource{
 		Reader: bufio.NewReader(strings.NewReader(`1234,1200,true,2
@@ -46,7 +46,7 @@ func TestExport(t *testing.T, prepareStore PrepareStoreFn, destroyStore DestroyS
 1238,1500,true,11
 1239,2700,false,12
 `)),
-		Delimiter: ",",
+		Delimiter: ',',
 	})
 
 	testCases := []struct {
diff --git a/internal/database/offline/test_impl/import.go b/internal/database/offline/test_impl/import.go
index fa33b3d8..1ff5d528 100644
--- a/internal/database/offline/test_impl/import.go
+++ b/internal/database/offline/test_impl/import.go
@@ -37,7 +37,7 @@ func TestImport(t *testing.T, prepareStore PrepareStoreFn, destroyStore DestroyS
 1236,huawei,5999
 1237,oneplus,3999
 `)),
-			Delimiter: ",",
+			Delimiter: ',',
 		},
 	}
 
diff --git a/internal/database/offline/test_impl/join.go b/internal/database/offline/test_impl/join.go
index 00b609c6..008387df 100644
--- a/internal/database/offline/test_impl/join.go
+++ b/internal/database/offline/test_impl/join.go
@@ -35,19 +35,19 @@ func TestJoin(t *testing.T, prepareStore PrepareStoreFn, destroyStore DestroySto
 		Reader: bufio.NewReader(strings.NewReader(`1234,xiaomi,100
 1235,apple,200
 `)),
-		Delimiter: ",",
+		Delimiter: ',',
 	})
 	buildTestSnapshotTable(ctx, t, store, oneGroupFeatures, 2, "offline_snapshot_1_2", &offline.CSVSource{
 		Reader: bufio.NewReader(strings.NewReader(`1234,galaxy,300
 1235,oneplus,240
 `)),
-		Delimiter: ",",
+		Delimiter: ',',
 	})
 	buildTestSnapshotTable(ctx, t, store, twoGroupFeatures[2:], 1, "offline_snapshot_2_1", &offline.CSVSource{
 		Reader: bufio.NewReader(strings.NewReader(`1234,1
 1235,0
 `)),
-		Delimiter: ",",
+		Delimiter: ',',
 	})
 
 	buildTestSnapshotTable(ctx, t, store, append(oneGroupFeatures, unixMilli), 1, "offline_cdc_1_1", &offline.CSVSource{
@@ -55,12 +55,12 @@ func TestJoin(t *testing.T, prepareStore PrepareStoreFn, destroyStore DestroySto
 1235,apple-2,115,14
 1234,xiaomi-1,130,10
 `)),
-		Delimiter: ",",
+		Delimiter: ',',
 	})
 	buildTestSnapshotTable(ctx, t, store, append(oneGroupFeatures, unixMilli), 2, "offline_cdc_1_2", &offline.CSVSource{
 		Reader: bufio.NewReader(strings.NewReader(`1234,galaxy-1,320,18
 `)),
-		Delimiter: ",",
+		Delimiter: ',',
 	})
 
 	testCases := []struct {
diff --git a/internal/database/offline/test_impl/snapshot.go b/internal/database/offline/test_impl/snapshot.go
index 5b0cddd7..24c269f7 100644
--- a/internal/database/offline/test_impl/snapshot.go
+++ b/internal/database/offline/test_impl/snapshot.go
@@ -33,7 +33,7 @@ func TestSnapshot(t *testing.T, prepareStore PrepareStoreFn, destroyStore Destro
 1235,apple,200,5
 1236,oneplus,155,8
 `)),
-		Delimiter: ",",
+		Delimiter: ',',
 	})
 
 	buildTestSnapshotTable(ctx, t, store, append(features, unixMilli), 2, "offline_stream_cdc_1_2", &offline.CSVSource{
@@ -43,7 +43,7 @@ func TestSnapshot(t *testing.T, prepareStore PrepareStoreFn, destroyStore Destro
 1237,pixel,200,11
 1235,apple-1,113,15
 `)),
-		Delimiter: ",",
+		Delimiter: ',',
 	})
 
 	err := store.Snapshot(ctx, offline.SnapshotOpt{