Repository: oom-ai/oomstore
Pull request #870 (2021-12-30 09:13:32)

fix: parse sqlite value 'true/false' into bool


Patch:
diff --git a/internal/database/metadata/sqlutil/util.go b/internal/database/metadata/sqlutil/util.go
index 19b7b2ec..d100f7e1 100644
--- a/internal/database/metadata/sqlutil/util.go
+++ b/internal/database/metadata/sqlutil/util.go
@@ -6,10 +6,10 @@ func OfflineBatchTableName(groupID int, revisionID int64) string {
 	return fmt.Sprintf("offline_batch_%d_%d", groupID, revisionID)
 }
 
-func OfflineStreamSnapshotTableName(groupID int, revision int64) string {
-	return fmt.Sprintf("offline_stream_snapshot_%d_%d", groupID, revision)
+func OfflineStreamSnapshotTableName(groupID int, revisionID int) string {
+	return fmt.Sprintf("offline_stream_snapshot_%d_%d", groupID, revisionID)
 }
 
-func OfflineStreamCdcTableName(groupID int, revision int64) string {
-	return fmt.Sprintf("offline_stream_cdc_%d_%d", groupID, revision)
+func OfflineStreamCdcTableName(groupID int, revisionID int) string {
+	return fmt.Sprintf("offline_stream_cdc_%d_%d", groupID, revisionID)
 }
diff --git a/internal/database/offline/sqlutil/snapshot.go b/internal/database/offline/sqlutil/snapshot.go
index 9ba53391..12095bef 100644
--- a/internal/database/offline/sqlutil/snapshot.go
+++ b/internal/database/offline/sqlutil/snapshot.go
@@ -59,9 +59,9 @@ ON l.{{ qt .EntityName }} = r.{{ qt .EntityName }}
 `
 
 func Snapshot(ctx context.Context, dbOpt dbutil.DBOpt, opt offline.SnapshotOpt) error {
-	prevSnapshotTableName := sqlutil.OfflineStreamSnapshotTableName(opt.Group.ID, opt.PrevRevision)
-	currSnapshotTableName := sqlutil.OfflineStreamSnapshotTableName(opt.Group.ID, opt.Revision)
-	currCdcTableName := sqlutil.OfflineStreamCdcTableName(opt.Group.ID, opt.Revision)
+	prevSnapshotTableName := sqlutil.OfflineStreamSnapshotTableName(opt.Group.ID, opt.PrevRevisionID)
+	currSnapshotTableName := sqlutil.OfflineStreamSnapshotTableName(opt.Group.ID, opt.RevisionID)
+	currCdcTableName := sqlutil.OfflineStreamCdcTableName(opt.Group.ID, opt.RevisionID)
 
 	if dbOpt.Backend == types.BackendBigQuery {
 		currSnapshotTableName = fmt.Sprintf("%s.%s", *dbOpt.DatasetID, currSnapshotTableName)
diff --git a/internal/database/offline/types.go b/internal/database/offline/types.go
index 16bac9e7..15cb333c 100644
--- a/internal/database/offline/types.go
+++ b/internal/database/offline/types.go
@@ -53,8 +53,8 @@ type CSVSource struct {
 }
 
 type SnapshotOpt struct {
-	Group        *types.Group
-	Features     types.FeatureList
-	Revision     int64
-	PrevRevision int64
+	Group          *types.Group
+	Features       types.FeatureList
+	RevisionID     int
+	PrevRevisionID int
 }
diff --git a/internal/database/online/sqlutil/get.go b/internal/database/online/sqlutil/get.go
index b8ac99a3..4ed1862f 100644
--- a/internal/database/online/sqlutil/get.go
+++ b/internal/database/online/sqlutil/get.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 
 	"github.com/jmoiron/sqlx"
+	"github.com/spf13/cast"
 
 	"github.com/oom-ai/oomstore/internal/database/dbutil"
 	"github.com/oom-ai/oomstore/internal/database/online"
@@ -93,6 +94,7 @@ func deserializeByTag(i interface{}, valueType types.ValueType, backend types.Ba
 		return nil, nil
 	}
 
+	// TODO: merge with DeserializeByValueType in internal/database/offline/sqlutil/serialize.go
 	switch valueType {
 	case types.String:
 		if backend == types.BackendMySQL {
@@ -101,9 +103,10 @@ func deserializeByTag(i interface{}, valueType types.ValueType, backend types.Ba
 		return i, nil
 	case types.Bool:
 		if backend == types.BackendMySQL || backend == types.BackendSQLite {
-			if i == int64(1) {
+			s := cast.ToString(i)
+			if s == "1" || s == "true" {
 				return true, nil
-			} else if i == int64(0) {
+			} else if s == "0" || s == "false" {
 				return false, nil
 			} else {
 				return nil, fmt.Errorf("invalid bool value: %s", i)
diff --git a/pkg/oomstore/snapshot.go b/pkg/oomstore/snapshot.go
index b96c1ed9..04b3dfb5 100644
--- a/pkg/oomstore/snapshot.go
+++ b/pkg/oomstore/snapshot.go
@@ -27,11 +27,11 @@ func (s *OomStore) Snapshot(ctx context.Context, groupName string) error {
 		if revision.SnapshotTable != "" {
 			continue
 		}
-		tableName := sqlutil.OfflineStreamSnapshotTableName(group.ID, revision.Revision)
+		tableName := sqlutil.OfflineStreamSnapshotTableName(group.ID, revision.ID)
 		if err = s.offline.Snapshot(ctx, offline.SnapshotOpt{
-			Group:        group,
-			Revision:     revisions[i].Revision,
-			PrevRevision: revisions[i-1].Revision,
+			Group:          group,
+			RevisionID:     revisions[i].ID,
+			PrevRevisionID: revisions[i-1].ID,
 		}); err != nil {
 			return err
 		}


Test patch:
diff --git a/internal/database/offline/test_impl/export.go b/internal/database/offline/test_impl/export.go
index 265acfab..cab77571 100644
--- a/internal/database/offline/test_impl/export.go
+++ b/internal/database/offline/test_impl/export.go
@@ -27,7 +27,7 @@ func TestExport(t *testing.T, prepareStore PrepareStoreFn, destroyStore DestroyS
 			ValueType: types.Int64,
 		},
 	}
-	buildTestSnapshotTable(ctx, t, store, features, 1, snapshotTable, &offline.CSVSource{
+	buildTestSnapshotTable(ctx, t, store, features, snapshotTable, &offline.CSVSource{
 		Reader: bufio.NewReader(strings.NewReader(`1234,xiaomi,100
 1235,apple,200
 1236,huawei,300
diff --git a/internal/database/offline/test_impl/join.go b/internal/database/offline/test_impl/join.go
index b1203465..b1a0d4af 100644
--- a/internal/database/offline/test_impl/join.go
+++ b/internal/database/offline/test_impl/join.go
@@ -34,33 +34,33 @@ func TestJoin(t *testing.T, prepareStore PrepareStoreFn, destroyStore DestroySto
 	oneGroupFeatures, oneGroupFeatureMap := prepareFeatures(true)
 	twoGroupFeatures, twoGroupFeatureMap := prepareFeatures(false)
 
-	buildTestSnapshotTable(ctx, t, store, oneGroupFeatures, 1, "offline_snapshot_1_1", &offline.CSVSource{
+	buildTestSnapshotTable(ctx, t, store, oneGroupFeatures, "offline_snapshot_1_1", &offline.CSVSource{
 		Reader: bufio.NewReader(strings.NewReader(`1234,xiaomi,100
 1235,apple,200
 `)),
 		Delimiter: ",",
 	})
-	buildTestSnapshotTable(ctx, t, store, oneGroupFeatures, 2, "offline_snapshot_1_2", &offline.CSVSource{
+	buildTestSnapshotTable(ctx, t, store, oneGroupFeatures, "offline_snapshot_1_2", &offline.CSVSource{
 		Reader: bufio.NewReader(strings.NewReader(`1234,galaxy,300
 1235,oneplus,240
 `)),
 		Delimiter: ",",
 	})
-	buildTestSnapshotTable(ctx, t, store, twoGroupFeatures[2:], 1, "offline_snapshot_2_1", &offline.CSVSource{
+	buildTestSnapshotTable(ctx, t, store, twoGroupFeatures[2:], "offline_snapshot_2_1", &offline.CSVSource{
 		Reader: bufio.NewReader(strings.NewReader(`1234,1
 1235,0
 `)),
 		Delimiter: ",",
 	})
 
-	buildTestSnapshotTable(ctx, t, store, append(oneGroupFeatures, unixMilli), 1, "offline_cdc_1_1", &offline.CSVSource{
+	buildTestSnapshotTable(ctx, t, store, append(oneGroupFeatures, unixMilli), "offline_cdc_1_1", &offline.CSVSource{
 		Reader: bufio.NewReader(strings.NewReader(`1234,xiaomi-1,120,2
 1235,apple-2,115,14
 1234,xiaomi-1,130,10
 `)),
 		Delimiter: ",",
 	})
-	buildTestSnapshotTable(ctx, t, store, append(oneGroupFeatures, unixMilli), 2, "offline_cdc_1_2", &offline.CSVSource{
+	buildTestSnapshotTable(ctx, t, store, append(oneGroupFeatures, unixMilli), "offline_cdc_1_2", &offline.CSVSource{
 		Reader: bufio.NewReader(strings.NewReader(`1234,galaxy-1,320,18
 `)),
 		Delimiter: ",",
diff --git a/internal/database/offline/test_impl/snapshot.go b/internal/database/offline/test_impl/snapshot.go
index 21299a94..d6e756f8 100644
--- a/internal/database/offline/test_impl/snapshot.go
+++ b/internal/database/offline/test_impl/snapshot.go
@@ -33,7 +33,7 @@ func TestSnapshot(t *testing.T, prepareStore PrepareStoreFn, destroyStore Destro
 	}
 	features, _ := prepareFeatures(true)
 
-	buildTestSnapshotTable(ctx, t, store, features, 1, "offline_stream_snapshot_1_1", &offline.CSVSource{
+	buildTestSnapshotTable(ctx, t, store, features, "offline_stream_snapshot_1_1", &offline.CSVSource{
 		Reader: bufio.NewReader(strings.NewReader(`1234,xiaomi,100
 1235,apple,200
 1236,oneplus,155
@@ -41,7 +41,7 @@ func TestSnapshot(t *testing.T, prepareStore PrepareStoreFn, destroyStore Destro
 		Delimiter: ",",
 	})
 
-	buildTestSnapshotTable(ctx, t, store, append(features, unixMilli), 2, "offline_stream_cdc_1_2", &offline.CSVSource{
+	buildTestSnapshotTable(ctx, t, store, append(features, unixMilli), "offline_stream_cdc_1_2", &offline.CSVSource{
 		Reader: bufio.NewReader(strings.NewReader(`1234,xiaomi-1,120,2
 1235,apple-2,115,14
 1234,xiaomi-1,130,10
@@ -52,10 +52,10 @@ func TestSnapshot(t *testing.T, prepareStore PrepareStoreFn, destroyStore Destro
 	})
 
 	err := store.Snapshot(ctx, offline.SnapshotOpt{
-		Group:        group,
-		Features:     features,
-		Revision:     2,
-		PrevRevision: 1,
+		Group:          group,
+		Features:       features,
+		RevisionID:     2,
+		PrevRevisionID: 1,
 	})
 	assert.NoError(t, err)
 
diff --git a/internal/database/offline/test_impl/util.go b/internal/database/offline/test_impl/util.go
index bc468741..0b862077 100644
--- a/internal/database/offline/test_impl/util.go
+++ b/internal/database/offline/test_impl/util.go
@@ -13,15 +13,7 @@ type PrepareStoreFn func(t *testing.T) (context.Context, offline.Store)
 
 type DestroyStoreFn func()
 
-func buildTestSnapshotTable(
-	ctx context.Context,
-	t *testing.T,
-	store offline.Store,
-	features []*types.Feature,
-	revision int64,
-	snapshotTable string,
-	source *offline.CSVSource,
-) {
+func buildTestSnapshotTable(ctx context.Context, t *testing.T, store offline.Store, features []*types.Feature, snapshotTable string, source *offline.CSVSource) {
 	entity := &types.Entity{
 		Name:   "device",
 		Length: 10,
@@ -36,7 +28,6 @@ func buildTestSnapshotTable(
 		Features:          features,
 		Header:            header,
 		Source:            source,
-		Revision:          &revision,
 	}
 	_, err := store.Import(ctx, opt)
 	require.NoError(t, err)