Repository: lf-edge/ekuiper
Pull request #2807 (2024-04-25 07:03:31)

feat(sink): split encryptor


Patch:
diff --git a/etc/kuiper.yaml b/etc/kuiper.yaml
index 140d208b..a0a057b3 100644
--- a/etc/kuiper.yaml
+++ b/etc/kuiper.yaml
@@ -61,6 +61,8 @@ basic:
   cfgStorageType: file
   # enableOpenZiti indicates whether to enable OpenZiti for eKuiper REST service. Currently, it is only supported to work with EdgeX secure mode.
   enableOpenZiti: false
+  # AES Key, base64 encoded
+  aesKey: MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3
 
 # The default options for all rules. Each rule can override this setting by defining its own option
 rule:
diff --git a/internal/conf/conf.go b/internal/conf/conf.go
index 9bbdc679..bfc071b6 100644
--- a/internal/conf/conf.go
+++ b/internal/conf/conf.go
@@ -15,6 +15,7 @@
 package conf
 
 import (
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
@@ -193,6 +194,7 @@ type KuiperConf struct {
 		RulePatrolInterval  string      `yaml:"rulePatrolInterval"`
 		CfgStorageType      string      `yaml:"cfgStorageType"`
 		EnableOpenZiti      bool        `yaml:"enableOpenZiti"`
+		AesKey              string      `yaml:"aesKey"`
 	}
 	Rule   def.RuleOption
 	Sink   *SinkConf
@@ -218,6 +220,7 @@ type KuiperConf struct {
 		PythonBin   string `yaml:"pythonBin"`
 		InitTimeout int    `yaml:"initTimeout"`
 	}
+	AesKey []byte
 }
 
 func SetLogLevel(level string, debug bool) {
@@ -362,6 +365,14 @@ func InitConf() {
 		}
 	}
 
+	if Config.Basic.AesKey != "" {
+		key, err := base64.StdEncoding.DecodeString(Config.Basic.AesKey)
+		if err != nil {
+			Log.Fatal(err)
+		}
+		Config.AesKey = key
+	}
+
 	if Config.Store.Type == "redis" && Config.Store.Redis.ConnectionSelector != "" {
 		if err := RedisStorageConSelectorApply(Config.Store.Redis.ConnectionSelector, Config); err != nil {
 			Log.Fatal(err)
diff --git a/internal/encryptor/aes/aes.go b/internal/encryptor/aes/aes.go
new file mode 100644
index 00000000..4ad19e5d
--- /dev/null
+++ b/internal/encryptor/aes/aes.go
@@ -0,0 +1,57 @@
+// Copyright 2023-2024 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aes
+
+import (
+	"crypto/cipher"
+	"fmt"
+	"io"
+)
+
+type StreamEncrypter struct {
+	stream cipher.Stream
+	iv     []byte
+}
+
+func (a *StreamEncrypter) Encrypt(data []byte) []byte {
+	ciphertext := make([]byte, len(data))
+	a.stream.XORKeyStream(ciphertext, data)
+	result := append(a.iv, ciphertext...)
+	return result
+}
+
+func NewStreamEncrypter(key, iv []byte) (*StreamEncrypter, error) {
+	s, err := newAesStream(key, iv)
+	if err != nil {
+		return nil, err
+	}
+	return &StreamEncrypter{
+		stream: s,
+		iv:     iv,
+	}, nil
+}
+
+func NewStreamWriter(key, iv []byte, output io.Writer) (*cipher.StreamWriter, error) {
+	blockMode, err := newAesStream(key, iv)
+	if err != nil {
+		return nil, err
+	}
+	writer := &cipher.StreamWriter{S: blockMode, W: output}
+	_, err = writer.W.Write(iv)
+	if err != nil {
+		return nil, fmt.Errorf("failed to write iv: %v", err)
+	}
+	return writer, nil
+}
diff --git a/internal/encryptor/aes/cipher.go b/internal/encryptor/aes/cipher.go
new file mode 100644
index 00000000..9bc8fa8e
--- /dev/null
+++ b/internal/encryptor/aes/cipher.go
@@ -0,0 +1,30 @@
+// Copyright 2023 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aes
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"fmt"
+)
+
+func newAesStream(key, iv []byte) (cipher.Stream, error) {
+	// Create a new AES cipher block using the key
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, fmt.Errorf("Error creating AES cipher block: %v", err)
+	}
+	return cipher.NewCFBEncrypter(block, iv), nil
+}
diff --git a/internal/encryptor/encryptor.go b/internal/encryptor/encryptor.go
new file mode 100644
index 00000000..d150cfde
--- /dev/null
+++ b/internal/encryptor/encryptor.go
@@ -0,0 +1,43 @@
+// Copyright 2023-2024 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encryptor
+
+import (
+	"crypto/rand"
+	"fmt"
+
+	"github.com/lf-edge/ekuiper/v2/internal/conf"
+	"github.com/lf-edge/ekuiper/v2/internal/encryptor/aes"
+	"github.com/lf-edge/ekuiper/v2/pkg/message"
+)
+
+func GetEncryptor(name string) (message.Encryptor, error) {
+	if name == "aes" {
+		if conf.Config == nil || conf.Config.AesKey == nil {
+			return nil, fmt.Errorf("AES key is not defined")
+		}
+		key, iv := getKeyIv()
+		return aes.NewStreamEncrypter(key, iv)
+	}
+	return nil, fmt.Errorf("unsupported encryptor: %s", name)
+}
+
+func getKeyIv() ([]byte, []byte) {
+	key := conf.Config.AesKey
+	iv := make([]byte, 16)
+	// Use the crypto/rand package to generate random bytes
+	_, _ = rand.Read(iv)
+	return key, iv
+}
diff --git a/internal/topo/node/encrypt_op.go b/internal/topo/node/encrypt_op.go
new file mode 100644
index 00000000..87306f0e
--- /dev/null
+++ b/internal/topo/node/encrypt_op.go
@@ -0,0 +1,66 @@
+// Copyright 2024 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+	"fmt"
+
+	"github.com/lf-edge/ekuiper/contract/v2/api"
+	"github.com/lf-edge/ekuiper/v2/internal/encryptor"
+	"github.com/lf-edge/ekuiper/v2/internal/pkg/def"
+	"github.com/lf-edge/ekuiper/v2/pkg/infra"
+	"github.com/lf-edge/ekuiper/v2/pkg/message"
+)
+
+type EncryptNode struct {
+	*defaultSinkNode
+	tool message.Encryptor
+}
+
+func NewEncryptOp(name string, rOpt *def.RuleOption, encryptMethod string) (*EncryptNode, error) {
+	dc, err := encryptor.GetEncryptor(encryptMethod)
+	if err != nil {
+		return nil, fmt.Errorf("get encryptor %s fail with error: %v", encryptMethod, err)
+	}
+	return &EncryptNode{
+		defaultSinkNode: newDefaultSinkNode(name, rOpt),
+		tool:            dc,
+	}, nil
+}
+
+func (o *EncryptNode) Exec(ctx api.StreamContext, errCh chan<- error) {
+	o.prepareExec(ctx, errCh, "op")
+	go func() {
+		err := infra.SafeRun(func() error {
+			runWithOrder(ctx, o.defaultSinkNode, o.concurrency, o.Worker)
+			return nil
+		})
+		if err != nil {
+			infra.DrainError(ctx, err, errCh)
+		}
+	}()
+}
+
+func (o *EncryptNode) Worker(_ api.Logger, item any) []any {
+	o.statManager.ProcessTimeStart()
+	defer o.statManager.ProcessTimeEnd()
+	switch d := item.(type) {
+	case []byte:
+		r := o.tool.Encrypt(d)
+		return []any{r}
+	default:
+		return []any{fmt.Errorf("unsupported data received: %v", d)}
+	}
+}
diff --git a/internal/topo/node/props.go b/internal/topo/node/props.go
index d3f96093..9c6cfe6d 100644
--- a/internal/topo/node/props.go
+++ b/internal/topo/node/props.go
@@ -37,6 +37,7 @@ type SinkConf struct {
 	BatchSize      int      `json:"batchSize"`
 	LingerInterval int      `json:"lingerInterval"`
 	Compression    string   `json:"compression"`
+	Encryption     string   `json:"encryption"`
 	conf.SinkConf
 }
 
diff --git a/internal/topo/planner/planner_sink.go b/internal/topo/planner/planner_sink.go
index 38898bdb..029d07b3 100644
--- a/internal/topo/planner/planner_sink.go
+++ b/internal/topo/planner/planner_sink.go
@@ -113,6 +113,16 @@ func splitSink(tp *topo.Topo, inputs []node.Emitter, s api.Sink, sinkName string
 			tp.AddOperator(newInputs, compressOp)
 			newInputs = []node.Emitter{compressOp}
 		}
+
+		if sc.Encryption != "" {
+			encryptOp, err := node.NewEncryptOp(fmt.Sprintf("%s_%d_encrypt", sinkName, index), options, sc.Encryption)
+			if err != nil {
+				return nil, err
+			}
+			index++
+			tp.AddOperator(newInputs, encryptOp)
+			newInputs = []node.Emitter{encryptOp}
+		}
 	}
 	return newInputs, nil
 }
diff --git a/pkg/message/artifacts.go b/pkg/message/artifacts.go
index 037417c2..3708a42c 100644
--- a/pkg/message/artifacts.go
+++ b/pkg/message/artifacts.go
@@ -60,3 +60,8 @@ type Compressor interface {
 type Decompressor interface {
 	Decompress([]byte) ([]byte, error)
 }
+
+// Encryptor encrypts bytes
+type Encryptor interface {
+	Encrypt([]byte) []byte
+}


Test patch:
diff --git a/internal/encryptor/aes/aes_test.go b/internal/encryptor/aes/aes_test.go
new file mode 100644
index 00000000..3756efa2
--- /dev/null
+++ b/internal/encryptor/aes/aes_test.go
@@ -0,0 +1,81 @@
+// Copyright 2023-2024 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aes
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMessage(t *testing.T) {
+	key := []byte("0123456789abcdef01234567")
+	iv := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}
+	// plaintext
+	pt := "Using the Input type selection, choose the type of input – a text string or a file. In case of the text string input, enter your input into the Input text textarea1,2. Otherwise, use the \"Browse\" button to select the input file to upload. Then select the cryptographic function you want to use in the Function field. Depending on the selected function the Initialization vector (IV) field is shown or hidden. Initialization vector is always a sequence of bytes, each byte has to be represented in hexadecimal form."
+
+	stream, err := NewStreamEncrypter(key, iv)
+	assert.NoError(t, err)
+	secret := stream.Encrypt([]byte(pt))
+
+	niv := secret[:aes.BlockSize]
+	assert.Equal(t, iv, niv)
+	secret = secret[aes.BlockSize:]
+	dstream, err := NewAESStreamDecrypter(key, iv)
+	assert.NoError(t, err)
+	revert := make([]byte, len(secret))
+	dstream.XORKeyStream(revert, secret)
+	assert.Equal(t, pt, string(revert))
+}
+
+func TestStreamWriter(t *testing.T) {
+	key := []byte("0123456789abcdef01234567")
+	iv := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}
+	// plaintext
+	pt := "Using the Input type selection, choose the type of input – a text string or a file. In case of the text string input, enter your input into the Input text textarea1,2. Otherwise, use the \"Browse\" button to select the input file to upload. Then select the cryptographic function you want to use in the Function field. Depending on the selected function the Initialization vector (IV) field is shown or hidden. Initialization vector is always a sequence of bytes, each byte has to be represented in hexadecimal form."
+
+	output := new(bytes.Buffer)
+	writer, err := NewStreamWriter(key, iv, output)
+	assert.NoError(t, err)
+	_, err = writer.Write([]byte(pt))
+	assert.NoError(t, err)
+	err = writer.Close()
+	assert.NoError(t, err)
+	secret := output.Bytes()
+
+	// Read the appended iv
+	niv := secret[:aes.BlockSize]
+	assert.Equal(t, iv, niv)
+	secret = secret[aes.BlockSize:]
+	// Decrypt
+	dstream, err := NewAESStreamDecrypter(key, iv)
+	assert.NoError(t, err)
+	revert := make([]byte, len(secret))
+	dstream.XORKeyStream(revert, secret)
+	assert.Equal(t, pt, string(revert))
+}
+
+func NewAESStreamDecrypter(key, iv []byte) (cipher.Stream, error) {
+	// Create a new AES cipher block using the key
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, fmt.Errorf("Error creating AES cipher block: %v", err)
+	}
+	return cipher.NewCFBDecrypter(block, iv), nil
+}
diff --git a/internal/encryptor/encryptor_test.go b/internal/encryptor/encryptor_test.go
new file mode 100644
index 00000000..07b28d35
--- /dev/null
+++ b/internal/encryptor/encryptor_test.go
@@ -0,0 +1,31 @@
+// Copyright 2023-2024 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encryptor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/lf-edge/ekuiper/v2/internal/conf"
+)
+
+func TestGetEncryptor(t *testing.T) {
+	conf.InitConf()
+	_, err := GetEncryptor("aes")
+	assert.NoError(t, err)
+	_, err = GetEncryptor("unknown")
+	assert.Error(t, err)
+}
diff --git a/internal/topo/node/encrypt_op_test.go b/internal/topo/node/encrypt_op_test.go
new file mode 100644
index 00000000..3ead6bb3
--- /dev/null
+++ b/internal/topo/node/encrypt_op_test.go
@@ -0,0 +1,78 @@
+// Copyright 2024 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/lf-edge/ekuiper/v2/internal/conf"
+	"github.com/lf-edge/ekuiper/v2/internal/pkg/def"
+	mockContext "github.com/lf-edge/ekuiper/v2/pkg/mock/context"
+)
+
+func TestNewEncryptOp(t *testing.T) {
+	_, err := NewEncryptOp("test", &def.RuleOption{}, "non")
+	assert.Error(t, err)
+	assert.Equal(t, "get encryptor non fail with error: unsupported encryptor: non", err.Error())
+	_, err = NewEncryptOp("test", &def.RuleOption{}, "aes")
+	assert.Error(t, err)
+	assert.Equal(t, errors.New("get encryptor aes fail with error: AES key is not defined"), err)
+}
+
+func TestEncryptOp_Exec(t *testing.T) {
+	conf.InitConf()
+	op, err := NewEncryptOp("test", &def.RuleOption{BufferLength: 10, SendError: true}, "aes")
+	assert.NoError(t, err)
+	op.tool = &MockEncryptor{}
+	out := make(chan any, 100)
+	err = op.AddOutput(out, "test")
+	assert.NoError(t, err)
+	ctx := mockContext.NewMockContext("test1", "compress_test")
+	errCh := make(chan error)
+	op.Exec(ctx, errCh)
+
+	cases := []any{
+		[]byte("{\"a\":1,\"b\":2}"),
+		errors.New("go through error"),
+		"invalid",
+	}
+	expects := [][]any{
+		{[]byte("mock encrypt")},
+		{errors.New("go through error")},
+		{errors.New("unsupported data received: invalid")},
+	}
+
+	for i, c := range cases {
+		op.input <- c
+		for _, e := range expects[i] {
+			r := <-out
+			switch tr := r.(type) {
+			case error:
+				assert.EqualError(t, e.(error), tr.Error())
+			default:
+				assert.Equal(t, e, r)
+			}
+		}
+	}
+}
+
+type MockEncryptor struct{}
+
+func (m *MockEncryptor) Encrypt(_ []byte) []byte {
+	return []byte("mock encrypt")
+}
diff --git a/internal/topo/planner/planner_sink_test.go b/internal/topo/planner/planner_sink_test.go
index 8c53c86a..472c6a1c 100644
--- a/internal/topo/planner/planner_sink_test.go
+++ b/internal/topo/planner/planner_sink_test.go
@@ -120,19 +120,55 @@ func TestSinkPlan(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "encrypt and compress sink plan",
+			rule: &def.Rule{
+				Actions: []map[string]any{
+					{
+						"log": map[string]any{
+							"compression": "gzip",
+							"encryption":  "aes",
+						},
+					},
+				},
+				Options: defaultOption,
+			},
+			topo: &def.PrintableTopo{
+				Sources: []string{"source_src1"},
+				Edges: map[string][]any{
+					"source_src1": {
+						"op_log_0_0_transform",
+					},
+					"op_log_0_0_transform": {
+						"op_log_0_1_encode",
+					},
+					"op_log_0_1_encode": {
+						"op_log_0_2_compress",
+					},
+					"op_log_0_2_compress": {
+						"op_log_0_3_encrypt",
+					},
+					"op_log_0_3_encrypt": {
+						"sink_log_0",
+					},
+				},
+			},
+		},
 	}
 	for _, c := range tc {
-		tp, err := topo.NewWithNameAndOptions("test", c.rule.Options)
-		assert.NoError(t, err)
-		si, err := io.Source("memory")
-		assert.NoError(t, err)
-		n, err := node.NewSourceNode(tp.GetContext(), "src1", si, map[string]any{"datasource": "demo"}, &def.RuleOption{SendError: false})
-		assert.NoError(t, err)
-		tp.AddSrc(n)
-		inputs := []node.Emitter{n}
-		err = buildActions(tp, c.rule, inputs, 1)
-		assert.NoError(t, err)
-		assert.Equal(t, c.topo, tp.GetTopo())
+		t.Run(c.name, func(t *testing.T) {
+			tp, err := topo.NewWithNameAndOptions("test", c.rule.Options)
+			assert.NoError(t, err)
+			si, err := io.Source("memory")
+			assert.NoError(t, err)
+			n, err := node.NewSourceNode(tp.GetContext(), "src1", si, map[string]any{"datasource": "demo"}, &def.RuleOption{SendError: false})
+			assert.NoError(t, err)
+			tp.AddSrc(n)
+			inputs := []node.Emitter{n}
+			err = buildActions(tp, c.rule, inputs, 1)
+			assert.NoError(t, err)
+			assert.Equal(t, c.topo, tp.GetTopo())
+		})
 	}
 }