Repository: tomarrell/lbadd
Pull request #94 (2020-03-25 13:06:31)

Add secure file system abstraction
Add a secure file system abstraction. Files are held in memory and encrypted, writes are immediately synced to disk.﻿



Patch:
diff --git a/cmd/lbadd/main.go b/cmd/lbadd/main.go
index 0b70223..e44cb52 100644
--- a/cmd/lbadd/main.go
+++ b/cmd/lbadd/main.go
@@ -71,7 +71,8 @@ waiting for incoming connections from lbadd worker nodes.`
 	startWorkerCmdShortDoc = "Start a worker node"
 	startWorkerCmdLongDoc  = `Start a worker node and connect it to the address that is specified
 in the addr flag. This will start an lbadd worker node, that
-connects to a already running master node on the given address.`
+connects to a already running master node on the given address.
+The used database file will be "db.lbadd".`
 )
 
 var (
@@ -98,11 +99,11 @@ var (
 	}
 
 	startMasterCmd = &cobra.Command{
-		Use:   "master",
+		Use:   "master [database file]",
 		Short: startMasterCmdShortDoc,
 		Long:  startMasterCmdLongDoc,
 		Run:   startMaster,
-		Args:  cobra.NoArgs,
+		Args:  cobra.ExactArgs(1),
 	}
 
 	startWorkerCmd = &cobra.Command{
@@ -180,12 +181,15 @@ func printVersion(cmd *cobra.Command, args []string) {
 func startMaster(cmd *cobra.Command, args []string) {
 	log := cmd.Context().Value(ctxKeyLog).(zerolog.Logger)
 
-	exec := createExecutor(log)
+	databaseFile := args[0]
 
 	masterLog := log.With().
 		Str("component", "master").
+		Str("dbfile", databaseFile).
 		Logger()
 
+	exec := createExecutor(log, databaseFile)
+
 	masterNode := master.New(masterLog, exec)
 	if err := masterNode.ListenAndServe(cmd.Context(), addr); err != nil {
 		log.Error().
@@ -198,12 +202,15 @@ func startMaster(cmd *cobra.Command, args []string) {
 func startWorker(cmd *cobra.Command, args []string) {
 	log := cmd.Context().Value(ctxKeyLog).(zerolog.Logger)
 
-	exec := createExecutor(log)
+	databaseFile := "db.lbadd"
 
 	workerLog := log.With().
 		Str("component", "worker").
+		Str("dbfile", databaseFile).
 		Logger()
 
+	exec := createExecutor(log, databaseFile)
+
 	workerNode := worker.New(workerLog, exec)
 	if err := workerNode.Connect(cmd.Context(), addr); err != nil {
 		log.Error().
@@ -252,11 +259,11 @@ func createLogger(stdin io.Reader, stdout, stderr io.Writer) zerolog.Logger {
 	return log
 }
 
-func createExecutor(log zerolog.Logger) executor.Executor {
+func createExecutor(log zerolog.Logger, databaseFile string) executor.Executor {
 	execLog := log.With().
 		Str("component", "executor").
 		Logger()
 
-	exec := executor.New(execLog)
+	exec := executor.New(execLog, databaseFile)
 	return exec
 }
diff --git a/go.mod b/go.mod
index 985b4de..2526f1a 100644
--- a/go.mod
+++ b/go.mod
@@ -3,9 +3,11 @@ module github.com/tomarrell/lbadd
 go 1.13
 
 require (
+	github.com/awnumar/memguard v0.22.1
 	github.com/google/go-cmp v0.4.0
 	github.com/kr/pretty v0.2.0 // indirect
 	github.com/rs/zerolog v1.18.0
+	github.com/spf13/afero v1.1.2
 	github.com/spf13/cobra v0.0.6
 	github.com/stretchr/testify v1.4.0
 	golang.org/x/text v0.3.2
diff --git a/go.sum b/go.sum
index 7d4dfcb..cb562ef 100644
--- a/go.sum
+++ b/go.sum
@@ -4,6 +4,10 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/awnumar/memcall v0.0.0-20191004114545-73db50fd9f80 h1:8kObYoBO4LNmQ+fLiScBfxEdxF1w2MHlvH/lr9MLaTg=
+github.com/awnumar/memcall v0.0.0-20191004114545-73db50fd9f80/go.mod h1:S911igBPR9CThzd/hYQQmTc9SWNu3ZHIlCGaWsWsoJo=
+github.com/awnumar/memguard v0.22.1 h1:01WQZjYtsfs07y+T+1Fy+qdaTAkGPkRu2r2se/fZaLs=
+github.com/awnumar/memguard v0.22.1/go.mod h1:33OwJBHC+T4eEfFcDrQb78TMlBMBvcOPCXWU9xE34gM=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -121,12 +125,15 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 h1:QmwruyY+bKbDDL0BaglrbZABEali68eoMFhTZpCjYVA=
+golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -140,6 +147,10 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
diff --git a/internal/database/storage/securefs/doc.go b/internal/database/storage/securefs/doc.go
new file mode 100644
index 0000000..c0d883e
--- /dev/null
+++ b/internal/database/storage/securefs/doc.go
@@ -0,0 +1,5 @@
+// Package securefs implements an afero.Fs that reads a file's content into
+// memory when opening or creating it. All data is held encrypted by using
+// github.com/awnumar/memguard. File writes need to be synced manually with the
+// disk contents by calling file.Sync().
+package securefs
diff --git a/internal/database/storage/securefs/secure_file.go b/internal/database/storage/securefs/secure_file.go
new file mode 100644
index 0000000..7528f19
--- /dev/null
+++ b/internal/database/storage/securefs/secure_file.go
@@ -0,0 +1,237 @@
+package securefs
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/awnumar/memguard"
+	"github.com/spf13/afero"
+)
+
+var _ afero.File = (*secureFile)(nil)
+
+type secureFile struct {
+	file afero.File
+
+	enclave *memguard.Enclave
+	pointer int64
+	closed  bool
+}
+
+func newSecureFile(file afero.File) (*secureFile, error) {
+	secureFile := &secureFile{
+		file: file,
+	}
+	if err := secureFile.load(); err != nil {
+		return nil, fmt.Errorf("load: %w", err)
+	}
+	return secureFile, nil
+}
+
+// ReadAt reads len(p) bytes into p, starting from off. If EOF is reached before
+// reading was finished, all read bytes are returned, together with an io.EOF
+// error. BE AWARE THAT BYTES ARE COPIED FROM A SECURE AREA TO A POTENTIALLY
+// INSECURE (your byte slice), AND THAT ALL READ BYTES ARE NO LONGER SECURE.
+func (f *secureFile) ReadAt(p []byte, off int64) (int, error) {
+	if err := f.ensureOpen(); err != nil {
+		return 0, err
+	}
+
+	buffer, err := f.enclave.Open()
+	if err != nil {
+		return 0, fmt.Errorf("open enclave: %w", err)
+	}
+	defer func() {
+		f.enclave = buffer.Seal()
+	}()
+	data := buffer.Bytes()
+
+	n := copy(p, data[off:])
+	if n < len(p) {
+		return n, io.EOF
+	}
+	return n, nil
+}
+
+func (f *secureFile) WriteAt(p []byte, off int64) (n int, err error) {
+	if err = f.ensureOpen(); err != nil {
+		return 0, err
+	}
+
+	if f.enclave == nil || int(off)+len(p) > f.enclave.Size() {
+		if err := f.grow(int(off) + len(p)); err != nil {
+			return 0, fmt.Errorf("grow: %w", err)
+		}
+	}
+
+	buffer, err := f.enclave.Open()
+	if err != nil {
+		return 0, fmt.Errorf("open enclave: %w", err)
+	}
+	defer func() {
+		f.enclave = buffer.Seal()
+
+		if syncErr := f.Sync(); syncErr != nil {
+			err = fmt.Errorf("sync: %w", syncErr)
+		}
+	}()
+	buffer.Melt()
+	data := buffer.Bytes()
+
+	n = copy(data[off:off+int64(len(p))], p)
+	if n != len(p) {
+		return n, fmt.Errorf("unable to write all bytes")
+	}
+	return n, nil
+}
+
+func (f *secureFile) Close() error {
+	if f.closed {
+		return nil
+	}
+
+	if err := f.Sync(); err != nil {
+		return fmt.Errorf("sync: %w", err)
+	}
+	if err := f.file.Close(); err != nil {
+		return fmt.Errorf("close underlying: %w", err)
+	}
+	f.enclave = nil
+	f.closed = true
+	return nil
+}
+
+// Reads len(p) bytes into p and returns the number of bytes read. BE AWARE THAT
+// BYTES ARE COPIED FROM A SECURE AREA TO A POTENTIALLY INSECURE (your byte
+// slice), AND THAT ALL READ BYTES ARE NO LONGER SECURE.
+func (f *secureFile) Read(p []byte) (n int, err error) {
+	if err := f.ensureOpen(); err != nil {
+		return 0, err
+	}
+	return f.ReadAt(p, f.pointer)
+}
+
+func (f *secureFile) Seek(offset int64, whence int) (int64, error) {
+	if err := f.ensureOpen(); err != nil {
+		return 0, err
+	}
+
+	switch whence {
+	case io.SeekCurrent:
+		f.pointer += offset
+	case io.SeekStart:
+		f.pointer = offset
+	case io.SeekEnd:
+		f.pointer = int64(f.enclave.Size()) - offset
+	default:
+		return f.pointer, fmt.Errorf("unsupported whence: %v", whence)
+	}
+	return f.pointer, nil
+}
+
+func (f *secureFile) Write(p []byte) (n int, err error) {
+	if err := f.ensureOpen(); err != nil {
+		return 0, err
+	}
+
+	return f.WriteAt(p, f.pointer)
+}
+
+func (f *secureFile) Name() string {
+	return f.file.Name()
+}
+
+func (f *secureFile) Readdir(count int) ([]os.FileInfo, error) {
+	return f.file.Readdir(count)
+}
+
+func (f *secureFile) Readdirnames(n int) ([]string, error) {
+	return f.file.Readdirnames(n)
+}
+
+func (f *secureFile) Stat() (os.FileInfo, error) {
+	return f.file.Stat()
+}
+
+func (f *secureFile) Sync() error {
+	if err := f.ensureOpen(); err != nil {
+		return err
+	}
+
+	buffer, err := f.enclave.Open()
+	if err != nil {
+		return fmt.Errorf("open enclave: %w", err)
+	}
+	defer func() {
+		f.enclave = buffer.Seal()
+	}()
+
+	if err = f.file.Truncate(0); err != nil {
+		return fmt.Errorf("truncate: %w", err)
+	}
+	// Truncate alone doesn't work for, golang doesn't touch the file position
+	// indicator on truncate at all
+	_, err = f.file.Seek(0, io.SeekStart)
+	if err != nil {
+		return fmt.Errorf("seek: %w", err)
+	}
+
+	_, err = buffer.Reader().WriteTo(f.file)
+	if err != nil {
+		return fmt.Errorf("write to: %w", err)
+	}
+	return nil
+}
+
+func (f *secureFile) Truncate(size int64) error {
+	if err := f.grow(int(size)); err != nil {
+		return fmt.Errorf("grow: %w", err)
+	}
+	return nil
+}
+
+func (f *secureFile) WriteString(s string) (ret int, err error) {
+	if err := f.ensureOpen(); err != nil {
+		return 0, err
+	}
+
+	return f.Write([]byte(s))
+}
+
+func (f *secureFile) load() error {
+	buffer, err := memguard.NewBufferFromEntireReader(f.file)
+	if err != nil {
+		return fmt.Errorf("read all: %w", err)
+	}
+	f.enclave = buffer.Seal()
+	return nil
+}
+
+func (f *secureFile) grow(newSize int) error {
+	if f.enclave == nil {
+		f.enclave = memguard.NewBuffer(newSize).Seal()
+		return nil
+	}
+
+	oldBuffer, err := f.enclave.Open()
+	if err != nil {
+		return fmt.Errorf("open enclave: %w", err)
+	}
+	defer oldBuffer.Destroy()
+
+	// allocate new memory and copy old data
+	newBuffer := memguard.NewBuffer(newSize)
+	newBuffer.Melt()
+	newBuffer.Copy(oldBuffer.Bytes())
+
+	f.enclave = newBuffer.Seal()
+	return nil
+}
+
+func (f *secureFile) ensureOpen() error {
+	if f.closed {
+		return afero.ErrFileClosed
+	}
+	return nil
+}
diff --git a/internal/database/storage/securefs/secure_fs.go b/internal/database/storage/securefs/secure_fs.go
new file mode 100644
index 0000000..5dbd675
--- /dev/null
+++ b/internal/database/storage/securefs/secure_fs.go
@@ -0,0 +1,85 @@
+package securefs
+
+import (
+	"os"
+	"time"
+
+	"github.com/spf13/afero"
+)
+
+var _ afero.Fs = (*secureFs)(nil)
+
+type secureFs struct {
+	fs afero.Fs
+}
+
+// New creates a new secure fs, that only holds data in memory encrypted. Read
+// operations read bytes in passed in byte slices, which makes the bytes leave a
+// protected area. The given byte slice should also be in a protected area, but
+// this is the caller's responsibility. Files that are opened and/or created
+// with the returned Fs are 100% in memory.
+func New(fs afero.Fs) afero.Fs {
+	return &secureFs{
+		fs: fs,
+	}
+}
+
+func (fs *secureFs) Create(name string) (afero.File, error) {
+	file, err := fs.fs.Create(name)
+	if err != nil {
+		return nil, err
+	}
+	return newSecureFile(file)
+}
+
+func (fs *secureFs) Mkdir(name string, perm os.FileMode) error {
+	return fs.fs.Mkdir(name, perm)
+}
+
+func (fs *secureFs) MkdirAll(path string, perm os.FileMode) error {
+	return fs.fs.MkdirAll(path, perm)
+}
+
+func (fs *secureFs) Open(name string) (afero.File, error) {
+	file, err := fs.fs.Open(name)
+	if err != nil {
+		return nil, err
+	}
+	return newSecureFile(file)
+}
+
+func (fs *secureFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
+	file, err := fs.fs.OpenFile(name, flag, perm)
+	if err != nil {
+		return nil, err
+	}
+	return newSecureFile(file)
+}
+
+func (fs *secureFs) Remove(name string) error {
+	return fs.fs.Remove(name)
+}
+
+func (fs *secureFs) RemoveAll(path string) error {
+	return fs.fs.RemoveAll(path)
+}
+
+func (fs *secureFs) Rename(oldname, newname string) error {
+	return fs.fs.Rename(oldname, newname)
+}
+
+func (fs *secureFs) Stat(name string) (os.FileInfo, error) {
+	return fs.fs.Stat(name)
+}
+
+func (fs *secureFs) Name() string {
+	return "secure/" + fs.fs.Name()
+}
+
+func (fs *secureFs) Chmod(name string, mode os.FileMode) error {
+	return fs.fs.Chmod(name, mode)
+}
+
+func (fs *secureFs) Chtimes(name string, atime time.Time, mtime time.Time) error {
+	return fs.fs.Chtimes(name, atime, mtime)
+}
diff --git a/internal/executor/executor.go b/internal/executor/executor.go
index 5902d3f..10638a2 100644
--- a/internal/executor/executor.go
+++ b/internal/executor/executor.go
@@ -15,6 +15,6 @@ type Executor interface {
 }
 
 // New creates a new, ready to use Executor.
-func New(log zerolog.Logger) Executor {
-	return newSimpleExecutor(log)
+func New(log zerolog.Logger, databaseFile string) Executor {
+	return newSimpleExecutor(log, databaseFile)
 }
diff --git a/internal/executor/simple_executor.go b/internal/executor/simple_executor.go
index d6292f8..8172a27 100644
--- a/internal/executor/simple_executor.go
+++ b/internal/executor/simple_executor.go
@@ -10,12 +10,14 @@ import (
 var _ Executor = (*simpleExecutor)(nil)
 
 type simpleExecutor struct {
-	log zerolog.Logger
+	log          zerolog.Logger
+	databaseFile string
 }
 
-func newSimpleExecutor(log zerolog.Logger) *simpleExecutor {
+func newSimpleExecutor(log zerolog.Logger, databaseFile string) *simpleExecutor {
 	return &simpleExecutor{
-		log: log,
+		log:          log,
+		databaseFile: databaseFile,
 	}
 }
 


Test patch:
diff --git a/internal/database/storage/securefs/secure_fs_test.go b/internal/database/storage/securefs/secure_fs_test.go
new file mode 100644
index 0000000..f1b314f
--- /dev/null
+++ b/internal/database/storage/securefs/secure_fs_test.go
@@ -0,0 +1,52 @@
+package securefs_test
+
+import (
+	"io"
+	"io/ioutil"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/tomarrell/lbadd/internal/database/storage/securefs"
+)
+
+func mustRead(t *testing.T, r io.Reader) []byte {
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		assert.NoError(t, err)
+	}
+	return data
+}
+
+func TestSecureFs_FileOperations(t *testing.T) {
+	assert := assert.New(t)
+
+	underlying := afero.NewMemMapFs()
+	fs := securefs.New(underlying)
+
+	filename := "myfile.dat"
+	content := "hello, world!"
+	modContent := "hello, World!"
+
+	file, err := fs.Create(filename)
+	assert.NoError(err)
+	assert.Equal(filename, file.Name())
+
+	n, err := file.WriteString(content)
+	assert.NoError(err)
+	assert.Equal(len(content), n)
+
+	underlyingFile, err := underlying.Open(filename)
+	assert.NoError(err)
+	assert.Equal(content, string(mustRead(t, underlyingFile)))
+	assert.NoError(underlyingFile.Close())
+
+	n, err = file.WriteAt([]byte("W"), 7)
+	assert.Equal(1, n)
+	assert.NoError(err)
+
+	underlyingFile, err = underlying.Open(filename)
+	assert.NoError(err)
+	assert.Equal(modContent, string(mustRead(t, underlyingFile)))
+	assert.NoError(underlyingFile.Close())
+}