Repository: threefoldtech/zos
Pull request #970 (2020-09-23 13:38:13)

Container/secure logs url
Fiexs #945 
Relates to https://github.com/threefoldtech/js-sdk/pull/1260


Patch:
diff --git a/.github/workflows/bins-extra-pre-release.yaml b/.github/workflows/bins-extra-pre-release.yaml
index 5e51239d..f7032399 100644
--- a/.github/workflows/bins-extra-pre-release.yaml
+++ b/.github/workflows/bins-extra-pre-release.yaml
@@ -432,7 +432,7 @@ jobs:
         root: bins/releases/zufs
         name: ${{ steps.package.outputs.name }}.flist
 
-    - name: Crosslink flist (tf-zos-bins.dev)
+    - name: Crosslink flist (tf-zos-bins.test)
       if: success()
       uses: threefoldtech/publish-flist@master
       with:
diff --git a/.github/workflows/bins-extra-release.yaml b/.github/workflows/bins-extra-release.yaml
index 526901cb..837e0415 100644
--- a/.github/workflows/bins-extra-release.yaml
+++ b/.github/workflows/bins-extra-release.yaml
@@ -433,7 +433,7 @@ jobs:
         root: bins/releases/zufs
         name: ${{ steps.package.outputs.name }}.flist
 
-    - name: Crosslink flist (tf-zos-bins.dev)
+    - name: Crosslink flist (tf-zos-bins)
       if: success()
       uses: threefoldtech/publish-flist@master
       with:
diff --git a/go.mod b/go.mod
index d4dbb5f1..e2707937 100644
--- a/go.mod
+++ b/go.mod
@@ -60,7 +60,7 @@ require (
 	github.com/stretchr/testify v1.6.1
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
 	github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae
-	github.com/threefoldtech/tfexplorer v0.3.2-0.20200918135528-58e333897454
+	github.com/threefoldtech/tfexplorer v0.4.1-0.20200923122946-a429c33ade21
 	github.com/threefoldtech/zbus v0.1.3
 	github.com/urfave/cli v1.22.4
 	github.com/vishvananda/netlink v1.1.0
diff --git a/go.sum b/go.sum
index ebef0dfc..36ccb234 100644
--- a/go.sum
+++ b/go.sum
@@ -559,9 +559,11 @@ github.com/spf13/viper v0.0.0-20150621231900-db7ff930a189/go.mod h1:A8kyI5cUJhb8
 github.com/stellar/go v0.0.0-20200325172527-9cabbc6b9388/go.mod h1:hmRfHZUrIkvTCrZ5kAQa4uwUa2OsSeFhHh6urLlnPQg=
 github.com/stellar/go v0.0.0-20200520124219-6cdb4e841dc7 h1:a98X4Lbv4mYreR7GVd4vNhazQk7ZbMT2v0r0UAuZXcA=
 github.com/stellar/go v0.0.0-20200520124219-6cdb4e841dc7/go.mod h1:Q4JDgZhGw7Ytu1924PzNkwr2GsLptrzAjwC/icuXnLU=
+github.com/stellar/go v0.0.0-20200917104244-8dc2a7354bf8 h1:hdrW8jhTrR6bagF4gzfItdiI/xeRv3FjptWco2Toz7I=
 github.com/stellar/go v0.0.0-20200917104244-8dc2a7354bf8/go.mod h1:PGVMUzNBHfCpregVTwNK1uNTQTgsw9yrZZN53csq5JA=
 github.com/stellar/go-xdr v0.0.0-20180917104419-0bc96f33a18e h1:n/hfey8pO+RYMoGXyvyzuw5pdO8IFDoyAL/g5OiCesY=
 github.com/stellar/go-xdr v0.0.0-20180917104419-0bc96f33a18e/go.mod h1:gpOLVzy6TVYTQ3LvHSN9RJC700FkhFCpSE82u37aNRM=
+github.com/stellar/go-xdr v0.0.0-20200331223602-71a1e6d555f2 h1:K9H+A+eWe8ZlnpNha+pXbEK+jtIluQp/2dKxkK8k7OE=
 github.com/stellar/go-xdr v0.0.0-20200331223602-71a1e6d555f2/go.mod h1:yoxyU/M8nl9LKeWIoBrbDPQ7Cy+4jxRcWcOayZ4BMps=
 github.com/stellar/throttled v2.2.3-0.20190823235211-89d75816f59d+incompatible/go.mod h1:7CJ23pXirXBJq45DqvO6clzTEGM/l1SfKrgrzLry8b4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -586,10 +588,13 @@ github.com/threefoldtech/tfexplorer v0.3.2-0.20200824142536-b4319e0a006d h1:/4U3
 github.com/threefoldtech/tfexplorer v0.3.2-0.20200824142536-b4319e0a006d/go.mod h1:oFtPBaRLW4c6YuEfTrWYzAGw+wb6I4+og133zjDOYP0=
 github.com/threefoldtech/tfexplorer v0.3.2-0.20200918135528-58e333897454 h1:MUtGnBw3P5Zp6CS26dOw6/WksisommGCd4hqP4G4D3w=
 github.com/threefoldtech/tfexplorer v0.3.2-0.20200918135528-58e333897454/go.mod h1:fA2oR3D6Wre9tXSiWFiapeuUI5UA9aXP9fejr6hDEYM=
+github.com/threefoldtech/tfexplorer v0.4.1-0.20200923122946-a429c33ade21 h1:p3Hpq3bmBNKDaMCA7gn8ltMk84ZAU9OurMUF1K93Kqk=
+github.com/threefoldtech/tfexplorer v0.4.1-0.20200923122946-a429c33ade21/go.mod h1:wxnWTecsczkzkiJoZB19tt49l8CWkTNLD2VkKBK6mSc=
 github.com/threefoldtech/zbus v0.1.3 h1:18DnIzximRbATle5ZdZz0i84n/bCYB8k/gkhr2dXayc=
 github.com/threefoldtech/zbus v0.1.3/go.mod h1:ZtiRpcqzEBJetVQDsEbw0p48h/AF3O1kf0tvd30I0BU=
 github.com/threefoldtech/zos v0.2.4-rc2/go.mod h1:7A2oflcmSVsHFC4slOcydWgJyFBMFMH9wsaTRv+CnTA=
 github.com/threefoldtech/zos v0.2.4/go.mod h1:B1SeNF7YWpDEUFX4dgAXImHO49HB5zDuD+q3YRY5xvA=
+github.com/threefoldtech/zos v0.4.0-rc9-b.0.20200918140104-b46553b0c680/go.mod h1:Fyh+Cwa2GlUwlDpu4SlZttW//fKgWc1RePtKSuJAfdE=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tyler-smith/go-bip39 v0.0.0-20180618194314-52158e4697b8/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
diff --git a/pkg/container/logger/logger.go b/pkg/container/logger/logger.go
index 21e92ae4..84edc825 100644
--- a/pkg/container/logger/logger.go
+++ b/pkg/container/logger/logger.go
@@ -1,5 +1,19 @@
 package logger
 
+import (
+	"encoding/json"
+	"io/ioutil"
+)
+
+// RedisType defines redis logger type name
+const RedisType = "redis"
+
+// FileType defines file logger type name
+const FileType = "file"
+
+// ConsoleType defines console logger type name
+const ConsoleType = "console"
+
 // Logs defines a custom backend with variable settings
 type Logs struct {
 	Type string    `json:"type"`
@@ -14,3 +28,29 @@ type LogsRedis struct {
 	// Stderr is the redis url for stderr (redis://host/channel)
 	Stderr string `json:"stderr"`
 }
+
+// Serialize dumps logs array into a json file
+func Serialize(path string, logs []Logs) error {
+	data, err := json.Marshal(logs)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(path, data, 0644)
+}
+
+// Deserialize reads json from disks and returns []Logs
+func Deserialize(path string) ([]Logs, error) {
+	logs := []Logs{}
+
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return logs, err
+	}
+
+	if err := json.Unmarshal(data, &logs); err != nil {
+		return logs, err
+	}
+
+	return logs, nil
+}
diff --git a/pkg/container/logger/logger_console.go b/pkg/container/logger/logger_console.go
deleted file mode 100644
index 081c2e64..00000000
--- a/pkg/container/logger/logger_console.go
+++ /dev/null
@@ -1,45 +0,0 @@
-package logger
-
-import (
-	"fmt"
-	"io"
-	"os"
-
-	"github.com/rs/zerolog/log"
-)
-
-// ConsoleType defines console logger type name
-const ConsoleType = "console"
-
-// Console does nothing else that print
-// logs on console stdout/stderr, there are no config
-type Console struct {
-	prefix string
-	target *os.File
-}
-
-// NewConsole does nothing, it's here for consistancy
-func NewConsole() (io.Writer, io.Writer, error) {
-	log.Debug().Msg("initializing console logging")
-	stdout := &Console{
-		prefix: "O: ",
-		target: os.Stdout,
-	}
-
-	stderr := &Console{
-		prefix: "E: ",
-		target: os.Stderr,
-	}
-
-	return stdout, stderr, nil
-}
-
-func (c *Console) Write(data []byte) (n int, err error) {
-	fmt.Fprintf(c.target, "%s", c.prefix)
-
-	// note: cannot use Fprintf("%s%s", c.prefix, data)
-	// caller seems to expect that return amount of byte
-	// matches to data length
-
-	return c.target.Write(data)
-}
diff --git a/pkg/container/logger/logger_file.go b/pkg/container/logger/logger_file.go
deleted file mode 100644
index 7a0b4737..00000000
--- a/pkg/container/logger/logger_file.go
+++ /dev/null
@@ -1,60 +0,0 @@
-package logger
-
-import (
-	"io"
-	"os"
-
-	"github.com/rs/zerolog/log"
-)
-
-// FileType defines file logger type name
-const FileType = "file"
-
-// File write stdout/stderr to files
-type File struct {
-	target *os.File
-}
-
-// NewFile open file and prepare logs writing
-func NewFile(stdout string, stderr string) (io.Writer, io.Writer, error) {
-	log.Debug().Str("stdout", stdout).Str("stderr", stderr).Msg("initializing localfile logging")
-
-	fo, err := os.Create(stdout)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	// If stdout and stderr are the same, only one file is open
-	fe := fo
-
-	if stdout != stderr {
-		fe, err = os.Create(stderr)
-		if err != nil {
-			return nil, nil, err
-		}
-	}
-
-	fstdout := &File{
-		target: fo,
-	}
-
-	fstderr := &File{
-		target: fe,
-	}
-
-	return fstdout, fstderr, nil
-}
-
-// Write forwards write to underlaying layer
-func (c *File) Write(data []byte) (int, error) {
-	n, err := c.target.Write(data)
-	if err != nil {
-		log.Error().Err(err).Msg("log file write")
-	}
-
-	if n != len(data) {
-		log.Error().Int("expected", len(data)).Int("written", n).Msg("log file write not complete")
-	}
-
-	return n, err
-}
diff --git a/pkg/container/logger/logger_redis.go b/pkg/container/logger/logger_redis.go
deleted file mode 100644
index a2de265e..00000000
--- a/pkg/container/logger/logger_redis.go
+++ /dev/null
@@ -1,104 +0,0 @@
-package logger
-
-import (
-	"fmt"
-	"github.com/gomodule/redigo/redis"
-	"github.com/rs/zerolog/log"
-	"io"
-	"net/url"
-)
-
-// RedisType defines redis logger type name
-const RedisType = "redis"
-
-// Redis send stdout/stderr to a Redis PubSub channel
-type Redis struct {
-	channel string
-	conn    redis.Conn
-}
-
-// RedisParseURL parse an url and returns interresting part after validation
-func RedisParseURL(address string) (host string, channel string, err error) {
-	u, err := url.Parse(address)
-	if err != nil {
-		return "", "", err
-	}
-
-	if u.Scheme != "redis" {
-		err = fmt.Errorf("invalid scheme, expected redis://")
-		return "", "", err
-	}
-
-	if u.RequestURI() == "/" {
-		err = fmt.Errorf("missing channel name, expected: redis://host/channel")
-		return "", "", err
-	}
-
-	host = u.Host
-
-	if u.Port() == "" {
-		host += ":6379"
-	}
-
-	channel = u.RequestURI()[1:]
-
-	return host, channel, err
-}
-
-// NewRedis create new redis backend and initialize connection
-func NewRedis(stdout string, stderr string) (io.WriteCloser, io.WriteCloser, error) {
-	log.Debug().Msg("initializing redis logging")
-
-	ohost, ochannel, err := RedisParseURL(stdout)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	rhost, rchannel, err := RedisParseURL(stderr)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	cout, err := redis.Dial("tcp", ohost)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	cerr, err := redis.Dial("tcp", rhost)
-	if err != nil {
-		cout.Close()
-		return nil, nil, err
-	}
-
-	log.Debug().Str("stdout", ohost).Str("channel", ochannel).Msg("redis stdout")
-	log.Debug().Str("stderr", rhost).Str("channel", rchannel).Msg("redis stder")
-
-	rstdout := &Redis{
-		channel: ochannel,
-		conn:    cout,
-	}
-
-	rstderr := &Redis{
-		channel: rchannel,
-		conn:    cerr,
-	}
-
-	return rstdout, rstderr, nil
-}
-
-// Write will write to the channel
-func (c *Redis) Write(data []byte) (int, error) {
-	_, err := c.conn.Do("PUBLISH", c.channel, data)
-	if err != nil {
-		return 0, err
-	}
-
-	return len(data), nil
-}
-
-// Close closes redis connection
-func (c *Redis) Close() error {
-	log.Debug().Str("channel", c.channel).Msg("closing redis backend")
-	c.conn.Close()
-	return nil
-}
diff --git a/pkg/container/logger/loggers.go b/pkg/container/logger/loggers.go
deleted file mode 100644
index 19bc5b6d..00000000
--- a/pkg/container/logger/loggers.go
+++ /dev/null
@@ -1,73 +0,0 @@
-package logger
-
-import (
-	"encoding/json"
-	"io"
-	"io/ioutil"
-
-	"github.com/containerd/containerd/cio"
-)
-
-// Loggers keeps stdout and stderr backend list
-type Loggers struct {
-	stdouts []io.Writer
-	stderrs []io.Writer
-}
-
-// Serialize dumps logs array into a json file
-func Serialize(path string, logs []Logs) error {
-	data, err := json.Marshal(logs)
-	if err != nil {
-		return err
-	}
-
-	return ioutil.WriteFile(path, data, 0644)
-}
-
-// Deserialize reads json from disks and returns []Logs
-func Deserialize(path string) ([]Logs, error) {
-	logs := []Logs{}
-
-	data, err := ioutil.ReadFile(path)
-	if err != nil {
-		return logs, err
-	}
-
-	if err := json.Unmarshal(data, &logs); err != nil {
-		return logs, err
-	}
-
-	return logs, nil
-}
-
-// NewLoggers initialize empty lists
-func NewLoggers() *Loggers {
-	return &Loggers{
-		stdouts: []io.Writer{},
-		stderrs: []io.Writer{},
-	}
-}
-
-// Add adds a defined backend on the list
-func (c *Loggers) Add(stdout io.Writer, stderr io.Writer) {
-	c.stdouts = append(c.stdouts, stdout)
-	c.stderrs = append(c.stderrs, stderr)
-}
-
-// Stdouts returns list of stdout availables
-func (c *Loggers) Stdouts() []io.Writer {
-	return c.stdouts
-}
-
-// Stderrs returns list of stderr availables
-func (c *Loggers) Stderrs() []io.Writer {
-	return c.stderrs
-}
-
-// Log create the containers logs redirector
-func (c *Loggers) Log() cio.Creator {
-	mwo := io.MultiWriter(c.stdouts...)
-	mwe := io.MultiWriter(c.stderrs...)
-
-	return cio.NewCreator(cio.WithStreams(nil, mwo, mwe))
-}
diff --git a/pkg/provision/primitives/container.go b/pkg/provision/primitives/container.go
index fe3347c0..6ff4d946 100644
--- a/pkg/provision/primitives/container.go
+++ b/pkg/provision/primitives/container.go
@@ -35,6 +35,27 @@ type Mount struct {
 	Mountpoint string `json:"mountpoint"`
 }
 
+// Logs defines a custom backend with variable settings
+type Logs struct {
+	Type string   `json:"type"`
+	Data LogsData `json:"data"`
+}
+
+// LogsData structure
+type LogsData struct {
+	// Stdout is the redis url for stdout (redis://host/channel)
+	Stdout string `json:"stdout"`
+
+	// Stderr is the redis url for stderr (redis://host/channel)
+	Stderr string `json:"stderr"`
+
+	// SecretStdout like stdout but encrypted with node public key
+	SecretStdout string `json:"secret_stdout"`
+
+	// SecretStderr like stderr but encrypted with node public key
+	SecretStderr string `json:"secret_stderr"`
+}
+
 //Container creation info
 type Container struct {
 	// URL of the flist
@@ -58,7 +79,7 @@ type Container struct {
 	// ContainerCapacity is the amount of resource to allocate to the container
 	Capacity ContainerCapacity `json:"capacity"`
 	// Logs contains a list of endpoint where to send containerlogs
-	Logs []logger.Logs `json:"logs,omitempty"`
+	Logs []Logs `json:"logs,omitempty"`
 	// StatsAggregator container metrics backend
 	StatsAggregator []stats.Aggregator
 }
@@ -155,6 +176,33 @@ func (p *Provisioner) containerProvisionImpl(ctx context.Context, reservation *p
 		env = append(env, fmt.Sprintf("%s=%s", k, v))
 	}
 
+	var logs []logger.Logs
+	for _, log := range config.Logs {
+		stdout := log.Data.Stdout
+		stderr := log.Data.Stderr
+
+		if len(log.Data.SecretStdout) > 0 {
+			stdout, err = decryptSecret(p.zbus, log.Data.SecretStdout)
+			if err != nil {
+				return ContainerResult{}, errors.Wrap(err, "failed to decrypt log.secret_stdout var")
+			}
+		}
+
+		if len(log.Data.SecretStderr) > 0 {
+			stderr, err = decryptSecret(p.zbus, log.Data.SecretStderr)
+			if err != nil {
+				return ContainerResult{}, errors.Wrap(err, "failed to decrypt log.secret_stdout var")
+			}
+		}
+		logs = append(logs, logger.Logs{
+			Type: log.Type,
+			Data: logger.LogsRedis{
+				Stdout: stdout,
+				Stderr: stderr,
+			},
+		})
+	}
+
 	// prepare container network
 	ips := make([]string, len(config.Network.IPs))
 	for i, ip := range config.Network.IPs {
@@ -252,7 +300,7 @@ func (p *Provisioner) containerProvisionImpl(ctx context.Context, reservation *p
 			Interactive:     config.Interactive,
 			CPU:             config.Capacity.CPU,
 			Memory:          config.Capacity.Memory * mib,
-			Logs:            config.Logs,
+			Logs:            logs,
 			StatsAggregator: config.StatsAggregator,
 		},
 	)
diff --git a/pkg/provision/primitives/converter.go b/pkg/provision/primitives/converter.go
index 2b4eb901..f4fb59e4 100644
--- a/pkg/provision/primitives/converter.go
+++ b/pkg/provision/primitives/converter.go
@@ -46,7 +46,7 @@ func ContainerToProvisionType(w workloads.Workloader, reservationID string) (Con
 		Entrypoint:      c.Entrypoint,
 		Interactive:     c.Interactive,
 		Mounts:          make([]Mount, len(c.Volumes)),
-		Logs:            make([]logger.Logs, len(c.Logs)),
+		Logs:            make([]Logs, len(c.Logs)),
 		StatsAggregator: make([]stats.Aggregator, len(c.StatsAggregator)),
 		Capacity: ContainerCapacity{
 			CPU:      uint(c.Capacity.Cpu),
@@ -78,20 +78,18 @@ func ContainerToProvisionType(w workloads.Workloader, reservationID string) (Con
 	for i, lg := range c.Logs {
 		// Only support redis for now
 		if lg.Type != logger.RedisType {
-			container.Logs[i] = logger.Logs{
+			container.Logs[i] = Logs{
 				Type: "unknown",
-				Data: logger.LogsRedis{
-					Stdout: "",
-					Stderr: "",
-				},
 			}
 		}
 
-		container.Logs[i] = logger.Logs{
+		container.Logs[i] = Logs{
 			Type: lg.Type,
-			Data: logger.LogsRedis{
-				Stdout: lg.Data.Stdout,
-				Stderr: lg.Data.Stderr,
+			Data: LogsData{
+				Stdout:       lg.Data.Stdout,
+				Stderr:       lg.Data.Stderr,
+				SecretStdout: lg.Data.SecretStdout,
+				SecretStderr: lg.Data.SecretStderr,
 			},
 		}
 	}


Test patch:
diff --git a/pkg/provision/primitives/converter_test.go b/pkg/provision/primitives/converter_test.go
index 79d84b27..cd9a43c6 100644
--- a/pkg/provision/primitives/converter_test.go
+++ b/pkg/provision/primitives/converter_test.go
@@ -12,7 +12,6 @@ import (
 
 	"github.com/stretchr/testify/require"
 	schema "github.com/threefoldtech/tfexplorer/schema"
-	"github.com/threefoldtech/zos/pkg/container/logger"
 	"github.com/threefoldtech/zos/pkg/container/stats"
 	"gotest.tools/assert"
 )
@@ -81,7 +80,7 @@ func TestTfgridReservationContainer1_ToProvisionType(t *testing.T) {
 				Interactive:     false,
 				Mounts:          []Mount{},
 				Network:         Network{},
-				Logs:            []logger.Logs{},
+				Logs:            []Logs{},
 				StatsAggregator: []stats.Aggregator{},
 				Capacity: ContainerCapacity{
 					CPU:      2,
@@ -146,7 +145,7 @@ func TestTfgridReservationContainer1_ToProvisionType(t *testing.T) {
 					NetworkID: "net1",
 					IPs:       []net.IP{net.ParseIP("10.0.0.1")},
 				},
-				Logs:            []logger.Logs{},
+				Logs:            []Logs{},
 				StatsAggregator: []stats.Aggregator{},
 				Capacity: ContainerCapacity{
 					CPU:      2,