Repository: threefoldtech/zos
Pull request #601 (2020-03-22 09:19:46)

Make pagination start from page 1 and have default sort
Signed-off-by: Jo De Boeck <deboeck.jo@gmail.com>

codecov[bot]: # [Codecov](https://codecov.io/gh/threefoldtech/zos/pull/601?src=pr&el=h1) Report
> Merging [#601](https://codecov.io/gh/threefoldtech/zos/pull/601?src=pr&el=desc) into [master](https://codecov.io/gh/threefoldtech/zos/commit/205e8bf586b372ca8bd529c1be872045c665569e&el=desc) will **not change** coverage by `%`.
> The diff coverage is `n/a`.

[![Impacted file tree graph](https://codecov.io/gh/threefoldtech/zos/pull/601/graphs/tree.svg?width=650&height=150&src=pr&token=f7Vmq1OAEm)](https://codecov.io/gh/threefoldtech/zos/pull/601?src=pr&el=tree)

```diff
@@           Coverage Diff           @@
##           master     #601   +/-   ##
=======================================
  Coverage   35.90%   35.90%           
=======================================
  Files          71       71           
  Lines        5289     5289           
=======================================
  Hits         1899     1899           
  Misses       3118     3118           
  Partials      272      272           
```



------

[Continue to review full report at Codecov](https://codecov.io/gh/threefoldtech/zos/pull/601?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/threefoldtech/zos/pull/601?src=pr&el=footer). Last update [205e8bf...ccbd522](https://codecov.io/gh/threefoldtech/zos/pull/601?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/cmds/provisiond/main.go b/cmds/provisiond/main.go
index c7ceb13c..4a1488b9 100644
--- a/cmds/provisiond/main.go
+++ b/cmds/provisiond/main.go
@@ -5,6 +5,7 @@ import (
 	"flag"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/threefoldtech/zos/pkg"
@@ -52,6 +53,14 @@ func main() {
 		zerolog.SetGlobalLevel(zerolog.DebugLevel)
 	}
 
+	// keep checking if limited-cache flag is set
+	if app.CheckFlag(app.LimitedCache) {
+		log.Error().Msg("failed cache reservation! Retrying every 30 seconds...")
+		for app.CheckFlag(app.LimitedCache) {
+			time.Sleep(time.Second * 30)
+		}
+	}
+
 	flag.Parse()
 
 	if err := os.MkdirAll(storageDir, 0770); err != nil {
@@ -107,6 +116,7 @@ func main() {
 	)
 
 	engine := provision.New(nodeID.Identity(), source, localStore, remoteStore)
+
 	server.Register(zbus.ObjectID{Name: module, Version: "0.0.1"}, pkg.ProvisionMonitor(engine))
 
 	log.Info().
diff --git a/cmds/storaged/main.go b/cmds/storaged/main.go
index b9f6f731..082bf922 100644
--- a/cmds/storaged/main.go
+++ b/cmds/storaged/main.go
@@ -38,7 +38,7 @@ func main() {
 
 	storageModule, err := storage.New()
 	if err != nil {
-		log.Fatal().Err(err).Msg("Error initializing storage module")
+		log.Fatal().Err(err).Msg("failed to initialize storage module")
 	}
 
 	server, err := zbus.NewRedisServer(module, msgBrokerCon, workerNr)
@@ -46,14 +46,15 @@ func main() {
 		log.Fatal().Err(err).Msg("fail to connect to message broker server")
 	}
 
+	server.Register(zbus.ObjectID{Name: "storage", Version: "0.0.1"}, storageModule)
+
 	vdiskModule, err := storage.NewVDiskModule(storageModule)
 	if err != nil {
-		log.Fatal().Err(err)
+		log.Error().Err(err).Bool("limited-cache", app.CheckFlag(app.LimitedCache)).Msg("failed to initialize virtual disk module")
+	} else {
+		server.Register(zbus.ObjectID{Name: "vdisk", Version: "0.0.1"}, vdiskModule)
 	}
 
-	server.Register(zbus.ObjectID{Name: "storage", Version: "0.0.1"}, storageModule)
-	server.Register(zbus.ObjectID{Name: "vdisk", Version: "0.0.1"}, vdiskModule)
-
 	log.Info().
 		Str("broker", msgBrokerCon).
 		Uint("worker nr", workerNr).
diff --git a/pkg/app/flag.go b/pkg/app/flag.go
new file mode 100644
index 00000000..957e78b4
--- /dev/null
+++ b/pkg/app/flag.go
@@ -0,0 +1,53 @@
+package app
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	flagsDir = "/tmp/flags"
+	// LimitedCache represent the flag cache couldn't mount on ssd or hdd
+	LimitedCache = "limited-cache"
+)
+
+// SetFlag is used when the /var/cache cannot be mounted on a SSD or HDD,
+// it will mount the cache disk on a temporary file system in memory.
+func SetFlag(key string) error {
+	// creating a file that will be used as a flag
+	// the flag can 'warn' other deamons that the /var/cache is not on HDD or SDD
+	if err := os.MkdirAll(flagsDir, os.ModePerm); err != nil {
+		return errors.Wrap(err, "failed to create flags directory")
+	}
+
+	f, err := os.Create(filepath.Join(flagsDir, key))
+	if err != nil {
+		return errors.Wrap(err, "failed to create the flag file")
+	}
+
+	return f.Close()
+}
+
+// CheckFlag checks the status of a flag based on a key
+func CheckFlag(key string) bool {
+	_, err := os.Stat(filepath.Join(flagsDir, key))
+	return !os.IsNotExist(err)
+}
+
+// DeleteFlag deletes (unsets) a given flag based on a key
+func DeleteFlag(key string) error {
+	// to avoid "path injection"
+	path := filepath.Join(flagsDir, key)
+	if filepath.Dir(path) != flagsDir {
+		return fmt.Errorf("trying to delete a directory outside of the flags boundaries")
+	}
+
+	if err := os.RemoveAll(filepath.Join(flagsDir, key)); err != nil {
+		errors.Wrap(err, "failed to remove the flag file")
+		return err
+	}
+	return nil
+}
diff --git a/pkg/container/startup.go b/pkg/container/startup.go
index fa85f45b..ddff1b96 100644
--- a/pkg/container/startup.go
+++ b/pkg/container/startup.go
@@ -17,14 +17,26 @@ type entry struct {
 type args struct {
 	Name string
 	Dir  string
+	Args []string
 	Env  map[string]string
 }
 
 func (e entry) Entrypoint() string {
 	if e.Name == "core.system" ||
 		e.Name == "core.base" && e.Args.Name != "" {
-		return e.Args.Name
+		var buf strings.Builder
+		buf.WriteString(e.Args.Name)
+		for _, arg := range e.Args.Args {
+			buf.WriteRune(' ')
+			arg = strings.Replace(arg, "\"", "\\\"", -1)
+			buf.WriteRune('"')
+			buf.WriteString(arg)
+			buf.WriteRune('"')
+		}
+
+		return buf.String()
 	}
+
 	return ""
 }
 
diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go
index c3722d1d..3413b6d1 100644
--- a/pkg/storage/storage.go
+++ b/pkg/storage/storage.go
@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/google/uuid"
@@ -14,6 +15,7 @@ import (
 	log "github.com/rs/zerolog/log"
 	"github.com/shirou/gopsutil/disk"
 	"github.com/threefoldtech/zos/pkg"
+	"github.com/threefoldtech/zos/pkg/app"
 	"github.com/threefoldtech/zos/pkg/storage/filesystem"
 )
 
@@ -365,7 +367,12 @@ func (s *storageModule) ensureCache() error {
 
 	var cacheFs filesystem.Volume
 
-	// check if we already have a cache
+	if filesystem.IsMountPoint(CacheTarget) {
+		log.Debug().Msgf("Cache partition already mounted in %s", CacheTarget)
+		return nil
+	}
+
+	// check if cache volume available
 	for idx := range s.volumes {
 		filesystems, err := s.volumes[idx].Volumes()
 		if err != nil {
@@ -386,23 +393,41 @@ func (s *storageModule) ensureCache() error {
 	if cacheFs == nil {
 		log.Debug().Msgf("No cache found, try to create new cache")
 
+		log.Debug().Msgf("Trying to create new cache on SSD")
 		fs, err := s.createSubvol(cacheSize, cacheLabel, pkg.SSDDevice)
-		if errors.Is(err, pkg.ErrNotEnoughSpace{}) {
-			// No space on SSD (probably no SSD in the node at all), try HDD
-			fs, err = s.createSubvol(cacheSize, cacheLabel, pkg.HDDDevice)
+
+		if err != nil {
+			log.Warn().Err(err).Msg("failed to create new cache on SSD")
+		} else {
+			cacheFs = fs
 		}
+	}
+
+	if cacheFs == nil {
+		log.Debug().Msgf("Trying to create new cache on HDD")
+		fs, err := s.createSubvol(cacheSize, cacheLabel, pkg.HDDDevice)
+
 		if err != nil {
-			return err
+			log.Warn().Err(err).Msg("failed to create new cache on HDD")
+		} else {
+			cacheFs = fs
 		}
-		cacheFs = fs
 	}
 
-	if !filesystem.IsMountPoint(CacheTarget) {
-		log.Debug().Msgf("Mounting cache partition in %s", CacheTarget)
-		return filesystem.BindMount(cacheFs, CacheTarget)
+	if cacheFs == nil {
+		log.Warn().Msg("failed to create persisted cache disk. Running on limited cache")
+
+		// set limited cache flag
+		if err := app.SetFlag("limited-cache"); err != nil {
+			return err
+		}
+
+		// when everything failed, mount the Tmpfs
+		return syscall.Mount("", "/var/cache", "tmpfs", 0, "size=500M")
 	}
-	log.Debug().Msgf("Cache partition already mounted in %s", CacheTarget)
-	return nil
+
+	log.Debug().Msgf("Mounting cache partition in %s", CacheTarget)
+	return filesystem.BindMount(cacheFs, CacheTarget)
 }
 
 // createSubvol creates a subvolume with the given name and limits it to the given size
diff --git a/tools/bcdb_mock/models/utils.go b/tools/bcdb_mock/models/utils.go
index b670468a..35146c53 100644
--- a/tools/bcdb_mock/models/utils.go
+++ b/tools/bcdb_mock/models/utils.go
@@ -4,6 +4,7 @@ import (
 	"net/http"
 	"strconv"
 
+	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
@@ -22,7 +23,7 @@ func Page(p int64, size ...int64) Pager {
 		ps = size[0]
 	}
 	skip := p * ps
-	return options.Find().SetLimit(ps).SetSkip(skip)
+	return options.Find().SetLimit(ps).SetSkip(skip).SetSort(bson.D{{Key: "_id", Value: -1}})
 }
 
 // PageFromRequest return page information from the page & size url params
@@ -37,6 +38,11 @@ func PageFromRequest(r *http.Request) Pager {
 
 	if len(p) != 0 {
 		page, _ = strconv.ParseInt(p, 10, 64)
+		// user facing pages are start from 1
+		page--
+		if page < 0 {
+			page = 0
+		}
 	}
 	if len(s) != 0 {
 		size, _ = strconv.ParseInt(s, 10, 64)


Test patch:
diff --git a/pkg/app/flag_test.go b/pkg/app/flag_test.go
new file mode 100644
index 00000000..bce547f7
--- /dev/null
+++ b/pkg/app/flag_test.go
@@ -0,0 +1,19 @@
+package app
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDeleteFlag(t *testing.T) {
+	key := LimitedCache
+	err := DeleteFlag(key)
+	require.NoError(t, err)
+}
+
+func TestDeleteBadFlag(t *testing.T) {
+	key := "/"
+	err := DeleteFlag(key)
+	require.Error(t, err)
+}
diff --git a/pkg/container/startup_test.go b/pkg/container/startup_test.go
index 3313c358..6ebc6768 100644
--- a/pkg/container/startup_test.go
+++ b/pkg/container/startup_test.go
@@ -5,6 +5,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/google/shlex"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/BurntSushi/toml"
@@ -19,6 +20,11 @@ name = "core.system"
 [startup.entry.args]
 name = "/start"
 dir = "/data"
+args = [
+	"extra",
+	"argument with spaces",
+	"with \"skip\"",
+]
 
 [startup.entry.args.env]
 DIFFICULTY = "easy"
@@ -37,6 +43,8 @@ func TestParseStartup(t *testing.T) {
 	assert.Equal(t, "core.system", entry.Name)
 	assert.Equal(t, "/start", entry.Args.Name)
 	assert.Equal(t, "/data", entry.Args.Dir)
+	assert.Equal(t, []string{"extra", "argument with spaces", "with \"skip\""}, entry.Args.Args)
+
 	assert.Equal(t, map[string]string{
 		"DIFFICULTY":  "easy",
 		"LEVEL":       "world",
@@ -52,7 +60,12 @@ func TestStartupEntrypoint(t *testing.T) {
 
 	entry, ok := e.Entries["entry"]
 	require.True(t, ok)
-	assert.Equal(t, entry.Entrypoint(), "/start")
+	assert.Equal(t, entry.Entrypoint(), `/start "extra" "argument with spaces" "with \"skip\""`)
+
+	parts, err := shlex.Split(entry.Entrypoint())
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{"/start", "extra", "argument with spaces", "with \"skip\""}, parts)
 }
 
 func TestStartupEnvs(t *testing.T) {