Repository: threefoldtech/zos
Pull request #999 (2020-10-14 14:13:10)

cleanup cached files
Make sure cached files older than 30 days are cleaned from the cache.


Patch:
diff --git a/cmds/flistd/main.go b/cmds/flistd/main.go
index b4e5d558..ea10e7de 100644
--- a/cmds/flistd/main.go
+++ b/cmds/flistd/main.go
@@ -16,7 +16,12 @@ import (
 	"github.com/threefoldtech/zos/pkg/version"
 )
 
-const module = "flist"
+const (
+	module = "flist"
+
+	cacheAge     = time.Hour * 24 * 30 // 30 days
+	cacheCleanup = time.Hour * 24
+)
 
 func main() {
 	app.Initialize()
@@ -55,7 +60,8 @@ func main() {
 	ctx, _ := utils.WithSignal(context.Background())
 
 	if cleaner, ok := mod.(flist.Cleaner); ok {
-		go cleaner.Cleaner(ctx, time.Minute)
+		go cleaner.MountsCleaner(ctx, time.Minute)
+		go cleaner.CacheCleaner(ctx, cacheCleanup, cacheAge)
 	}
 
 	log.Info().
diff --git a/pkg/flist/cleanup.go b/pkg/flist/cleanup.go
index 05ebee94..cd12993d 100644
--- a/pkg/flist/cleanup.go
+++ b/pkg/flist/cleanup.go
@@ -17,9 +17,10 @@ import (
 // Cleaner interface, implementer of this interface
 // can start a cleaner job
 type Cleaner interface {
-	// Cleaner runs the clean process, Cleaner should be
-	// blocking. Caller then can do `go Cleaner()` to run it in the background
-	Cleaner(ctx context.Context, every time.Duration)
+	// MountsCleaner runs the clean process, MountsCleaner should be
+	// blocking. Caller then can do `go MountsCleaner()` to run it in the background
+	MountsCleaner(ctx context.Context, every time.Duration)
+	CacheCleaner(ctx context.Context, every time.Duration, age time.Duration)
 }
 
 var _ Cleaner = (*flistModule)(nil)
@@ -127,7 +128,7 @@ func (f *flistModule) cleanupAll() error {
 
 // Cleaner runs forever, checks the tracker files for filesystem processes
 // that requires cleanup
-func (f *flistModule) Cleaner(ctx context.Context, every time.Duration) {
+func (f *flistModule) MountsCleaner(ctx context.Context, every time.Duration) {
 	log := app.SampledLogger()
 
 	for {
@@ -141,3 +142,51 @@ func (f *flistModule) Cleaner(ctx context.Context, every time.Duration) {
 		}
 	}
 }
+
+func (f *flistModule) CacheCleaner(ctx context.Context, every time.Duration, age time.Duration) {
+	log := app.SampledLogger()
+
+	// we need to run it at least one time on
+	// entry
+	if err := f.cleanCache(time.Now(), age); err != nil {
+		log.Error().Err(err).Msg("failed to cleanup cache")
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+		case <-time.After(every):
+			log.Debug().Msg("running cache cleaner job")
+			if err := f.cleanCache(time.Now(), age); err != nil {
+				log.Error().Err(err).Msg("failed to clean cache")
+			}
+		}
+	}
+}
+
+func (f *flistModule) cleanCache(now time.Time, age time.Duration) error {
+	return filepath.Walk(f.cache, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		sys := info.Sys()
+
+		if sys == nil {
+			log.Debug().Str("path", path).Msg("failed to check stat of cached file")
+			return nil
+		}
+
+		if sys, ok := sys.(*syscall.Stat_t); ok {
+			atime := time.Unix(sys.Atim.Sec, sys.Atim.Nsec)
+
+			if now.Sub(atime) > age {
+				if err := os.Remove(path); err != nil {
+					log.Error().Err(err).Msg("failed to delete cached file")
+				}
+			}
+		}
+
+		return nil
+	})
+}


Test patch:
diff --git a/pkg/flist/cleaner_test.go b/pkg/flist/cleaner_test.go
new file mode 100644
index 00000000..7b0242be
--- /dev/null
+++ b/pkg/flist/cleaner_test.go
@@ -0,0 +1,67 @@
+package flist
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCacheCleaner(t *testing.T) {
+
+	cache, err := ioutil.TempDir("", "clean-cache-*")
+
+	defer os.RemoveAll(cache)
+
+	require.NoError(t, err)
+	flister := flistModule{
+		cache: cache,
+	}
+
+	// fixing the now
+	now := time.Now()
+
+	create := func(name string, atime time.Time) error {
+		file, err := os.Create(name)
+		if err != nil {
+			return err
+		}
+
+		file.Close()
+
+		cmd := exec.Command("touch",
+			"-a",
+			// format from touch man page is [[CC]YY]MMDDhhmm[.ss]
+			"-t", atime.Format("200601021504"),
+			name)
+
+		return cmd.Run()
+
+	}
+
+	count := 100
+	// each file is one day older than previous one.
+	for i := 0; i < count; i++ {
+		name := filepath.Join(cache, fmt.Sprintf("file-%02d", i))
+		atime := now.Add(time.Duration(-i*24) * time.Hour)
+		err := create(name, atime)
+		require.NoError(t, err)
+	}
+
+	err = flister.cleanCache(now, 50*24*time.Hour) // this should delete 50 files!
+	require.NoError(t, err)
+
+	files, err := ioutil.ReadDir(cache)
+	require.NoError(t, err)
+
+	assert.Len(t, files, 50)
+
+	assert.Equal(t, "file-00", files[0].Name())
+	assert.Equal(t, "file-49", files[49].Name())
+}