Repository: threefoldtech/zos (go)

Issue:
flistd: fail to umount flist
```
[+] provisiond: 2020-09-22T05:36:23Z info check for expired reservation
[+] provisiond: 2020-09-22T05:36:23Z info reservation expired created=2020-08-11T15:02:30Z duration=1h4m25s id=10908-1 type=kubernetes
[+] provisiond: 2020-09-22T05:36:23Z info start decommissioning reservation duration=1h4m25s expired=true id=10908-1 tag= to-delete=false type=kubernetes
[+] networkd: 2020-09-22T05:36:23Z info Removing tap interface network-id=Qh89hEofqsf3J
[+] flistd: 2020-09-22T05:36:23Z info request unmount flist path=/var/cache/modules/flistd/mountpoint/10908-1
[+] flistd: 2020-09-22T05:36:23Z error fail to umount flist error="invalid argument" path=/var/cache/modules/flistd/mountpoint/10908-1
[+] flistd: 2020-09-22T05:36:25Z error 0-fs daemon did not stop properly error="context deadline exceeded" path=/var/cache/modules/flistd/mountpoint/10908-1
[+] provisiond: 2020-09-22T05:36:25Z error failed to decommission reservation 10908-1 error="decommissioning of reservation failed: could not unmount flist: context deadline exceeded"
```

Seems it can happen the mountpoint is dead but the 0-fs process is still running.

Pull request description:
flistd: force stop the 0-fs process even if mountpoint is already dead
fixes #956

Patch:
diff --git a/pkg/flist/flist.go b/pkg/flist/flist.go
index 7ac346ee..6c0f9e78 100644
--- a/pkg/flist/flist.go
+++ b/pkg/flist/flist.go
@@ -378,10 +378,21 @@ func (f *flistModule) Umount(path string) error {
 
 	if err := syscall.Unmount(path, syscall.MNT_DETACH); err != nil {
 		log.Error().Err(err).Str("path", path).Msg("fail to umount flist")
+
 	}
 
 	if err := waitPidFile(time.Second*2, pidPath, false); err != nil {
 		log.Error().Err(err).Str("path", path).Msg("0-fs daemon did not stop properly")
+
+		pid, err := f.getPid(pidPath)
+		if err != nil {
+			return err
+		}
+
+		if err := forceStop(int(pid)); err != nil {
+			log.Error().Int64("pid", pid).Err(err).Msg("failed to kill 0-fs process")
+		}
+
 		return err
 	}
 
@@ -576,4 +587,33 @@ func waitMountedLog(timeout time.Duration, logfile string) error {
 	return <-cErr
 }
 
+func forceStop(pid int) error {
+	slog := log.With().Int("pid", pid).Logger()
+	slog.Info().Msg("trying to force stop by killing the process")
+
+	p, err := os.FindProcess(int(pid))
+	if err != nil {
+		return err
+	}
+
+	if err := p.Signal(syscall.SIGTERM); err != nil {
+		slog.Error().Err(err).Msg("failed to send SIGTERM to process")
+	}
+
+	time.Sleep(time.Second)
+	if pidExist(p) {
+		slog.Info().Msgf("process didn't stop gracefully, lets kill it")
+		if err := p.Signal(syscall.SIGKILL); err != nil {
+			slog.Error().Err(err).Msg("failed to send SIGKILL to process")
+		}
+	}
+
+	return nil
+}
+
+func pidExist(p *os.Process) bool {
+	// https://github.com/golang/go/issues/14146#issuecomment-176888204
+	return p.Signal(syscall.Signal(0)) == nil
+}
+
 var _ pkg.Flister = (*flistModule)(nil)


Test patch:
diff --git a/pkg/flist/flist_test.go b/pkg/flist/flist_test.go
index db0abcc3..fab7b3a0 100644
--- a/pkg/flist/flist_test.go
+++ b/pkg/flist/flist_test.go
@@ -256,3 +256,23 @@ func TestWaitPIDFileTimeout(t *testing.T) {
 	err := <-out
 	require.Equal(context.DeadlineExceeded, err)
 }
+
+func Test_forceStop(t *testing.T) {
+	ctx, _ := context.WithTimeout(context.Background(), time.Second*10)
+
+	cmd := exec.CommandContext(ctx, "sleep", "50")
+	err := cmd.Start()
+	require.NoError(t, err)
+
+	go func() {
+		cmd.Wait()
+	}()
+
+	go func() {
+		<-ctx.Done()
+		t.Error("didn't stop the process in time")
+	}()
+
+	err = forceStop(cmd.Process.Pid)
+	assert.NoError(t, err)
+}