Repository: replicatedhq/troubleshoot
Pull request #324 (2021-02-19 00:01:27)

Analyze random read IOPS for a directory
The random read IOPS benchmark attempts to replicate the results of this
fio command:

fio --ioengine=psync --direct=1 --bs=4k --size=1Gi --readwrite=randread --serialize_overlap=1

Across three tests the fio command reported 1877 IOPS and the preflight
1822 IOPS with the same block and file size.


Patch:
diff --git a/examples/preflight/host/filesystem-performance.yaml b/examples/preflight/host/filesystem-performance.yaml
index bf52e560..d8e7c89b 100644
--- a/examples/preflight/host/filesystem-performance.yaml
+++ b/examples/preflight/host/filesystem-performance.yaml
@@ -14,6 +14,9 @@ spec:
     - filesystemPerformance:
         collectorName: etcd-perf
         outcomes:
+          - fail:
+              when: "iops < 50"
+              message: Insufficient random read IOPS
           - pass:
               when: "p99 < 3ms"
               message: Write latency is great!
diff --git a/examples/preflight/host/sample.yaml b/examples/preflight/host/sample.yaml
index 98696f5c..102862f8 100644
--- a/examples/preflight/host/sample.yaml
+++ b/examples/preflight/host/sample.yaml
@@ -9,6 +9,12 @@ spec:
     - diskUsage:
         collectorName: ephemeral
         path: /var/lib/kubelet
+    - filesystemPerformance:
+        collectorName: etcd-perf
+        directory: /var/lib/etcd
+        fileSize: 22Mi
+        operationSizeBytes: 2300
+        datasync: true
     - httpLoadBalancer:
         collectorName: httploadbalancer
         port: 80
@@ -68,6 +74,24 @@ spec:
               message: /var/lib/kubelet is more than 70% full
           - pass:
               message: /var/lib/kubelet has sufficient disk space available
+    - filesystemPerformance:
+        collectorName: etcd-perf
+        outcomes:
+          - fail:
+              when: "iops < 50"
+              message: Insufficient random read IOPS
+          - pass:
+              when: "p99 < 3ms"
+              message: Write latency is great!
+          - pass:
+              when: "p99 < 5ms"
+              message: Write latency is ok
+          - warn:
+              when: "p99 < 8ms"
+              message: Write latency is high
+          - fail:
+              when: "p99 >= 8ms"
+              message: Write latency is too high
     - httpLoadBalancer:
         collectorName: httploadbalancer
         outcomes:
diff --git a/pkg/analyze/host_filesystem_performance.go b/pkg/analyze/host_filesystem_performance.go
index 4c6df612..95bc59bb 100644
--- a/pkg/analyze/host_filesystem_performance.go
+++ b/pkg/analyze/host_filesystem_performance.go
@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -114,6 +115,14 @@ func compareHostFilesystemPerformanceConditionalToActual(conditional string, fsP
 	keyword := strings.ToLower(parts[0])
 	comparator := parts[1]
 
+	if keyword == "iops" {
+		desiredInt, err := strconv.ParseInt(parts[2], 10, 64)
+		if err != nil {
+			return false, errors.Wrapf(err, "failed to parse desired iops %q as integer", parts[2])
+		}
+		return doCompareHostFilesystemIOPS(comparator, fsPerf.IOPS, int(desiredInt))
+	}
+
 	desiredDuration, err := time.ParseDuration(parts[2])
 	if err != nil {
 		return false, errors.Wrapf(err, "failed to parse duration %q", parts[2])
@@ -124,6 +133,8 @@ func compareHostFilesystemPerformanceConditionalToActual(conditional string, fsP
 		return doCompareHostFilesystemPerformance(comparator, fsPerf.Min, desiredDuration)
 	case "max":
 		return doCompareHostFilesystemPerformance(comparator, fsPerf.Max, desiredDuration)
+	case "average":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.Average, desiredDuration)
 	case "p1":
 		return doCompareHostFilesystemPerformance(comparator, fsPerf.P1, desiredDuration)
 	case "p5":
@@ -177,5 +188,22 @@ func doCompareHostFilesystemPerformance(operator string, actual time.Duration, d
 		return actual == desired, nil
 	}
 
-	return false, fmt.Errorf("Unknown filesystem performance oeprator %q", operator)
+	return false, fmt.Errorf("Unknown filesystem performance operator %q", operator)
+}
+
+func doCompareHostFilesystemIOPS(operator string, actual int, desired int) (bool, error) {
+	switch operator {
+	case "<":
+		return actual < desired, nil
+	case "<=":
+		return actual <= desired, nil
+	case ">":
+		return actual > desired, nil
+	case ">=":
+		return actual >= desired, nil
+	case "=", "==", "===":
+		return actual == desired, nil
+	}
+
+	return false, fmt.Errorf("Unknown filesystem IOPS operator %q", operator)
 }
diff --git a/pkg/collect/host_filesystem_performance.go b/pkg/collect/host_filesystem_performance.go
index be070ff3..0d3e379f 100644
--- a/pkg/collect/host_filesystem_performance.go
+++ b/pkg/collect/host_filesystem_performance.go
@@ -1,13 +1,14 @@
 package collect
 
 import (
-	"crypto/rand"
 	"encoding/json"
 	"log"
 	"math"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"sort"
+	"sync"
 	"syscall"
 	"time"
 
@@ -15,26 +16,32 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 )
 
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
 type FSPerfResults struct {
-	Min   time.Duration
-	Max   time.Duration
-	P1    time.Duration
-	P5    time.Duration
-	P10   time.Duration
-	P20   time.Duration
-	P30   time.Duration
-	P40   time.Duration
-	P50   time.Duration
-	P60   time.Duration
-	P70   time.Duration
-	P80   time.Duration
-	P90   time.Duration
-	P95   time.Duration
-	P99   time.Duration
-	P995  time.Duration
-	P999  time.Duration
-	P9995 time.Duration
-	P9999 time.Duration
+	Min     time.Duration
+	Max     time.Duration
+	Average time.Duration
+	P1      time.Duration
+	P5      time.Duration
+	P10     time.Duration
+	P20     time.Duration
+	P30     time.Duration
+	P40     time.Duration
+	P50     time.Duration
+	P60     time.Duration
+	P70     time.Duration
+	P80     time.Duration
+	P90     time.Duration
+	P95     time.Duration
+	P99     time.Duration
+	P995    time.Duration
+	P999    time.Duration
+	P9995   time.Duration
+	P9999   time.Duration
+	IOPS    int
 }
 
 type Durations []time.Duration
@@ -78,7 +85,7 @@ func HostFilesystemPerformance(c *HostCollector) (map[string][]byte, error) {
 	}
 	filename := filepath.Join(c.Collect.FilesystemPerformance.Directory, "fsperf")
 
-	f, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600)
+	f, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 	if err != nil {
 		return nil, errors.Wrapf(err, "open %s", filename)
 	}
@@ -91,6 +98,7 @@ func HostFilesystemPerformance(c *HostCollector) (map[string][]byte, error) {
 		}
 	}()
 
+	// Sequential writes benchmark
 	var written uint64 = 0
 	var results Durations
 
@@ -130,27 +138,99 @@ func HostFilesystemPerformance(c *HostCollector) (map[string][]byte, error) {
 
 	sort.Sort(results)
 
+	var sum time.Duration
+	for _, d := range results {
+		sum += d
+	}
+
 	fsPerf := &FSPerfResults{
-		Min:   results[0],
-		Max:   results[len(results)-1],
-		P1:    results[getPercentileIndex(.01, len(results))],
-		P5:    results[getPercentileIndex(.05, len(results))],
-		P10:   results[getPercentileIndex(.1, len(results))],
-		P20:   results[getPercentileIndex(.2, len(results))],
-		P30:   results[getPercentileIndex(.3, len(results))],
-		P40:   results[getPercentileIndex(.4, len(results))],
-		P50:   results[getPercentileIndex(.5, len(results))],
-		P60:   results[getPercentileIndex(.6, len(results))],
-		P70:   results[getPercentileIndex(.7, len(results))],
-		P80:   results[getPercentileIndex(.8, len(results))],
-		P90:   results[getPercentileIndex(.9, len(results))],
-		P95:   results[getPercentileIndex(.95, len(results))],
-		P99:   results[getPercentileIndex(.99, len(results))],
-		P995:  results[getPercentileIndex(.995, len(results))],
-		P999:  results[getPercentileIndex(.999, len(results))],
-		P9995: results[getPercentileIndex(.9995, len(results))],
-		P9999: results[getPercentileIndex(.9999, len(results))],
+		Min:     results[0],
+		Max:     results[len(results)-1],
+		Average: sum / time.Duration(len(results)),
+		P1:      results[getPercentileIndex(.01, len(results))],
+		P5:      results[getPercentileIndex(.05, len(results))],
+		P10:     results[getPercentileIndex(.1, len(results))],
+		P20:     results[getPercentileIndex(.2, len(results))],
+		P30:     results[getPercentileIndex(.3, len(results))],
+		P40:     results[getPercentileIndex(.4, len(results))],
+		P50:     results[getPercentileIndex(.5, len(results))],
+		P60:     results[getPercentileIndex(.6, len(results))],
+		P70:     results[getPercentileIndex(.7, len(results))],
+		P80:     results[getPercentileIndex(.8, len(results))],
+		P90:     results[getPercentileIndex(.9, len(results))],
+		P95:     results[getPercentileIndex(.95, len(results))],
+		P99:     results[getPercentileIndex(.99, len(results))],
+		P995:    results[getPercentileIndex(.995, len(results))],
+		P999:    results[getPercentileIndex(.999, len(results))],
+		P9995:   results[getPercentileIndex(.9995, len(results))],
+		P9999:   results[getPercentileIndex(.9999, len(results))],
+	}
+
+	// Random IOPS benchmark
+
+	// Re-open the file read+write in direct mode to prevent caching
+	if err := f.Close(); err != nil {
+		return nil, errors.Wrapf(err, "close %s", filename)
 	}
+	f, err = os.OpenFile(filename, os.O_RDWR|syscall.O_DIRECT, 0600)
+	if err != nil {
+		return nil, errors.Wrapf(err, "open direct %s", filename)
+	}
+
+	offsets := make([]int64, len(results))
+
+	for index, p := range rand.Perm(len(results)) {
+		offsets[index] = int64(p) * int64(operationSize)
+	}
+
+	// Use multiple workers to keep the filesystem busy. Since operations are serialized on a single
+	// file, more than 2 does not improve IOPS.
+	workers := 2
+	wg := sync.WaitGroup{}
+	m := sync.Mutex{}
+
+	errs := make(chan error, workers)
+
+	start := time.Now()
+
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+
+		go func(i int) {
+			defer wg.Done()
+
+			data := make([]byte, int(operationSize))
+			fd := int(f.Fd())
+
+			for idx, offset := range offsets {
+				if idx%workers != i {
+					continue
+				}
+
+				m.Lock()
+				n, err := syscall.Pread(fd, data, offset)
+				m.Unlock()
+
+				if err != nil {
+					errs <- errors.Wrapf(err, "failed to pread %d bytes to %s at offset %d", len(data), filename, offset)
+				}
+				if n != len(data) {
+					errs <- errors.Wrapf(err, "pread %d of %d bytes to %s at offset %d", n, len(data), filename, offset)
+				}
+			}
+		}(i)
+	}
+
+	wg.Wait()
+	if len(errs) > 0 {
+		return nil, <-errs
+	}
+
+	d := time.Now().Sub(start)
+	nsPerIO := d / time.Duration(len(offsets))
+	iops := time.Second / nsPerIO
+
+	fsPerf.IOPS = int(iops)
 
 	collectorName := c.Collect.FilesystemPerformance.CollectorName
 	if collectorName == "" {
@@ -159,7 +239,7 @@ func HostFilesystemPerformance(c *HostCollector) (map[string][]byte, error) {
 	name := filepath.Join("filesystemPerformance", collectorName+".json")
 	b, err := json.Marshal(fsPerf)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to marsh fs perf results")
+		return nil, errors.Wrap(err, "failed to marshal fs perf results")
 	}
 
 	return map[string][]byte{


Test patch:
diff --git a/pkg/analyze/host_filesystem_performance_test.go b/pkg/analyze/host_filesystem_performance_test.go
index d2f492a1..60582503 100644
--- a/pkg/analyze/host_filesystem_performance_test.go
+++ b/pkg/analyze/host_filesystem_performance_test.go
@@ -19,28 +19,74 @@ func TestAnalyzeHostFilesystemPerformance(t *testing.T) {
 		result       *AnalyzeResult
 		expectErr    bool
 	}{
+		{
+			name: "IOPS",
+			fsPerf: &collect.FSPerfResults{
+				IOPS: 50,
+			},
+			hostAnalyzer: &troubleshootv1beta2.FilesystemPerformanceAnalyze{
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "iops < 20",
+							Message: "IOPS < 20",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "iops <= 20",
+							Message: "IOPS <= 30",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "iops > 70",
+							Message: "IOPS > 70",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "iops >= 100",
+							Message: "IOPS >= 100",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "iops == 50",
+							Message: "IOPS == 50",
+						},
+					},
+				},
+			},
+			result: &AnalyzeResult{
+				Title:   "Filesystem Performance",
+				IsPass:  true,
+				Message: "IOPS == 50",
+			},
+		},
 		{
 			name: "Cover",
 			fsPerf: &collect.FSPerfResults{
-				Min:   200 * time.Nanosecond,
-				Max:   time.Second,
-				P1:    1 * time.Microsecond,
-				P5:    5 * time.Microsecond,
-				P10:   10 * time.Microsecond,
-				P20:   20 * time.Microsecond,
-				P30:   30 * time.Microsecond,
-				P40:   40 * time.Microsecond,
-				P50:   50 * time.Microsecond,
-				P60:   60 * time.Microsecond,
-				P70:   70 * time.Microsecond,
-				P80:   80 * time.Microsecond,
-				P90:   90 * time.Microsecond,
-				P95:   95 * time.Microsecond,
-				P99:   99 * time.Microsecond,
-				P995:  995 * time.Microsecond,
-				P999:  999 * time.Microsecond,
-				P9995: 5 * time.Millisecond,
-				P9999: 9 * time.Millisecond,
+				Min:     200 * time.Nanosecond,
+				Max:     time.Second,
+				Average: 55 * time.Microsecond,
+				P1:      1 * time.Microsecond,
+				P5:      5 * time.Microsecond,
+				P10:     10 * time.Microsecond,
+				P20:     20 * time.Microsecond,
+				P30:     30 * time.Microsecond,
+				P40:     40 * time.Microsecond,
+				P50:     50 * time.Microsecond,
+				P60:     60 * time.Microsecond,
+				P70:     70 * time.Microsecond,
+				P80:     80 * time.Microsecond,
+				P90:     90 * time.Microsecond,
+				P95:     95 * time.Microsecond,
+				P99:     99 * time.Microsecond,
+				P995:    995 * time.Microsecond,
+				P999:    999 * time.Microsecond,
+				P9995:   5 * time.Millisecond,
+				P9999:   9 * time.Millisecond,
 			},
 			hostAnalyzer: &troubleshootv1beta2.FilesystemPerformanceAnalyze{
 				CollectorName: "etcd",
@@ -69,6 +115,12 @@ func TestAnalyzeHostFilesystemPerformance(t *testing.T) {
 							Message: "gte operator not working",
 						},
 					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "average == 0",
+							Message: "average not working",
+						},
+					},
 					{
 						Fail: &troubleshootv1beta2.SingleOutcome{
 							When:    "p1 < 1us",