Repository: replicatedhq/troubleshoot
Pull request #323 (2021-02-17 23:23:03)

Analyze fs sequential write performance
The included example found P99 latency of 2.6ms, compared to 2.5ms reported by `fio --rw=write --ioengine=sync --fdatasync=1 --directory=/var/lib/etcd --size=220m --bs=2300`.


Patch:
diff --git a/examples/preflight/host/filesystem-performance.yaml b/examples/preflight/host/filesystem-performance.yaml
new file mode 100644
index 00000000..bf52e560
--- /dev/null
+++ b/examples/preflight/host/filesystem-performance.yaml
@@ -0,0 +1,28 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: HostPreflight
+metadata:
+  name: fsperf
+spec:
+  collectors:
+    - filesystemPerformance:
+        collectorName: etcd-perf
+        directory: /var/lib/etcd
+        fileSize: 22Mi
+        operationSizeBytes: 2300
+        datasync: true
+  analyzers:
+    - filesystemPerformance:
+        collectorName: etcd-perf
+        outcomes:
+          - pass:
+              when: "p99 < 3ms"
+              message: Write latency is great!
+          - pass:
+              when: "p99 < 5ms"
+              message: Write latency is ok
+          - warn:
+              when: "p99 < 8ms"
+              message: Write latency is high
+          - fail:
+              when: "p99 >= 8ms"
+              message: Write latency is too high
diff --git a/pkg/analyze/analyzer.go b/pkg/analyze/analyzer.go
index a7c68471..dc91fdaa 100644
--- a/pkg/analyze/analyzer.go
+++ b/pkg/analyze/analyzer.go
@@ -118,6 +118,13 @@ func HostAnalyze(hostAnalyzer *troubleshootv1beta2.HostAnalyze, getFile getColle
 		}
 		return []*AnalyzeResult{result}, nil
 	}
+	if hostAnalyzer.FilesystemPerformance != nil {
+		result, err := analyzeHostFilesystemPerformance(hostAnalyzer.FilesystemPerformance, getFile)
+		if err != nil {
+			return nil, err
+		}
+		return []*AnalyzeResult{result}, nil
+	}
 
 	return nil, errors.New("invalid analyzer")
 }
diff --git a/pkg/analyze/host_filesystem_performance.go b/pkg/analyze/host_filesystem_performance.go
new file mode 100644
index 00000000..4c6df612
--- /dev/null
+++ b/pkg/analyze/host_filesystem_performance.go
@@ -0,0 +1,181 @@
+package analyzer
+
+import (
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/collect"
+)
+
+func analyzeHostFilesystemPerformance(hostAnalyzer *troubleshootv1beta2.FilesystemPerformanceAnalyze, getCollectedFileContents func(string) ([]byte, error)) (*AnalyzeResult, error) {
+	collectorName := hostAnalyzer.CollectorName
+	if collectorName == "" {
+		collectorName = "filesystemPerformance"
+	}
+	name := filepath.Join("filesystemPerformance", collectorName+".json")
+	contents, err := getCollectedFileContents(name)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get collected file %s", name)
+	}
+
+	fsPerf := collect.FSPerfResults{}
+	if err := json.Unmarshal(contents, &fsPerf); err != nil {
+		return nil, errors.Wrapf(err, "failed to unmarshal filesystem performance results from %s", name)
+	}
+
+	result := AnalyzeResult{}
+
+	title := hostAnalyzer.CheckName
+	if title == "" {
+		title = "Filesystem Performance"
+	}
+	result.Title = title
+
+	for _, outcome := range hostAnalyzer.Outcomes {
+		if outcome.Fail != nil {
+			if outcome.Fail.When == "" {
+				result.IsFail = true
+				result.Message = outcome.Fail.Message
+				result.URI = outcome.Fail.URI
+
+				return &result, nil
+			}
+
+			isMatch, err := compareHostFilesystemPerformanceConditionalToActual(outcome.Fail.When, fsPerf)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to compare %q", outcome.Fail.When)
+			}
+
+			if isMatch {
+				result.IsFail = true
+				result.Message = outcome.Fail.Message
+				result.URI = outcome.Fail.URI
+
+				return &result, nil
+			}
+		} else if outcome.Warn != nil {
+			if outcome.Warn.When == "" {
+				result.IsWarn = true
+				result.Message = outcome.Warn.Message
+				result.URI = outcome.Warn.URI
+
+				return &result, nil
+			}
+
+			isMatch, err := compareHostFilesystemPerformanceConditionalToActual(outcome.Warn.When, fsPerf)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to compare %q", outcome.Warn.When)
+			}
+
+			if isMatch {
+				result.IsWarn = true
+				result.Message = outcome.Warn.Message
+				result.URI = outcome.Warn.URI
+
+				return &result, nil
+			}
+		} else if outcome.Pass != nil {
+			if outcome.Pass.When == "" {
+				result.IsPass = true
+				result.Message = outcome.Pass.Message
+				result.URI = outcome.Pass.URI
+
+				return &result, nil
+			}
+
+			isMatch, err := compareHostFilesystemPerformanceConditionalToActual(outcome.Pass.When, fsPerf)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to compare %q", outcome.Pass.When)
+			}
+
+			if isMatch {
+				result.IsPass = true
+				result.Message = outcome.Pass.Message
+				result.URI = outcome.Pass.URI
+
+				return &result, nil
+			}
+		}
+	}
+
+	return &result, nil
+}
+
+func compareHostFilesystemPerformanceConditionalToActual(conditional string, fsPerf collect.FSPerfResults) (res bool, err error) {
+	parts := strings.Split(conditional, " ")
+	if len(parts) != 3 {
+		return false, fmt.Errorf("conditional must have exactly 3 parts, got %d", len(parts))
+	}
+	keyword := strings.ToLower(parts[0])
+	comparator := parts[1]
+
+	desiredDuration, err := time.ParseDuration(parts[2])
+	if err != nil {
+		return false, errors.Wrapf(err, "failed to parse duration %q", parts[2])
+	}
+
+	switch keyword {
+	case "min":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.Min, desiredDuration)
+	case "max":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.Max, desiredDuration)
+	case "p1":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P1, desiredDuration)
+	case "p5":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P5, desiredDuration)
+	case "p10":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P10, desiredDuration)
+	case "p20":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P20, desiredDuration)
+	case "p30":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P30, desiredDuration)
+	case "p40":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P40, desiredDuration)
+	case "p50":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P50, desiredDuration)
+	case "p60":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P60, desiredDuration)
+	case "p70":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P70, desiredDuration)
+	case "p80":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P80, desiredDuration)
+	case "p90":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P90, desiredDuration)
+	case "p95":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P95, desiredDuration)
+	case "p99":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P99, desiredDuration)
+	case "p995":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P995, desiredDuration)
+	case "p999":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P999, desiredDuration)
+	case "p9995":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P9995, desiredDuration)
+	case "p9999":
+		return doCompareHostFilesystemPerformance(comparator, fsPerf.P9999, desiredDuration)
+	}
+
+	return false, fmt.Errorf("Unknown filesystem performance keyword %q", keyword)
+}
+
+func doCompareHostFilesystemPerformance(operator string, actual time.Duration, desired time.Duration) (bool, error) {
+	switch operator {
+	case "<":
+		return actual < desired, nil
+	case "<=":
+		return actual <= desired, nil
+	case ">":
+		return actual > desired, nil
+	case ">=":
+		return actual >= desired, nil
+	case "=", "==", "===":
+		return actual == desired, nil
+	}
+
+	return false, fmt.Errorf("Unknown filesystem performance oeprator %q", operator)
+}
diff --git a/pkg/apis/troubleshoot/v1beta2/hostanalyzer_shared.go b/pkg/apis/troubleshoot/v1beta2/hostanalyzer_shared.go
index 57d1fd9a..aebed833 100644
--- a/pkg/apis/troubleshoot/v1beta2/hostanalyzer_shared.go
+++ b/pkg/apis/troubleshoot/v1beta2/hostanalyzer_shared.go
@@ -61,6 +61,12 @@ type IPV4InterfacesAnalyze struct {
 	Outcomes    []*Outcome `json:"outcomes" yaml:"outcomes"`
 }
 
+type FilesystemPerformanceAnalyze struct {
+	AnalyzeMeta   `json:",inline" yaml:",inline"`
+	CollectorName string     `json:"collectorName,omitempty" yaml:"collectorName,omitempty"`
+	Outcomes      []*Outcome `json:"outcomes" yaml:"outcomes"`
+}
+
 type HostAnalyze struct {
 	CPU *CPUAnalyze `json:"cpu,omitempty" yaml:"cpu,omitempty"`
 	//
@@ -82,4 +88,6 @@ type HostAnalyze struct {
 	TCPConnect *TCPConnectAnalyze `json:"tcpConnect" yaml:"tcpConnect"`
 
 	IPV4Interfaces *IPV4InterfacesAnalyze `json:"ipv4Interfaces" yaml:"ipv4Interfaces"`
+
+	FilesystemPerformance *FilesystemPerformanceAnalyze `json:"filesystemPerformance" yaml:"filesystemPerformance"`
 }
diff --git a/pkg/apis/troubleshoot/v1beta2/hostcollector_shared.go b/pkg/apis/troubleshoot/v1beta2/hostcollector_shared.go
index 81d9295b..013b6c3f 100644
--- a/pkg/apis/troubleshoot/v1beta2/hostcollector_shared.go
+++ b/pkg/apis/troubleshoot/v1beta2/hostcollector_shared.go
@@ -73,19 +73,29 @@ type TCPConnect struct {
 	Timeout           string `json:"timeout,omitempty"`
 }
 
+type FilesystemPerformance struct {
+	HostCollectorMeta  `json:",inline" yaml:",inline"`
+	OperationSizeBytes uint64 `json:"operationSize,omitempty"`
+	Directory          string `json:"directory,omitempty"`
+	FileSize           string `json:"fileSize,omitempty"`
+	Sync               bool   `json:"sync,omitempty"`
+	Datasync           bool   `json:"datasync,omitempty"`
+}
+
 type HostCollect struct {
-	CPU              *CPU              `json:"cpu,omitempty" yaml:"cpu,omitempty"`
-	Memory           *Memory           `json:"memory,omitempty" yaml:"memory,omitempty"`
-	TCPLoadBalancer  *TCPLoadBalancer  `json:"tcpLoadBalancer,omitempty" yaml:"tcpLoadBalancer,omitempty"`
-	HTTPLoadBalancer *HTTPLoadBalancer `json:"httpLoadBalancer,omitempty" yaml:"httpLoadBalancer,omitempty"`
-	TCPPortStatus    *TCPPortStatus    `json:"tcpPortStatus,omitempty" yaml:"tcpPortStatus,omitempty"`
-	Kubernetes       *Kubernetes       `json:"kubernetes,omitempty" yaml:"kubernetes,omitempty"`
-	IPV4Interfaces   *IPV4Interfaces   `json:"ipv4Interfaces,omitempty" yaml:"ipv4Interfaces,omitempty"`
-	DiskUsage        *DiskUsage        `json:"diskUsage,omitempty" yaml:"diskUsage,omitempty"`
-	HTTP             *HostHTTP         `json:"http,omitempty" yaml:"http,omitempty"`
-	Time             *HostTime         `json:"time,omitempty" yaml:"time,omitempty"`
-	BlockDevices     *HostBlockDevices `json:"blockDevices,omitempty" yaml:"blockDevices,omitempty"`
-	TCPConnect       *TCPConnect       `json:"tcpConnect,omitempty" yaml:"tcpConnect,omitempty"`
+	CPU                   *CPU                   `json:"cpu,omitempty" yaml:"cpu,omitempty"`
+	Memory                *Memory                `json:"memory,omitempty" yaml:"memory,omitempty"`
+	TCPLoadBalancer       *TCPLoadBalancer       `json:"tcpLoadBalancer,omitempty" yaml:"tcpLoadBalancer,omitempty"`
+	HTTPLoadBalancer      *HTTPLoadBalancer      `json:"httpLoadBalancer,omitempty" yaml:"httpLoadBalancer,omitempty"`
+	TCPPortStatus         *TCPPortStatus         `json:"tcpPortStatus,omitempty" yaml:"tcpPortStatus,omitempty"`
+	Kubernetes            *Kubernetes            `json:"kubernetes,omitempty" yaml:"kubernetes,omitempty"`
+	IPV4Interfaces        *IPV4Interfaces        `json:"ipv4Interfaces,omitempty" yaml:"ipv4Interfaces,omitempty"`
+	DiskUsage             *DiskUsage             `json:"diskUsage,omitempty" yaml:"diskUsage,omitempty"`
+	HTTP                  *HostHTTP              `json:"http,omitempty" yaml:"http,omitempty"`
+	Time                  *HostTime              `json:"time,omitempty" yaml:"time,omitempty"`
+	BlockDevices          *HostBlockDevices      `json:"blockDevices,omitempty" yaml:"blockDevices,omitempty"`
+	TCPConnect            *TCPConnect            `json:"tcpConnect,omitempty" yaml:"tcpConnect,omitempty"`
+	FilesystemPerformance *FilesystemPerformance `json:"filesystemPerformance" yaml:"filesystemPerformance"`
 }
 
 func (c *HostCollect) GetName() string {
diff --git a/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go b/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go
index ae164531..44250298 100644
--- a/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go
+++ b/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go
@@ -1001,6 +1001,49 @@ func (in *FileSelector) DeepCopy() *FileSelector {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FilesystemPerformance) DeepCopyInto(out *FilesystemPerformance) {
+	*out = *in
+	out.HostCollectorMeta = in.HostCollectorMeta
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FilesystemPerformance.
+func (in *FilesystemPerformance) DeepCopy() *FilesystemPerformance {
+	if in == nil {
+		return nil
+	}
+	out := new(FilesystemPerformance)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FilesystemPerformanceAnalyze) DeepCopyInto(out *FilesystemPerformanceAnalyze) {
+	*out = *in
+	out.AnalyzeMeta = in.AnalyzeMeta
+	if in.Outcomes != nil {
+		in, out := &in.Outcomes, &out.Outcomes
+		*out = make([]*Outcome, len(*in))
+		for i := range *in {
+			if (*in)[i] != nil {
+				in, out := &(*in)[i], &(*out)[i]
+				*out = new(Outcome)
+				(*in).DeepCopyInto(*out)
+			}
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FilesystemPerformanceAnalyze.
+func (in *FilesystemPerformanceAnalyze) DeepCopy() *FilesystemPerformanceAnalyze {
+	if in == nil {
+		return nil
+	}
+	out := new(FilesystemPerformanceAnalyze)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Get) DeepCopyInto(out *Get) {
 	*out = *in
@@ -1182,6 +1225,11 @@ func (in *HostAnalyze) DeepCopyInto(out *HostAnalyze) {
 		*out = new(IPV4InterfacesAnalyze)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.FilesystemPerformance != nil {
+		in, out := &in.FilesystemPerformance, &out.FilesystemPerformance
+		*out = new(FilesystemPerformanceAnalyze)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostAnalyze.
@@ -1273,6 +1321,11 @@ func (in *HostCollect) DeepCopyInto(out *HostCollect) {
 		*out = new(TCPConnect)
 		**out = **in
 	}
+	if in.FilesystemPerformance != nil {
+		in, out := &in.FilesystemPerformance, &out.FilesystemPerformance
+		*out = new(FilesystemPerformance)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostCollect.
diff --git a/pkg/collect/host_collector.go b/pkg/collect/host_collector.go
index 2b8bc1ea..6fddc9b5 100644
--- a/pkg/collect/host_collector.go
+++ b/pkg/collect/host_collector.go
@@ -40,6 +40,8 @@ func (c *HostCollector) RunCollectorSync() (result map[string][]byte, err error)
 		result, err = HostTCPConnect(c)
 	} else if c.Collect.IPV4Interfaces != nil {
 		result, err = HostIPV4Interfaces(c)
+	} else if c.Collect.FilesystemPerformance != nil {
+		result, err = HostFilesystemPerformance(c)
 	} else {
 		err = errors.New("no spec found to run")
 		return
diff --git a/pkg/collect/host_filesystem_performance.go b/pkg/collect/host_filesystem_performance.go
new file mode 100644
index 00000000..be070ff3
--- /dev/null
+++ b/pkg/collect/host_filesystem_performance.go
@@ -0,0 +1,175 @@
+package collect
+
+import (
+	"crypto/rand"
+	"encoding/json"
+	"log"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"syscall"
+	"time"
+
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+type FSPerfResults struct {
+	Min   time.Duration
+	Max   time.Duration
+	P1    time.Duration
+	P5    time.Duration
+	P10   time.Duration
+	P20   time.Duration
+	P30   time.Duration
+	P40   time.Duration
+	P50   time.Duration
+	P60   time.Duration
+	P70   time.Duration
+	P80   time.Duration
+	P90   time.Duration
+	P95   time.Duration
+	P99   time.Duration
+	P995  time.Duration
+	P999  time.Duration
+	P9995 time.Duration
+	P9999 time.Duration
+}
+
+type Durations []time.Duration
+
+func (d Durations) Len() int {
+	return len(d)
+}
+
+func (d Durations) Less(i, j int) bool {
+	return d[i] < d[j]
+}
+
+func (d Durations) Swap(i, j int) {
+	d[i], d[j] = d[j], d[i]
+}
+
+func HostFilesystemPerformance(c *HostCollector) (map[string][]byte, error) {
+	var operationSize uint64 = 1024
+	if c.Collect.FilesystemPerformance.OperationSizeBytes != 0 {
+		operationSize = c.Collect.FilesystemPerformance.OperationSizeBytes
+	}
+
+	var fileSize uint64 = 10 * 1024 * 1024
+	if c.Collect.FilesystemPerformance.FileSize != "" {
+		quantity, err := resource.ParseQuantity(c.Collect.FilesystemPerformance.FileSize)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to parse fileSize %q", c.Collect.FilesystemPerformance.FileSize)
+		}
+		fileSizeInt64, ok := quantity.AsInt64()
+		if !ok {
+			return nil, errors.Wrapf(err, "failed to parse fileSize %q", c.Collect.FilesystemPerformance.FileSize)
+		}
+		fileSize = uint64(fileSizeInt64)
+	}
+
+	if c.Collect.FilesystemPerformance.Directory == "" {
+		return nil, errors.New("Directory is required to collect filesystem performance info")
+	}
+	if err := os.MkdirAll(c.Collect.FilesystemPerformance.Directory, 0700); err != nil {
+		return nil, errors.Wrapf(err, "failed to mkdir %q", c.Collect.FilesystemPerformance.Directory)
+	}
+	filename := filepath.Join(c.Collect.FilesystemPerformance.Directory, "fsperf")
+
+	f, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600)
+	if err != nil {
+		return nil, errors.Wrapf(err, "open %s", filename)
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			log.Println(err.Error())
+		}
+		if err := os.Remove(filename); err != nil {
+			log.Println(err.Error())
+		}
+	}()
+
+	var written uint64 = 0
+	var results Durations
+
+	for {
+		if written >= fileSize {
+			break
+		}
+
+		data := make([]byte, int(operationSize))
+		rand.Read(data)
+
+		start := time.Now()
+
+		n, err := f.Write(data)
+		if err != nil {
+			return nil, errors.Wrapf(err, "write to %s", filename)
+		}
+		if c.Collect.FilesystemPerformance.Sync {
+			if err := f.Sync(); err != nil {
+				return nil, errors.Wrapf(err, "sync %s", filename)
+			}
+		} else if c.Collect.FilesystemPerformance.Datasync {
+			if err := syscall.Fdatasync(int(f.Fd())); err != nil {
+				return nil, errors.Wrapf(err, "datasync %s", filename)
+			}
+		}
+
+		d := time.Now().Sub(start)
+		results = append(results, d)
+
+		written += uint64(n)
+	}
+
+	if len(results) == 0 {
+		return nil, errors.New("No filesystem performance results collected")
+	}
+
+	sort.Sort(results)
+
+	fsPerf := &FSPerfResults{
+		Min:   results[0],
+		Max:   results[len(results)-1],
+		P1:    results[getPercentileIndex(.01, len(results))],
+		P5:    results[getPercentileIndex(.05, len(results))],
+		P10:   results[getPercentileIndex(.1, len(results))],
+		P20:   results[getPercentileIndex(.2, len(results))],
+		P30:   results[getPercentileIndex(.3, len(results))],
+		P40:   results[getPercentileIndex(.4, len(results))],
+		P50:   results[getPercentileIndex(.5, len(results))],
+		P60:   results[getPercentileIndex(.6, len(results))],
+		P70:   results[getPercentileIndex(.7, len(results))],
+		P80:   results[getPercentileIndex(.8, len(results))],
+		P90:   results[getPercentileIndex(.9, len(results))],
+		P95:   results[getPercentileIndex(.95, len(results))],
+		P99:   results[getPercentileIndex(.99, len(results))],
+		P995:  results[getPercentileIndex(.995, len(results))],
+		P999:  results[getPercentileIndex(.999, len(results))],
+		P9995: results[getPercentileIndex(.9995, len(results))],
+		P9999: results[getPercentileIndex(.9999, len(results))],
+	}
+
+	collectorName := c.Collect.FilesystemPerformance.CollectorName
+	if collectorName == "" {
+		collectorName = "filesystemPerformance"
+	}
+	name := filepath.Join("filesystemPerformance", collectorName+".json")
+	b, err := json.Marshal(fsPerf)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to marsh fs perf results")
+	}
+
+	return map[string][]byte{
+		name: b,
+	}, nil
+}
+
+func getPercentileIndex(p float64, items int) int {
+	if p >= 1 {
+		return items - 1
+	}
+	return int(math.Ceil(p*float64(items))) - 1
+}


Test patch:
diff --git a/pkg/analyze/host_filesystem_performance_test.go b/pkg/analyze/host_filesystem_performance_test.go
new file mode 100644
index 00000000..d2f492a1
--- /dev/null
+++ b/pkg/analyze/host_filesystem_performance_test.go
@@ -0,0 +1,313 @@
+package analyzer
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/collect"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAnalyzeHostFilesystemPerformance(t *testing.T) {
+	tests := []struct {
+		name         string
+		fsPerf       *collect.FSPerfResults
+		hostAnalyzer *troubleshootv1beta2.FilesystemPerformanceAnalyze
+		result       *AnalyzeResult
+		expectErr    bool
+	}{
+		{
+			name: "Cover",
+			fsPerf: &collect.FSPerfResults{
+				Min:   200 * time.Nanosecond,
+				Max:   time.Second,
+				P1:    1 * time.Microsecond,
+				P5:    5 * time.Microsecond,
+				P10:   10 * time.Microsecond,
+				P20:   20 * time.Microsecond,
+				P30:   30 * time.Microsecond,
+				P40:   40 * time.Microsecond,
+				P50:   50 * time.Microsecond,
+				P60:   60 * time.Microsecond,
+				P70:   70 * time.Microsecond,
+				P80:   80 * time.Microsecond,
+				P90:   90 * time.Microsecond,
+				P95:   95 * time.Microsecond,
+				P99:   99 * time.Microsecond,
+				P995:  995 * time.Microsecond,
+				P999:  999 * time.Microsecond,
+				P9995: 5 * time.Millisecond,
+				P9999: 9 * time.Millisecond,
+			},
+			hostAnalyzer: &troubleshootv1beta2.FilesystemPerformanceAnalyze{
+				CollectorName: "etcd",
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "min == 0",
+							Message: "min not working",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "min <= 50ns",
+							Message: "lte operator not working",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "max == 0",
+							Message: "max not working",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "max >= 1m",
+							Message: "gte operator not working",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p1 < 1us",
+							Message: "P1 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p1 > 1us",
+							Message: "P1 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p5 < 5us",
+							Message: "P5 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p5 > 5us",
+							Message: "P5 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p10 < 10us",
+							Message: "P10 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p10 > 10us",
+							Message: "P10 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p20 < 20us",
+							Message: "P20 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p20 > 20us",
+							Message: "P20 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p30 < 30us",
+							Message: "P30 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p30 > 30us",
+							Message: "P30 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p40 < 40us",
+							Message: "P40 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p40 > 40us",
+							Message: "P40 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p50 < 50us",
+							Message: "P50 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p50 > 50us",
+							Message: "P50 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p60 < 60us",
+							Message: "P60 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p60 > 60us",
+							Message: "P60 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p70 < 70us",
+							Message: "P70 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p70 > 70us",
+							Message: "P70 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p80 < 80us",
+							Message: "P80 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p80 > 80us",
+							Message: "P80 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p90 < 90us",
+							Message: "P90 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p90 > 90us",
+							Message: "P90 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p95 < 95us",
+							Message: "P95 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p95 > 95us",
+							Message: "P95 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p99 < 99us",
+							Message: "P99 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p99 > 99us",
+							Message: "P99 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p995 < 995us",
+							Message: "P995 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p995 > 995us",
+							Message: "P995 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p999 < 999us",
+							Message: "P999 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p999 > 999us",
+							Message: "P999 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p9995 < 5ms",
+							Message: "P9995 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p9995 > 5ms",
+							Message: "P9995 too high",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p9999 < 9ms",
+							Message: "P9999 too low",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "p9999 > 9ms",
+							Message: "P9999 too high",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "p9999 < 10ms",
+							Message: "Acceptable write latency",
+						},
+					},
+				},
+			},
+			result: &AnalyzeResult{
+				Title:   "Filesystem Performance",
+				IsPass:  true,
+				Message: "Acceptable write latency",
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			req := require.New(t)
+			b, err := json.Marshal(test.fsPerf)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			getCollectedFileContents := func(filename string) ([]byte, error) {
+				return b, nil
+			}
+
+			result, err := analyzeHostFilesystemPerformance(test.hostAnalyzer, getCollectedFileContents)
+			if test.expectErr {
+				req.Error(err)
+			} else {
+				req.NoError(err)
+			}
+
+			assert.Equal(t, test.result, result)
+		})
+	}
+}
diff --git a/pkg/collect/host_filesystem_performance_test.go b/pkg/collect/host_filesystem_performance_test.go
new file mode 100644
index 00000000..a78c71b6
--- /dev/null
+++ b/pkg/collect/host_filesystem_performance_test.go
@@ -0,0 +1,59 @@
+package collect
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestGetPercentileIndex(t *testing.T) {
+	tests := []struct {
+		length int
+		p      float64
+		answer int
+	}{
+		{
+			length: 2,
+			p:      0.49,
+			answer: 0,
+		},
+		{
+			length: 2,
+			p:      0.5,
+			answer: 0,
+		},
+		{
+			length: 2,
+			p:      0.51,
+			answer: 1,
+		},
+		{
+			length: 100,
+			p:      0.01,
+			answer: 0,
+		},
+		{
+			length: 100,
+			p:      0.99,
+			answer: 98,
+		},
+		{
+			length: 100,
+			p:      0.995,
+			answer: 99,
+		},
+		{
+			length: 10000,
+			p:      0.999,
+			answer: 9989,
+		},
+	}
+	for _, test := range tests {
+		name := fmt.Sprintf("(%f, %d) == %d", test.p, test.length, test.answer)
+		t.Run(name, func(t *testing.T) {
+			output := getPercentileIndex(test.p, test.length)
+			if output != test.answer {
+				t.Errorf("Got %d, want %d", output, test.answer)
+			}
+		})
+	}
+}