Repository: replicatedhq/troubleshoot
Pull request #359 (2021-04-16 22:53:25)

Honor maxAge for log collector if set in the spec


Patch:
diff --git a/README.md b/README.md
index 471d948a..e6123d05 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ For a details on creating the custom resource files that drive preflight checks,
 
 
 ## Support Bundle
-A support bundle is an archive that's created in-cluster, by collecting logs and cluster information, and executing specified commands (including redaction of sensitive information). After creating a support bundle, the cluster operator will normally deliver it to the 3rd-party application vendor for analysis and disconnected debugging. Another Replicated project, [KOTS](https://github.com/replicatedhq/kots), provides k8s apps an in-cluster UI for processing support bundles and viewing analyzers (as well as support bundle collection).
+A support bundle is an archive that's created in-cluster, by collecting logs and cluster information, and executing specified commands (including redaction of sensitive information). After creating a support bundle, the cluster operator will normally deliver it to the 3rd-party application vendor for analysis and disconnected debugging. Another Replicated project, [Kotsadm](https://github.com/replicatedhq/kotsadm), provides cluster operators with an in-cluster UI for processing support bundles and viewing analyzers (as well as support bundle collection).
 
 To collect a sample support bundle, install the troubleshoot kubectl plugin:
 
diff --git a/examples/preflight/host/filesystem-performance.yaml b/examples/preflight/host/filesystem-performance.yaml
index 0e6ba947..d8e7c89b 100644
--- a/examples/preflight/host/filesystem-performance.yaml
+++ b/examples/preflight/host/filesystem-performance.yaml
@@ -6,30 +6,26 @@ spec:
   collectors:
     - filesystemPerformance:
         collectorName: etcd-perf
-        timeout: 2m
         directory: /var/lib/etcd
         fileSize: 22Mi
         operationSizeBytes: 2300
         datasync: true
-        enableBackgroundIOPS: true
-        backgroundIOPSWarmupSeconds: 10
-        backgroundWriteIOPS: 300
-        backgroundWriteIOPSJobs: 6
-        backgroundReadIOPS: 50
-        backgroundReadIOPSJobs: 1
   analyzers:
     - filesystemPerformance:
         collectorName: etcd-perf
         outcomes:
+          - fail:
+              when: "iops < 50"
+              message: Insufficient random read IOPS
           - pass:
               when: "p99 < 3ms"
-              message: "Write latency is great! (p99: {{ .P99 }})"
+              message: Write latency is great!
           - pass:
               when: "p99 < 5ms"
-              message: "Write latency is ok (p99: {{ .P99 }})"
+              message: Write latency is ok
           - warn:
               when: "p99 < 8ms"
-              message: "Write latency is high {{ .String }}"
+              message: Write latency is high
           - fail:
               when: "p99 >= 8ms"
-              message: "Write latency is too high {{ .String }}"
+              message: Write latency is too high
diff --git a/pkg/analyze/download.go b/pkg/analyze/download.go
index e2ee1bbd..08f06ce7 100644
--- a/pkg/analyze/download.go
+++ b/pkg/analyze/download.go
@@ -180,21 +180,13 @@ func parseAnalyzers(spec string) ([]*troubleshootv1beta2.Analyze, error) {
 		return nil, errors.Wrap(err, "failed to convert to v1beta2")
 	}
 
-	obj, gvk, err := decode(convertedSpec, nil, nil)
+	obj, _, err := decode(convertedSpec, nil, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to decode analyzers")
 	}
 
-	// SupportBundle overwrites Analyzer if defined
-	if gvk.Group == "troubleshoot.sh" && gvk.Version == "v1beta2" && gvk.Kind == "SupportBundle" {
-		supportBundle := obj.(*troubleshootv1beta2.SupportBundle)
-		return supportBundle.Spec.Analyzers, nil
-	} else if gvk.Group == "troubleshoot.sh" && gvk.Version == "v1beta2" && gvk.Kind == "Analyzer" {
-		analyzer := obj.(*troubleshootv1beta2.Analyzer)
-		return analyzer.Spec.Analyzers, nil
-	}
-
-	return nil, errors.Errorf("invalid gvk %q", gvk)
+	analyzer := obj.(*troubleshootv1beta2.Analyzer)
+	return analyzer.Spec.Analyzers, nil
 }
 
 func getDefaultAnalyzers() ([]*troubleshootv1beta2.Analyze, error) {
diff --git a/pkg/analyze/host_filesystem_performance.go b/pkg/analyze/host_filesystem_performance.go
index 88fa60e1..824b42bb 100644
--- a/pkg/analyze/host_filesystem_performance.go
+++ b/pkg/analyze/host_filesystem_performance.go
@@ -1,13 +1,11 @@
 package analyzer
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
-	"log"
 	"path/filepath"
+	"strconv"
 	"strings"
-	"text/template"
 	"time"
 
 	"github.com/pkg/errors"
@@ -53,7 +51,7 @@ func (a *AnalyzeHostFilesystemPerformance) Analyze(getCollectedFileContents func
 		if outcome.Fail != nil {
 			if outcome.Fail.When == "" {
 				result.IsFail = true
-				result.Message = renderFSPerfOutcome(outcome.Fail.Message, fsPerf)
+				result.Message = outcome.Fail.Message
 				result.URI = outcome.Fail.URI
 
 				return &result, nil
@@ -66,7 +64,7 @@ func (a *AnalyzeHostFilesystemPerformance) Analyze(getCollectedFileContents func
 
 			if isMatch {
 				result.IsFail = true
-				result.Message = renderFSPerfOutcome(outcome.Fail.Message, fsPerf)
+				result.Message = outcome.Fail.Message
 				result.URI = outcome.Fail.URI
 
 				return &result, nil
@@ -74,7 +72,7 @@ func (a *AnalyzeHostFilesystemPerformance) Analyze(getCollectedFileContents func
 		} else if outcome.Warn != nil {
 			if outcome.Warn.When == "" {
 				result.IsWarn = true
-				result.Message = renderFSPerfOutcome(outcome.Warn.Message, fsPerf)
+				result.Message = outcome.Warn.Message
 				result.URI = outcome.Warn.URI
 
 				return &result, nil
@@ -87,7 +85,7 @@ func (a *AnalyzeHostFilesystemPerformance) Analyze(getCollectedFileContents func
 
 			if isMatch {
 				result.IsWarn = true
-				result.Message = renderFSPerfOutcome(outcome.Warn.Message, fsPerf)
+				result.Message = outcome.Warn.Message
 				result.URI = outcome.Warn.URI
 
 				return &result, nil
@@ -95,7 +93,7 @@ func (a *AnalyzeHostFilesystemPerformance) Analyze(getCollectedFileContents func
 		} else if outcome.Pass != nil {
 			if outcome.Pass.When == "" {
 				result.IsPass = true
-				result.Message = renderFSPerfOutcome(outcome.Pass.Message, fsPerf)
+				result.Message = outcome.Pass.Message
 				result.URI = outcome.Pass.URI
 
 				return &result, nil
@@ -108,7 +106,7 @@ func (a *AnalyzeHostFilesystemPerformance) Analyze(getCollectedFileContents func
 
 			if isMatch {
 				result.IsPass = true
-				result.Message = renderFSPerfOutcome(outcome.Pass.Message, fsPerf)
+				result.Message = outcome.Pass.Message
 				result.URI = outcome.Pass.URI
 
 				return &result, nil
@@ -127,6 +125,14 @@ func compareHostFilesystemPerformanceConditionalToActual(conditional string, fsP
 	keyword := strings.ToLower(parts[0])
 	comparator := parts[1]
 
+	if keyword == "iops" {
+		desiredInt, err := strconv.ParseInt(parts[2], 10, 64)
+		if err != nil {
+			return false, errors.Wrapf(err, "failed to parse desired iops %q as integer", parts[2])
+		}
+		return doCompareHostFilesystemIOPS(comparator, fsPerf.IOPS, int(desiredInt))
+	}
+
 	desiredDuration, err := time.ParseDuration(parts[2])
 	if err != nil {
 		return false, errors.Wrapf(err, "failed to parse duration %q", parts[2])
@@ -195,17 +201,19 @@ func doCompareHostFilesystemPerformance(operator string, actual time.Duration, d
 	return false, fmt.Errorf("Unknown filesystem performance operator %q", operator)
 }
 
-func renderFSPerfOutcome(outcome string, fsPerf collect.FSPerfResults) string {
-	t, err := template.New("").Parse(outcome)
-	if err != nil {
-		log.Printf("Failed to parse filesystem performance outcome: %v", err)
-		return outcome
-	}
-	var buf bytes.Buffer
-	err = t.Execute(&buf, fsPerf)
-	if err != nil {
-		log.Printf("Failed to render filesystem performance outcome: %v", err)
-		return outcome
+func doCompareHostFilesystemIOPS(operator string, actual int, desired int) (bool, error) {
+	switch operator {
+	case "<":
+		return actual < desired, nil
+	case "<=":
+		return actual <= desired, nil
+	case ">":
+		return actual > desired, nil
+	case ">=":
+		return actual >= desired, nil
+	case "=", "==", "===":
+		return actual == desired, nil
 	}
-	return buf.String()
+
+	return false, fmt.Errorf("Unknown filesystem IOPS operator %q", operator)
 }
diff --git a/pkg/apis/troubleshoot/v1beta2/hostcollector_shared.go b/pkg/apis/troubleshoot/v1beta2/hostcollector_shared.go
index 4b4eaf6b..f7f07437 100644
--- a/pkg/apis/troubleshoot/v1beta2/hostcollector_shared.go
+++ b/pkg/apis/troubleshoot/v1beta2/hostcollector_shared.go
@@ -73,45 +73,13 @@ type TCPConnect struct {
 	Timeout           string `json:"timeout,omitempty"`
 }
 
-// FilesystemPerformance benchmarks sequential write latency on a single file.
-// The optional background IOPS feature attempts to mimic real-world conditions by running read and
-// write workloads prior to and during benchmark execution.
 type FilesystemPerformance struct {
-	HostCollectorMeta `json:",inline" yaml:",inline"`
-	// The size of each write operation performed while benchmarking. This does not apply to the
-	// background IOPS feature if enabled, since those must be fixed at 4096.
+	HostCollectorMeta  `json:",inline" yaml:",inline"`
 	OperationSizeBytes uint64 `json:"operationSize,omitempty"`
-	// The directory where the benchmark will create files.
-	Directory string `json:"directory,omitempty"`
-	// The size of the file used in the benchmark. The number of IO operations for the benchmark
-	// will be FileSize / OperationSizeBytes. Accepts valid Kubernetes resource units such as Mi.
-	FileSize string `json:"fileSize,omitempty"`
-	// Whether to call sync on the file after each write. Does not apply to background IOPS task.
-	Sync bool `json:"sync,omitempty"`
-	// Whether to call datasync on the file after each write. Skipped if Sync is also true. Does not
-	// apply to background IOPS task.
-	Datasync bool `json:"datasync,omitempty"`
-	// Total timeout, including background IOPS setup and warmup if enabled.
-	Timeout string `json:"timeout,omitempty"`
-
-	// Enable the background IOPS feature.
-	EnableBackgroundIOPS bool `json:"enableBackgroundIOPS"`
-	// How long to run the background IOPS read and write workloads prior to starting the benchmarks.
-	BackgroundIOPSWarmupSeconds int `json:"backgroundIOPSWarmupSeconds"`
-	// The target write IOPS to run while benchmarking. This is a limit and there is no guarantee
-	// it will be reached. This is the total IOPS for all background write jobs.
-	BackgroundWriteIOPS int `json:"backgroundWriteIOPS"`
-	// The target read IOPS to run while benchmarking. This is a limit and there is no guarantee
-	// it will be reached. This is the total IOPS for all background read jobs.
-	BackgroundReadIOPS int `json:"backgroundReadIOPS"`
-	// Number of threads to use for background write IOPS. This should be set high enough to reach
-	// the target specified in BackgroundWriteIOPS.
-	// Example: If BackgroundWriteIOPS is 100 and write latency is 10ms then a single job would
-	// barely be able to reach 100 IOPS so this should be at least 2.
-	BackgroundWriteIOPSJobs int `json:"backgroundWriteIOPSJobs"`
-	// Number of threads to use for background read IOPS. This should be set high enough to reach
-	// the target specified in BackgrounReadIOPS.
-	BackgroundReadIOPSJobs int `json:"backgroundReadIOPSJobs"`
+	Directory          string `json:"directory,omitempty"`
+	FileSize           string `json:"fileSize,omitempty"`
+	Sync               bool   `json:"sync,omitempty"`
+	Datasync           bool   `json:"datasync,omitempty"`
 }
 
 type Certificate struct {
diff --git a/pkg/client/troubleshootclientset/fake/register.go b/pkg/client/troubleshootclientset/fake/register.go
index 89d94f58..8010c884 100644
--- a/pkg/client/troubleshootclientset/fake/register.go
+++ b/pkg/client/troubleshootclientset/fake/register.go
@@ -29,7 +29,7 @@ import (
 
 var scheme = runtime.NewScheme()
 var codecs = serializer.NewCodecFactory(scheme)
-
+var parameterCodec = runtime.NewParameterCodec(scheme)
 var localSchemeBuilder = runtime.SchemeBuilder{
 	troubleshootv1beta1.AddToScheme,
 	troubleshootv1beta2.AddToScheme,
diff --git a/pkg/collect/ceph.go b/pkg/collect/ceph.go
index 1c6fd5d5..fa733e24 100644
--- a/pkg/collect/ceph.go
+++ b/pkg/collect/ceph.go
@@ -18,11 +18,10 @@ const (
 )
 
 type CephCommand struct {
-	ID             string
-	Command        []string
-	Args           []string
-	Format         string
-	DefaultTimeout string
+	ID      string
+	Command []string
+	Args    []string
+	Format  string
 }
 
 var CephCommands = []CephCommand{
@@ -74,13 +73,6 @@ var CephCommands = []CephCommand{
 		Args:    []string{"-f", "json-pretty"},
 		Format:  "json",
 	},
-	{
-		ID:             "rgw-stats",
-		Command:        []string{"radosgw-admin", "bucket", "stats"},
-		Args:           []string{"--rgw-cache-enabled=false"},
-		Format:         "json",
-		DefaultTimeout: "30s", // include a default timeout because this command will hang if the RGW daemon isn't running/is unhealthy
-	},
 }
 
 func Ceph(c *Collector, cephCollector *troubleshootv1beta2.Ceph) (map[string][]byte, error) {
@@ -108,17 +100,12 @@ func Ceph(c *Collector, cephCollector *troubleshootv1beta2.Ceph) (map[string][]b
 }
 
 func cephCommandExec(ctx context.Context, c *Collector, cephCollector *troubleshootv1beta2.Ceph, pod *corev1.Pod, command CephCommand) (map[string][]byte, error) {
-	timeout := cephCollector.Timeout
-	if timeout == "" {
-		timeout = command.DefaultTimeout
-	}
-
 	execCollector := &troubleshootv1beta2.Exec{
 		Selector:  labelsToSelector(pod.Labels),
 		Namespace: pod.Namespace,
 		Command:   command.Command,
 		Args:      command.Args,
-		Timeout:   timeout,
+		Timeout:   cephCollector.Timeout,
 	}
 	results, err := Exec(c, execCollector)
 	if err != nil {
diff --git a/pkg/collect/host_filesystem_performance.go b/pkg/collect/host_filesystem_performance.go
index 599f8fbd..b6f1b2f7 100644
--- a/pkg/collect/host_filesystem_performance.go
+++ b/pkg/collect/host_filesystem_performance.go
@@ -1,10 +1,8 @@
 package collect
 
 import (
-	"bytes"
 	"math"
 	"math/rand"
-	"text/template"
 	"time"
 
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
@@ -51,6 +49,7 @@ type FSPerfResults struct {
 	P999    time.Duration
 	P9995   time.Duration
 	P9999   time.Duration
+	IOPS    int
 }
 
 func getPercentileIndex(p float64, items int) int {
@@ -59,33 +58,3 @@ func getPercentileIndex(p float64, items int) int {
 	}
 	return int(math.Ceil(p*float64(items))) - 1
 }
-
-var fsPerfTmpl = template.Must(template.New("").Parse(`
-   Min: {{ .Min }}
-   Max: {{ .Max }}
-   Avg: {{ .Average }}
-    p1: {{ .P1 }}
-    p5: {{ .P5 }}
-   p10: {{ .P10 }}
-   p20: {{ .P20 }}
-   p30: {{ .P30 }}
-   p40: {{ .P40 }}
-   p50: {{ .P50 }}
-   p60: {{ .P60 }}
-   p70: {{ .P70 }}
-   p80: {{ .P80 }}
-   p90: {{ .P90 }}
-   p95: {{ .P95 }}
-   p99: {{ .P99 }}
- p99.5: {{ .P995 }}
- p99.9: {{ .P999 }}
-p99.95: {{ .P9995 }}
-p99.99: {{ .P9999 }}`))
-
-func (f FSPerfResults) String() string {
-	var buf bytes.Buffer
-
-	fsPerfTmpl.Execute(&buf, f)
-
-	return buf.String()
-}
diff --git a/pkg/collect/host_filesystem_performance_linux.go b/pkg/collect/host_filesystem_performance_linux.go
index 6d17b576..eb9f4bcd 100644
--- a/pkg/collect/host_filesystem_performance_linux.go
+++ b/pkg/collect/host_filesystem_performance_linux.go
@@ -1,10 +1,7 @@
 package collect
 
 import (
-	"context"
 	"encoding/json"
-	"fmt"
-	"io"
 	"log"
 	"math/rand"
 	"os"
@@ -38,17 +35,6 @@ func (d Durations) Swap(i, j int) {
 }
 
 func collectHostFilesystemPerformance(hostCollector *troubleshootv1beta2.FilesystemPerformance) (map[string][]byte, error) {
-	timeout := time.Minute
-	if hostCollector.Timeout != "" {
-		d, err := time.ParseDuration(hostCollector.Timeout)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to parse timeout %q", hostCollector.Timeout)
-		}
-		timeout = d
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-
 	var operationSize uint64 = 1024
 	if hostCollector.OperationSizeBytes != 0 {
 		operationSize = hostCollector.OperationSizeBytes
@@ -78,7 +64,6 @@ func collectHostFilesystemPerformance(hostCollector *troubleshootv1beta2.Filesys
 
 	f, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 	if err != nil {
-		log.Panic(err)
 		return nil, errors.Wrapf(err, "open %s", filename)
 	}
 	defer func() {
@@ -90,38 +75,6 @@ func collectHostFilesystemPerformance(hostCollector *troubleshootv1beta2.Filesys
 		}
 	}()
 
-	// Start the background IOPS task and wait for warmup
-	if hostCollector.EnableBackgroundIOPS {
-		// The done channel waits for all jobs to delete their work file after the context is
-		// canceled
-		jobs := hostCollector.BackgroundReadIOPSJobs + hostCollector.BackgroundWriteIOPSJobs
-		done := make(chan bool, jobs)
-		defer func() {
-			cancel()
-			for i := 0; i < jobs; i++ {
-				<-done
-			}
-		}()
-
-		opts := backgroundIOPSOpts{
-			read:      true,
-			iopsLimit: hostCollector.BackgroundReadIOPS,
-			jobs:      hostCollector.BackgroundReadIOPSJobs,
-			directory: hostCollector.Directory,
-		}
-		backgroundIOPS(ctx, opts, done)
-
-		opts = backgroundIOPSOpts{
-			read:      false,
-			iopsLimit: hostCollector.BackgroundWriteIOPS,
-			jobs:      hostCollector.BackgroundWriteIOPSJobs,
-			directory: hostCollector.Directory,
-		}
-		backgroundIOPS(ctx, opts, done)
-
-		time.Sleep(time.Second * time.Duration(hostCollector.BackgroundIOPSWarmupSeconds))
-	}
-
 	// Sequential writes benchmark
 	var written uint64 = 0
 	var results Durations
@@ -154,10 +107,6 @@ func collectHostFilesystemPerformance(hostCollector *troubleshootv1beta2.Filesys
 		results = append(results, d)
 
 		written += uint64(n)
-
-		if ctx.Err() != nil {
-			break
-		}
 	}
 
 	if len(results) == 0 {
@@ -194,119 +143,83 @@ func collectHostFilesystemPerformance(hostCollector *troubleshootv1beta2.Filesys
 		P9999:   results[getPercentileIndex(.9999, len(results))],
 	}
 
-	collectorName := hostCollector.CollectorName
-	if collectorName == "" {
-		collectorName = "filesystemPerformance"
+	// Random IOPS benchmark
+
+	// Re-open the file read+write in direct mode to prevent caching
+	if err := f.Close(); err != nil {
+		return nil, errors.Wrapf(err, "close %s", filename)
 	}
-	name := filepath.Join("filesystemPerformance", collectorName+".json")
-	b, err := json.Marshal(fsPerf)
+	f, err = os.OpenFile(filename, os.O_RDWR|syscall.O_DIRECT, 0600)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to marshal fs perf results")
+		return nil, errors.Wrapf(err, "open direct %s", filename)
 	}
 
-	return map[string][]byte{
-		name: b,
-	}, nil
-}
+	offsets := make([]int64, len(results))
 
-type backgroundIOPSOpts struct {
-	jobs      int
-	iopsLimit int
-	read      bool
-	directory string
-}
+	for index, p := range rand.Perm(len(results)) {
+		offsets[index] = int64(p) * int64(operationSize)
+	}
 
-func backgroundIOPS(ctx context.Context, opts backgroundIOPSOpts, done chan bool) {
-	r := rand.New(rand.NewSource(time.Now().Unix()))
+	// Use multiple workers to keep the filesystem busy. Since operations are serialized on a single
+	// file, more than 2 does not improve IOPS.
+	workers := 2
+	wg := sync.WaitGroup{}
+	m := sync.Mutex{}
 
-	// Waits until files are prepared before returning
-	var wg sync.WaitGroup
+	errs := make(chan error, workers)
 
-	// Rate limit IOPS with fixed window. Every second is a new window. All jobs increment
-	// the same counter before performing an operation. If the counter for the current
-	// window has already reached the IOPS limit then sleep until the beginning of the next
-	// window.
-	windows := map[int64]int{}
-	var mtx sync.Mutex
+	start := time.Now()
 
-	for i := 0; i < opts.jobs; i++ {
+	for i := 0; i < workers; i++ {
 		wg.Add(1)
 
 		go func(i int) {
-			filename := fmt.Sprintf("background-write-%d", i)
-			if opts.read {
-				filename = fmt.Sprintf("background-read-%d", i)
-			}
-			filename = filepath.Join(opts.directory, filename)
-			f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC|syscall.O_DIRECT, 0600)
-			if err != nil {
-				log.Printf("Failed to create temp file for background IOPS job: %v", err)
-				done <- true
-				return
-			}
-			defer func() {
-				if err := os.Remove(filename); err != nil {
-					log.Println(err.Error())
+			defer wg.Done()
+
+			data := make([]byte, int(operationSize))
+			fd := int(f.Fd())
+
+			for idx, offset := range offsets {
+				if idx%workers != i {
+					continue
 				}
-				done <- true
-			}()
 
-			// For O_DIRECT I/O must be aligned on the sector size of the underlying block device.
-			// Usually that's 512 but may also be 4096. Use 4096 since that works in either case.
-			opSize := 4096
-			blocks := 25600
-			fileSize := int64(opSize * blocks)
+				m.Lock()
+				n, err := syscall.Pread(fd, data, offset)
+				m.Unlock()
 
-			if opts.read {
-				_, err := io.Copy(f, io.LimitReader(r, fileSize))
 				if err != nil {
-					log.Printf("Failed to write temp file for background read IOPS jobs: %v", err)
-					return
+					errs <- errors.Wrapf(err, "failed to pread %d bytes to %s at offset %d", len(data), filename, offset)
 				}
-			} else {
-				err := f.Truncate(int64(opSize * blocks))
-				if err != nil {
-					log.Printf("Failed to resize temp file for backgroupd write IOPS jobs: %v", err)
+				if n != len(data) {
+					errs <- errors.Wrapf(err, "pread %d of %d bytes to %s at offset %d", n, len(data), filename, offset)
 				}
 			}
+		}(i)
+	}
 
-			wg.Done()
-			for {
-				if ctx.Err() != nil {
-					return
-				}
-
-				mtx.Lock()
-				windowKey := time.Now().Unix()
+	wg.Wait()
+	if len(errs) > 0 {
+		return nil, <-errs
+	}
 
-				if windows[windowKey] >= opts.iopsLimit {
-					mtx.Unlock()
+	d := time.Now().Sub(start)
+	nsPerIO := d / time.Duration(len(offsets))
+	iops := time.Second / nsPerIO
 
-					nextWindow := windowKey + 1
-					timeUntilNextWindow := time.Until(time.Unix(nextWindow, 0))
-					time.Sleep(timeUntilNextWindow)
-					continue
-				}
+	fsPerf.IOPS = int(iops)
 
-				windows[windowKey]++
-				mtx.Unlock()
-
-				blockOffset := rand.Intn(blocks)
-				offset := int64(blockOffset) * int64(opSize)
-				data := make([]byte, opSize)
-
-				if opts.read {
-					syscall.Pread(int(f.Fd()), data, offset)
-				} else {
-					if _, err := rand.Read(data); err != nil {
-						log.Printf("Failed to generate data for background IOPS write job: %v", err)
-						return
-					}
-					syscall.Pwrite(int(f.Fd()), data, offset)
-				}
-			}
-		}(i)
+	collectorName := hostCollector.CollectorName
+	if collectorName == "" {
+		collectorName = "filesystemPerformance"
+	}
+	name := filepath.Join("filesystemPerformance", collectorName+".json")
+	b, err := json.Marshal(fsPerf)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to marshal fs perf results")
 	}
 
-	wg.Wait()
+	return map[string][]byte{
+		name: b,
+	}, nil
 }
diff --git a/pkg/collect/logs.go b/pkg/collect/logs.go
index f0e6ecda..ae4931bd 100644
--- a/pkg/collect/logs.go
+++ b/pkg/collect/logs.go
@@ -110,27 +110,7 @@ func getPodLogs(ctx context.Context, client *kubernetes.Clientset, pod corev1.Po
 		Container: container,
 	}
 
-	defaultMaxLines := int64(10000)
-	if limits == nil || limits.MaxLines == 0 {
-		podLogOpts.TailLines = &defaultMaxLines
-	} else {
-		podLogOpts.TailLines = &limits.MaxLines
-	}
-	if limits != nil && !limits.SinceTime.IsZero() {
-		podLogOpts.SinceTime = &limits.SinceTime
-
-	} else if limits != nil && limits.MaxAge != "" {
-		parsedDuration, err := time.ParseDuration(limits.MaxAge)
-		if err != nil {
-			logger.Printf("unable to parse time duration %s\n", limits.MaxAge)
-		} else {
-			now := time.Now()
-			then := now.Add(0 - parsedDuration)
-			kthen := metav1.NewTime(then)
-
-			podLogOpts.SinceTime = &kthen
-		}
-	}
+	setLogLimits(&podLogOpts, limits, convertMaxAgeToTime)
 
 	fileKey := fmt.Sprintf("%s/%s", name, pod.Name)
 	if container != "" {
@@ -172,6 +152,48 @@ func getPodLogs(ctx context.Context, client *kubernetes.Clientset, pod corev1.Po
 	return result, nil
 }
 
+func convertMaxAgeToTime(maxAge string) *metav1.Time {
+	parsedDuration, err := time.ParseDuration(maxAge)
+	if err != nil {
+		logger.Printf("unable to parse time duration %s\n", maxAge)
+		return nil
+	}
+
+	now := time.Now()
+	then := now.Add(0 - parsedDuration)
+	kthen := metav1.NewTime(then)
+
+	return &kthen
+}
+
+func setLogLimits(podLogOpts *corev1.PodLogOptions, limits *troubleshootv1beta2.LogLimits, maxAgeParser func(maxAge string) *metav1.Time) {
+	if podLogOpts == nil {
+		return
+	}
+
+	defaultMaxLines := int64(10000)
+	if limits == nil {
+		podLogOpts.TailLines = &defaultMaxLines
+		return
+	}
+
+	if !limits.SinceTime.IsZero() {
+		podLogOpts.SinceTime = &limits.SinceTime
+		return
+	}
+
+	if limits.MaxAge != "" {
+		podLogOpts.SinceTime = maxAgeParser(limits.MaxAge)
+		return
+	}
+
+	if limits.MaxLines == 0 {
+		podLogOpts.TailLines = &defaultMaxLines
+	} else {
+		podLogOpts.TailLines = &limits.MaxLines
+	}
+}
+
 func getLogsErrorsFileName(logsCollector *troubleshootv1beta2.Logs) string {
 	if len(logsCollector.Name) > 0 {
 		return fmt.Sprintf("%s/errors.json", logsCollector.Name)


Test patch:
diff --git a/pkg/analyze/host_filesystem_performance_test.go b/pkg/analyze/host_filesystem_performance_test.go
index c52c6c7a..a885f473 100644
--- a/pkg/analyze/host_filesystem_performance_test.go
+++ b/pkg/analyze/host_filesystem_performance_test.go
@@ -19,6 +19,51 @@ func TestAnalyzeHostFilesystemPerformance(t *testing.T) {
 		result       *AnalyzeResult
 		expectErr    bool
 	}{
+		{
+			name: "IOPS",
+			fsPerf: &collect.FSPerfResults{
+				IOPS: 50,
+			},
+			hostAnalyzer: &troubleshootv1beta2.FilesystemPerformanceAnalyze{
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "iops < 20",
+							Message: "IOPS < 20",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "iops <= 20",
+							Message: "IOPS <= 30",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "iops > 70",
+							Message: "IOPS > 70",
+						},
+					},
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "iops >= 100",
+							Message: "IOPS >= 100",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "iops == 50",
+							Message: "IOPS == 50",
+						},
+					},
+				},
+			},
+			result: &AnalyzeResult{
+				Title:   "Filesystem Performance",
+				IsPass:  true,
+				Message: "IOPS == 50",
+			},
+		},
 		{
 			name: "Cover",
 			fsPerf: &collect.FSPerfResults{
diff --git a/pkg/collect/logs_test.go b/pkg/collect/logs_test.go
new file mode 100644
index 00000000..98a02293
--- /dev/null
+++ b/pkg/collect/logs_test.go
@@ -0,0 +1,79 @@
+package collect
+
+import (
+	"testing"
+	"time"
+
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func Test_setLogLimits(t *testing.T) {
+	defaultMaxLines := int64(10000)
+	customLines := int64(20)
+	maxAge := "10h"
+	sinceWhen := metav1.NewTime(time.Now().Add(-10 * time.Hour))
+
+	convertMaxAgeToTime := func(maxAge string) *metav1.Time {
+		return &sinceWhen
+	}
+
+	tests := []struct {
+		name     string
+		limits   *troubleshootv1beta2.LogLimits
+		expected corev1.PodLogOptions
+		validate func(t *testing.T, podLogOpts *corev1.PodLogOptions)
+	}{
+		{
+			name:   "default limits",
+			limits: nil,
+			expected: corev1.PodLogOptions{
+				TailLines: &defaultMaxLines,
+			},
+		},
+		{
+			name: "custom limit lines",
+			limits: &troubleshootv1beta2.LogLimits{
+				MaxLines: customLines,
+			},
+			expected: corev1.PodLogOptions{
+				TailLines: &customLines,
+			},
+		},
+		{
+			name: "max age",
+			limits: &troubleshootv1beta2.LogLimits{
+				MaxAge: maxAge,
+			},
+			expected: corev1.PodLogOptions{
+				SinceTime: &sinceWhen,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			req := require.New(t)
+
+			actual := corev1.PodLogOptions{}
+			setLogLimits(&actual, test.limits, convertMaxAgeToTime)
+
+			if test.expected.TailLines != nil {
+				req.NotNil(actual.TailLines)
+				assert.Equal(t, *test.expected.TailLines, *actual.TailLines)
+			} else {
+				req.Nil(actual.TailLines)
+			}
+
+			if test.expected.SinceTime != nil {
+				req.NotNil(actual.SinceTime)
+				assert.Equal(t, *test.expected.SinceTime, *actual.SinceTime)
+			} else {
+				req.Nil(actual.SinceTime)
+			}
+		})
+	}
+}