Repository: gotestyourself/gotestsum
Pull request #151 (2020-10-10 20:56:38)

Move the main Run functions into the cmd package
This allows someone to run `gotestsum` as a library, by passing in a command line.

For example, a project could add a file `test/main.go` with contents:

```go
package main

import (
	"fmt"
	"os"

	"gotest.tools/gotestsum/cmd"
)

func main() {
	name, args := os.Args[0], os.Args[1:]
	args = append([]string{"-f=testname", "--"}, args...)
	if err := cmd.Run(name, args); err != nil {
		fmt.Fprintln(os.Stderr, "ERROR ", err.Error())
		os.Exit(1)
	}
}
```

And run their tests with `go run ./test/`, which would run `gotestsum` without having to install the binary.


Patch:
diff --git a/.goreleaser.yml b/.goreleaser.yml
index 7e8097a..07bba2c 100644
--- a/.goreleaser.yml
+++ b/.goreleaser.yml
@@ -16,6 +16,7 @@ builds:
       - amd64
       - arm64
     env: [CGO_ENABLED=0]
+    ldflags: ["-s -w -X gotest.tools/gotestsum/cmd.version={{.Version}}"]
 
 checksum:
   name_template: '{{ .ProjectName }}-{{ .Version }}-checksums.txt'
diff --git a/flags.go b/cmd/flags.go
similarity index 99%
rename from flags.go
rename to cmd/flags.go
index 28c72ba..966c268 100644
--- a/flags.go
+++ b/cmd/flags.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"encoding/csv"
diff --git a/handler.go b/cmd/handler.go
similarity index 99%
rename from handler.go
rename to cmd/handler.go
index a074f2b..3b9b7e6 100644
--- a/handler.go
+++ b/cmd/handler.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"fmt"
diff --git a/internal/signalhandlerdriver/main.go b/cmd/internal/signalhandlerdriver/main.go
similarity index 100%
rename from internal/signalhandlerdriver/main.go
rename to cmd/internal/signalhandlerdriver/main.go
diff --git a/cmd/main.go b/cmd/main.go
new file mode 100644
index 0000000..619eb83
--- /dev/null
+++ b/cmd/main.go
@@ -0,0 +1,406 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"os/signal"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+	"gotest.tools/gotestsum/log"
+	"gotest.tools/gotestsum/testjson"
+)
+
+var version = "dev"
+
+func Run(name string, args []string) error {
+	flags, opts := setupFlags(name)
+	switch err := flags.Parse(args); {
+	case err == pflag.ErrHelp:
+		return nil
+	case err != nil:
+		usage(os.Stderr, name, flags)
+		return err
+	}
+	opts.args = flags.Args()
+	setupLogging(opts)
+
+	if opts.version {
+		fmt.Fprintf(os.Stdout, "gotestsum version %s\n", version)
+		return nil
+	}
+	return run(opts)
+}
+
+func setupFlags(name string) (*pflag.FlagSet, *options) {
+	opts := &options{
+		noSummary:                    newNoSummaryValue(),
+		junitTestCaseClassnameFormat: &junitFieldFormatValue{},
+		junitTestSuiteNameFormat:     &junitFieldFormatValue{},
+		postRunHookCmd:               &commandValue{},
+		stdout:                       os.Stdout,
+		stderr:                       os.Stderr,
+	}
+	flags := pflag.NewFlagSet(name, pflag.ContinueOnError)
+	flags.SetInterspersed(false)
+	flags.Usage = func() {
+		usage(os.Stdout, name, flags)
+	}
+	flags.StringVarP(&opts.format, "format", "f",
+		lookEnvWithDefault("GOTESTSUM_FORMAT", "short"),
+		"print format of test input")
+	flags.BoolVar(&opts.rawCommand, "raw-command", false,
+		"don't prepend 'go test -json' to the 'go test' command")
+	flags.StringVar(&opts.jsonFile, "jsonfile",
+		lookEnvWithDefault("GOTESTSUM_JSONFILE", ""),
+		"write all TestEvents to file")
+	flags.BoolVar(&opts.noColor, "no-color", color.NoColor, "disable color output")
+	flags.Var(opts.noSummary, "no-summary",
+		"do not print summary of: "+testjson.SummarizeAll.String())
+	flags.Var(opts.postRunHookCmd, "post-run-command",
+		"command to run after the tests have completed")
+
+	flags.StringVar(&opts.junitFile, "junitfile",
+		lookEnvWithDefault("GOTESTSUM_JUNITFILE", ""),
+		"write a JUnit XML file")
+	flags.Var(opts.junitTestSuiteNameFormat, "junitfile-testsuite-name",
+		"format the testsuite name field as: "+junitFieldFormatValues)
+	flags.Var(opts.junitTestCaseClassnameFormat, "junitfile-testcase-classname",
+		"format the testcase classname field as: "+junitFieldFormatValues)
+
+	flags.IntVar(&opts.rerunFailsMaxAttempts, "rerun-fails", 0,
+		"rerun failed tests until they all pass, or attempts exceeds maximum. Defaults to max 2 reruns when enabled.")
+	flags.Lookup("rerun-fails").NoOptDefVal = "2"
+	flags.IntVar(&opts.rerunFailsMaxInitialFailures, "rerun-fails-max-failures", 10,
+		"do not rerun any tests if the initial run has more than this number of failures")
+	flags.Var((*stringSlice)(&opts.packages), "packages",
+		"space separated list of package to test")
+	flags.StringVar(&opts.rerunFailsReportFile, "rerun-fails-report", "",
+		"write a report to the file, of the tests that were rerun")
+	flags.BoolVar(&opts.rerunFailsOnlyRootCases, "rerun-fails-only-root-testcases", false,
+		"rerun only root testcaes, instead of only subtests")
+	flags.Lookup("rerun-fails-only-root-testcases").Hidden = true
+
+	flags.BoolVar(&opts.debug, "debug", false, "enabled debug logging")
+	flags.BoolVar(&opts.version, "version", false, "show version and exit")
+	return flags, opts
+}
+
+func usage(out io.Writer, name string, flags *pflag.FlagSet) {
+	fmt.Fprintf(out, `Usage:
+    %[1]s [flags] [--] [go test flags]
+    %[1]s [command]
+
+Flags:
+`, name)
+	flags.SetOutput(out)
+	flags.PrintDefaults()
+	fmt.Fprint(out, `
+Formats:
+    dots                    print a character for each test
+    dots-v2                 experimental dots format, one package per line
+    pkgname                 print a line for each package
+    pkgname-and-test-fails  print a line for each package and failed test output
+    testname                print a line for each test and package
+    standard-quiet          standard go test format
+    standard-verbose        standard go test -v format
+
+Commands:
+    tool                    tools for working with test2json output
+`)
+}
+
+func lookEnvWithDefault(key, defValue string) string {
+	if value := os.Getenv(key); value != "" {
+		return value
+	}
+	return defValue
+}
+
+type options struct {
+	args                         []string
+	format                       string
+	debug                        bool
+	rawCommand                   bool
+	jsonFile                     string
+	junitFile                    string
+	postRunHookCmd               *commandValue
+	noColor                      bool
+	noSummary                    *noSummaryValue
+	junitTestSuiteNameFormat     *junitFieldFormatValue
+	junitTestCaseClassnameFormat *junitFieldFormatValue
+	rerunFailsMaxAttempts        int
+	rerunFailsMaxInitialFailures int
+	rerunFailsReportFile         string
+	rerunFailsOnlyRootCases      bool
+	packages                     []string
+	version                      bool
+
+	// shims for testing
+	stdout io.Writer
+	stderr io.Writer
+}
+
+func (o options) Validate() error {
+	if o.rerunFailsMaxAttempts > 0 && len(o.args) > 0 && !o.rawCommand && len(o.packages) == 0 {
+		return fmt.Errorf(
+			"when go test args are used with --rerun-fails-max-attempts " +
+				"the list of packages to test must be specified by the --packages flag")
+	}
+	return nil
+}
+
+func setupLogging(opts *options) {
+	if opts.debug {
+		log.SetLevel(log.DebugLevel)
+	}
+	color.NoColor = opts.noColor
+}
+
+func run(opts *options) error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	if err := opts.Validate(); err != nil {
+		return err
+	}
+
+	goTestProc, err := startGoTestFn(ctx, goTestCmdArgs(opts, rerunOpts{}))
+	if err != nil {
+		return err
+	}
+
+	handler, err := newEventHandler(opts)
+	if err != nil {
+		return err
+	}
+	defer handler.Close() // nolint: errcheck
+	cfg := testjson.ScanConfig{
+		Stdout:  goTestProc.stdout,
+		Stderr:  goTestProc.stderr,
+		Handler: handler,
+	}
+	exec, err := testjson.ScanTestOutput(cfg)
+	if err != nil {
+		return err
+	}
+	exitErr := goTestProc.cmd.Wait()
+	if exitErr == nil || opts.rerunFailsMaxAttempts == 0 {
+		return finishRun(opts, exec, exitErr)
+	}
+	if err := hasErrors(exitErr, exec); err != nil {
+		return finishRun(opts, exec, err)
+	}
+
+	failed := len(rerunFailsFilter(opts)(exec.Failed()))
+	if failed > opts.rerunFailsMaxInitialFailures {
+		err := fmt.Errorf(
+			"number of test failures (%d) exceeds maximum (%d) set by --rerun-fails-max-failures",
+			failed, opts.rerunFailsMaxInitialFailures)
+		return finishRun(opts, exec, err)
+	}
+
+	cfg = testjson.ScanConfig{Execution: exec, Handler: handler}
+	exitErr = rerunFailed(ctx, opts, cfg)
+	if err := writeRerunFailsReport(opts, exec); err != nil {
+		return err
+	}
+	return finishRun(opts, exec, exitErr)
+}
+
+func finishRun(opts *options, exec *testjson.Execution, exitErr error) error {
+	testjson.PrintSummary(opts.stdout, exec, opts.noSummary.value)
+
+	if err := writeJUnitFile(opts, exec); err != nil {
+		return err
+	}
+	if err := postRunHook(opts, exec); err != nil {
+		return err
+	}
+	return exitErr
+}
+
+func goTestCmdArgs(opts *options, rerunOpts rerunOpts) []string {
+	if opts.rawCommand {
+		var result []string
+		result = append(result, opts.args...)
+		result = append(result, rerunOpts.Args()...)
+		return result
+	}
+
+	args := opts.args
+	result := []string{"go", "test"}
+
+	if len(args) == 0 {
+		result = append(result, "-json")
+		if rerunOpts.runFlag != "" {
+			result = append(result, rerunOpts.runFlag)
+		}
+		return append(result, cmdArgPackageList(opts, rerunOpts, "./...")...)
+	}
+
+	if boolArgIndex("json", args) < 0 {
+		result = append(result, "-json")
+	}
+
+	if rerunOpts.runFlag != "" {
+		// Remove any existing run arg, it needs to be replaced with our new one
+		// and duplicate args are not allowed by 'go test'.
+		runIndex, runIndexEnd := argIndex("run", args)
+		if runIndex >= 0 && runIndexEnd < len(args) {
+			args = append(args[:runIndex], args[runIndexEnd+1:]...)
+		}
+		result = append(result, rerunOpts.runFlag)
+	}
+
+	pkgArgIndex := findPkgArgPosition(args)
+	result = append(result, args[:pkgArgIndex]...)
+	result = append(result, cmdArgPackageList(opts, rerunOpts)...)
+	result = append(result, args[pkgArgIndex:]...)
+	return result
+}
+
+func cmdArgPackageList(opts *options, rerunOpts rerunOpts, defPkgList ...string) []string {
+	switch {
+	case rerunOpts.pkg != "":
+		return []string{rerunOpts.pkg}
+	case len(opts.packages) > 0:
+		return opts.packages
+	case os.Getenv("TEST_DIRECTORY") != "":
+		return []string{os.Getenv("TEST_DIRECTORY")}
+	default:
+		return defPkgList
+	}
+}
+
+func boolArgIndex(flag string, args []string) int {
+	for i, arg := range args {
+		if arg == "-"+flag || arg == "--"+flag {
+			return i
+		}
+	}
+	return -1
+}
+
+func argIndex(flag string, args []string) (start, end int) {
+	for i, arg := range args {
+		if arg == "-"+flag || arg == "--"+flag {
+			return i, i + 1
+		}
+		if strings.HasPrefix(arg, "-"+flag+"=") || strings.HasPrefix(arg, "--"+flag+"=") {
+			return i, i
+		}
+	}
+	return -1, -1
+}
+
+// The package list is before the -args flag, or at the end of the args list
+// if the -args flag is not in args.
+// The -args flag is a 'go test' flag that indicates that all subsequent
+// args should be passed to the test binary. It requires that the list of
+// packages comes before -args, so we re-use it as a placeholder in the case
+// where some args must be passed to the test binary.
+func findPkgArgPosition(args []string) int {
+	if i := boolArgIndex("args", args); i >= 0 {
+		return i
+	}
+	return len(args)
+}
+
+type proc struct {
+	cmd    waiter
+	stdout io.Reader
+	stderr io.Reader
+}
+
+type waiter interface {
+	Wait() error
+}
+
+func startGoTest(ctx context.Context, args []string) (proc, error) {
+	if len(args) == 0 {
+		return proc{}, errors.New("missing command to run")
+	}
+
+	cmd := exec.CommandContext(ctx, args[0], args[1:]...)
+	p := proc{cmd: cmd}
+	log.Debugf("exec: %s", cmd.Args)
+	var err error
+	p.stdout, err = cmd.StdoutPipe()
+	if err != nil {
+		return p, err
+	}
+	p.stderr, err = cmd.StderrPipe()
+	if err != nil {
+		return p, err
+	}
+	if err := cmd.Start(); err != nil {
+		return p, errors.Wrapf(err, "failed to run %s", strings.Join(cmd.Args, " "))
+	}
+	log.Debugf("go test pid: %d", cmd.Process.Pid)
+
+	ctx, cancel := context.WithCancel(ctx)
+	newSignalHandler(ctx, cmd.Process.Pid)
+	p.cmd = &cancelWaiter{cancel: cancel, wrapped: p.cmd}
+	return p, nil
+}
+
+// ExitCodeWithDefault returns the ExitStatus of a process from the error returned by
+// exec.Run(). If the exit status is not available an error is returned.
+func ExitCodeWithDefault(err error) int {
+	if err == nil {
+		return 0
+	}
+	if exiterr, ok := err.(exitCoder); ok {
+		if code := exiterr.ExitCode(); code != -1 {
+			return code
+		}
+	}
+	return 127
+}
+
+type exitCoder interface {
+	ExitCode() int
+}
+
+func newSignalHandler(ctx context.Context, pid int) {
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+
+	go func() {
+		defer signal.Stop(c)
+
+		select {
+		case <-ctx.Done():
+			return
+		case s := <-c:
+			proc, err := os.FindProcess(pid)
+			if err != nil {
+				log.Errorf("failed to find pid of 'go test': %v", err)
+				return
+			}
+			if err := proc.Signal(s); err != nil {
+				log.Errorf("failed to interrupt 'go test': %v", err)
+				return
+			}
+		}
+	}()
+}
+
+// cancelWaiter wraps a waiter to cancel the context after the wrapped
+// Wait exits.
+type cancelWaiter struct {
+	cancel  func()
+	wrapped waiter
+}
+
+func (w *cancelWaiter) Wait() error {
+	err := w.wrapped.Wait()
+	w.cancel()
+	return err
+}
diff --git a/rerunfails.go b/cmd/rerunfails.go
similarity index 98%
rename from rerunfails.go
rename to cmd/rerunfails.go
index 56845a5..5ec7b0e 100644
--- a/rerunfails.go
+++ b/cmd/rerunfails.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"context"
@@ -89,7 +89,7 @@ func hasErrors(err error, exec *testjson.Execution) error {
 	case len(exec.Errors()) > 0:
 		return fmt.Errorf("rerun aborted because previous run had errors")
 	// Exit code 0 and 1 are expected.
-	case exitCodeWithDefault(err) > 1:
+	case ExitCodeWithDefault(err) > 1:
 		return fmt.Errorf("unexpected go test exit code: %v", err)
 	default:
 		return nil
diff --git a/testdata/e2e/flaky/flaky_test.go b/cmd/testdata/e2e/flaky/flaky_test.go
similarity index 100%
rename from testdata/e2e/flaky/flaky_test.go
rename to cmd/testdata/e2e/flaky/flaky_test.go
diff --git a/testdata/event-handler-missing-test-fail-expected b/cmd/testdata/event-handler-missing-test-fail-expected
similarity index 100%
rename from testdata/event-handler-missing-test-fail-expected
rename to cmd/testdata/event-handler-missing-test-fail-expected
diff --git a/testdata/go-test-json-flaky-rerun.out b/cmd/testdata/go-test-json-flaky-rerun.out
similarity index 100%
rename from testdata/go-test-json-flaky-rerun.out
rename to cmd/testdata/go-test-json-flaky-rerun.out
diff --git a/testdata/gotestsum-help-text b/cmd/testdata/gotestsum-help-text
similarity index 100%
rename from testdata/gotestsum-help-text
rename to cmd/testdata/gotestsum-help-text
diff --git a/testdata/post-run-hook-expected b/cmd/testdata/post-run-hook-expected
similarity index 100%
rename from testdata/post-run-hook-expected
rename to cmd/testdata/post-run-hook-expected
diff --git a/testdata/postrunhook/main.go b/cmd/testdata/postrunhook/main.go
similarity index 100%
rename from testdata/postrunhook/main.go
rename to cmd/testdata/postrunhook/main.go
diff --git a/do b/do
index ed26229..aed4df1 100755
--- a/do
+++ b/do
@@ -23,7 +23,7 @@ update-golden() {
 
 _update-golden() {
     PATH="$PWD/dist:$PATH" gotestsum -- \
-        . ./testjson ./internal/junitxml ./cmd/tool/slowest \
+        ./cmd ./testjson ./internal/junitxml ./cmd/tool/slowest \
         -test.update-golden
 }
 
diff --git a/main.go b/main.go
index 023c79d..c532be1 100644
--- a/main.go
+++ b/main.go
@@ -1,25 +1,14 @@
 package main
 
 import (
-	"context"
-	"fmt"
-	"io"
 	"os"
 	"os/exec"
-	"os/signal"
-	"strings"
 
-	"github.com/fatih/color"
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
 	"gotest.tools/gotestsum/cmd"
 	"gotest.tools/gotestsum/cmd/tool"
 	"gotest.tools/gotestsum/log"
-	"gotest.tools/gotestsum/testjson"
 )
 
-var version = "master"
-
 func main() {
 	err := route(os.Args)
 	switch err.(type) {
@@ -28,7 +17,7 @@ func main() {
 	case *exec.ExitError:
 		// go test should already report the error to stderr, exit with
 		// the same status code
-		os.Exit(exitCodeWithDefault(err))
+		os.Exit(cmd.ExitCodeWithDefault(err))
 	default:
 		log.Error(err.Error())
 		os.Exit(3)
@@ -42,393 +31,6 @@ func route(args []string) error {
 	case "tool":
 		return tool.Run(name+" "+next, rest)
 	default:
-		return runMain(name, args[1:])
-	}
-}
-
-func runMain(name string, args []string) error {
-	flags, opts := setupFlags(name)
-	switch err := flags.Parse(args); {
-	case err == pflag.ErrHelp:
-		return nil
-	case err != nil:
-		usage(os.Stderr, name, flags)
-		return err
-	}
-	opts.args = flags.Args()
-	setupLogging(opts)
-
-	if opts.version {
-		fmt.Fprintf(os.Stdout, "gotestsum version %s\n", version)
-		return nil
-	}
-	return run(opts)
-}
-
-func setupFlags(name string) (*pflag.FlagSet, *options) {
-	opts := &options{
-		noSummary:                    newNoSummaryValue(),
-		junitTestCaseClassnameFormat: &junitFieldFormatValue{},
-		junitTestSuiteNameFormat:     &junitFieldFormatValue{},
-		postRunHookCmd:               &commandValue{},
-		stdout:                       os.Stdout,
-		stderr:                       os.Stderr,
-	}
-	flags := pflag.NewFlagSet(name, pflag.ContinueOnError)
-	flags.SetInterspersed(false)
-	flags.Usage = func() {
-		usage(os.Stdout, name, flags)
-	}
-	flags.StringVarP(&opts.format, "format", "f",
-		lookEnvWithDefault("GOTESTSUM_FORMAT", "short"),
-		"print format of test input")
-	flags.BoolVar(&opts.rawCommand, "raw-command", false,
-		"don't prepend 'go test -json' to the 'go test' command")
-	flags.StringVar(&opts.jsonFile, "jsonfile",
-		lookEnvWithDefault("GOTESTSUM_JSONFILE", ""),
-		"write all TestEvents to file")
-	flags.BoolVar(&opts.noColor, "no-color", color.NoColor, "disable color output")
-	flags.Var(opts.noSummary, "no-summary",
-		"do not print summary of: "+testjson.SummarizeAll.String())
-	flags.Var(opts.postRunHookCmd, "post-run-command",
-		"command to run after the tests have completed")
-
-	flags.StringVar(&opts.junitFile, "junitfile",
-		lookEnvWithDefault("GOTESTSUM_JUNITFILE", ""),
-		"write a JUnit XML file")
-	flags.Var(opts.junitTestSuiteNameFormat, "junitfile-testsuite-name",
-		"format the testsuite name field as: "+junitFieldFormatValues)
-	flags.Var(opts.junitTestCaseClassnameFormat, "junitfile-testcase-classname",
-		"format the testcase classname field as: "+junitFieldFormatValues)
-
-	flags.IntVar(&opts.rerunFailsMaxAttempts, "rerun-fails", 0,
-		"rerun failed tests until they all pass, or attempts exceeds maximum. Defaults to max 2 reruns when enabled.")
-	flags.Lookup("rerun-fails").NoOptDefVal = "2"
-	flags.IntVar(&opts.rerunFailsMaxInitialFailures, "rerun-fails-max-failures", 10,
-		"do not rerun any tests if the initial run has more than this number of failures")
-	flags.Var((*stringSlice)(&opts.packages), "packages",
-		"space separated list of package to test")
-	flags.StringVar(&opts.rerunFailsReportFile, "rerun-fails-report", "",
-		"write a report to the file, of the tests that were rerun")
-	flags.BoolVar(&opts.rerunFailsOnlyRootCases, "rerun-fails-only-root-testcases", false,
-		"rerun only root testcaes, instead of only subtests")
-	flags.Lookup("rerun-fails-only-root-testcases").Hidden = true
-
-	flags.BoolVar(&opts.debug, "debug", false, "enabled debug logging")
-	flags.BoolVar(&opts.version, "version", false, "show version and exit")
-	return flags, opts
-}
-
-func usage(out io.Writer, name string, flags *pflag.FlagSet) {
-	fmt.Fprintf(out, `Usage:
-    %[1]s [flags] [--] [go test flags]
-    %[1]s [command]
-
-Flags:
-`, name)
-	flags.SetOutput(out)
-	flags.PrintDefaults()
-	fmt.Fprint(out, `
-Formats:
-    dots                    print a character for each test
-    dots-v2                 experimental dots format, one package per line
-    pkgname                 print a line for each package
-    pkgname-and-test-fails  print a line for each package and failed test output
-    testname                print a line for each test and package
-    standard-quiet          standard go test format
-    standard-verbose        standard go test -v format
-
-Commands:
-    tool                    tools for working with test2json output
-`)
-}
-
-func lookEnvWithDefault(key, defValue string) string {
-	if value := os.Getenv(key); value != "" {
-		return value
-	}
-	return defValue
-}
-
-type options struct {
-	args                         []string
-	format                       string
-	debug                        bool
-	rawCommand                   bool
-	jsonFile                     string
-	junitFile                    string
-	postRunHookCmd               *commandValue
-	noColor                      bool
-	noSummary                    *noSummaryValue
-	junitTestSuiteNameFormat     *junitFieldFormatValue
-	junitTestCaseClassnameFormat *junitFieldFormatValue
-	rerunFailsMaxAttempts        int
-	rerunFailsMaxInitialFailures int
-	rerunFailsReportFile         string
-	rerunFailsOnlyRootCases      bool
-	packages                     []string
-	version                      bool
-
-	// shims for testing
-	stdout io.Writer
-	stderr io.Writer
-}
-
-func (o options) Validate() error {
-	if o.rerunFailsMaxAttempts > 0 && len(o.args) > 0 && !o.rawCommand && len(o.packages) == 0 {
-		return fmt.Errorf(
-			"when go test args are used with --rerun-fails-max-attempts " +
-				"the list of packages to test must be specified by the --packages flag")
-	}
-	return nil
-}
-
-func setupLogging(opts *options) {
-	if opts.debug {
-		log.SetLevel(log.DebugLevel)
-	}
-	color.NoColor = opts.noColor
-}
-
-func run(opts *options) error {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	if err := opts.Validate(); err != nil {
-		return err
-	}
-
-	goTestProc, err := startGoTestFn(ctx, goTestCmdArgs(opts, rerunOpts{}))
-	if err != nil {
-		return err
-	}
-
-	handler, err := newEventHandler(opts)
-	if err != nil {
-		return err
-	}
-	defer handler.Close() // nolint: errcheck
-	cfg := testjson.ScanConfig{
-		Stdout:  goTestProc.stdout,
-		Stderr:  goTestProc.stderr,
-		Handler: handler,
-	}
-	exec, err := testjson.ScanTestOutput(cfg)
-	if err != nil {
-		return err
-	}
-	exitErr := goTestProc.cmd.Wait()
-	if exitErr == nil || opts.rerunFailsMaxAttempts == 0 {
-		return finishRun(opts, exec, exitErr)
-	}
-	if err := hasErrors(exitErr, exec); err != nil {
-		return finishRun(opts, exec, err)
-	}
-
-	failed := len(rerunFailsFilter(opts)(exec.Failed()))
-	if failed > opts.rerunFailsMaxInitialFailures {
-		err := fmt.Errorf(
-			"number of test failures (%d) exceeds maximum (%d) set by --rerun-fails-max-failures",
-			failed, opts.rerunFailsMaxInitialFailures)
-		return finishRun(opts, exec, err)
-	}
-
-	cfg = testjson.ScanConfig{Execution: exec, Handler: handler}
-	exitErr = rerunFailed(ctx, opts, cfg)
-	if err := writeRerunFailsReport(opts, exec); err != nil {
-		return err
-	}
-	return finishRun(opts, exec, exitErr)
-}
-
-func finishRun(opts *options, exec *testjson.Execution, exitErr error) error {
-	testjson.PrintSummary(opts.stdout, exec, opts.noSummary.value)
-
-	if err := writeJUnitFile(opts, exec); err != nil {
-		return err
-	}
-	if err := postRunHook(opts, exec); err != nil {
-		return err
-	}
-	return exitErr
-}
-
-func goTestCmdArgs(opts *options, rerunOpts rerunOpts) []string {
-	if opts.rawCommand {
-		var result []string
-		result = append(result, opts.args...)
-		result = append(result, rerunOpts.Args()...)
-		return result
-	}
-
-	args := opts.args
-	result := []string{"go", "test"}
-
-	if len(args) == 0 {
-		result = append(result, "-json")
-		if rerunOpts.runFlag != "" {
-			result = append(result, rerunOpts.runFlag)
-		}
-		return append(result, cmdArgPackageList(opts, rerunOpts, "./...")...)
-	}
-
-	if boolArgIndex("json", args) < 0 {
-		result = append(result, "-json")
-	}
-
-	if rerunOpts.runFlag != "" {
-		// Remove any existing run arg, it needs to be replaced with our new one
-		// and duplicate args are not allowed by 'go test'.
-		runIndex, runIndexEnd := argIndex("run", args)
-		if runIndex >= 0 && runIndexEnd < len(args) {
-			args = append(args[:runIndex], args[runIndexEnd+1:]...)
-		}
-		result = append(result, rerunOpts.runFlag)
-	}
-
-	pkgArgIndex := findPkgArgPosition(args)
-	result = append(result, args[:pkgArgIndex]...)
-	result = append(result, cmdArgPackageList(opts, rerunOpts)...)
-	result = append(result, args[pkgArgIndex:]...)
-	return result
-}
-
-func cmdArgPackageList(opts *options, rerunOpts rerunOpts, defPkgList ...string) []string {
-	switch {
-	case rerunOpts.pkg != "":
-		return []string{rerunOpts.pkg}
-	case len(opts.packages) > 0:
-		return opts.packages
-	case os.Getenv("TEST_DIRECTORY") != "":
-		return []string{os.Getenv("TEST_DIRECTORY")}
-	default:
-		return defPkgList
-	}
-}
-
-func boolArgIndex(flag string, args []string) int {
-	for i, arg := range args {
-		if arg == "-"+flag || arg == "--"+flag {
-			return i
-		}
-	}
-	return -1
-}
-
-func argIndex(flag string, args []string) (start, end int) {
-	for i, arg := range args {
-		if arg == "-"+flag || arg == "--"+flag {
-			return i, i + 1
-		}
-		if strings.HasPrefix(arg, "-"+flag+"=") || strings.HasPrefix(arg, "--"+flag+"=") {
-			return i, i
-		}
-	}
-	return -1, -1
-}
-
-// The package list is before the -args flag, or at the end of the args list
-// if the -args flag is not in args.
-// The -args flag is a 'go test' flag that indicates that all subsequent
-// args should be passed to the test binary. It requires that the list of
-// packages comes before -args, so we re-use it as a placeholder in the case
-// where some args must be passed to the test binary.
-func findPkgArgPosition(args []string) int {
-	if i := boolArgIndex("args", args); i >= 0 {
-		return i
-	}
-	return len(args)
-}
-
-type proc struct {
-	cmd    waiter
-	stdout io.Reader
-	stderr io.Reader
-}
-
-type waiter interface {
-	Wait() error
-}
-
-func startGoTest(ctx context.Context, args []string) (proc, error) {
-	if len(args) == 0 {
-		return proc{}, errors.New("missing command to run")
+		return cmd.Run(name, args[1:])
 	}
-
-	cmd := exec.CommandContext(ctx, args[0], args[1:]...)
-	p := proc{cmd: cmd}
-	log.Debugf("exec: %s", cmd.Args)
-	var err error
-	p.stdout, err = cmd.StdoutPipe()
-	if err != nil {
-		return p, err
-	}
-	p.stderr, err = cmd.StderrPipe()
-	if err != nil {
-		return p, err
-	}
-	if err := cmd.Start(); err != nil {
-		return p, errors.Wrapf(err, "failed to run %s", strings.Join(cmd.Args, " "))
-	}
-	log.Debugf("go test pid: %d", cmd.Process.Pid)
-
-	ctx, cancel := context.WithCancel(ctx)
-	newSignalHandler(ctx, cmd.Process.Pid)
-	p.cmd = &cancelWaiter{cancel: cancel, wrapped: p.cmd}
-	return p, nil
-}
-
-// GetExitCode returns the ExitStatus of a process from the error returned by
-// exec.Run(). If the exit status is not available an error is returned.
-func exitCodeWithDefault(err error) int {
-	if err == nil {
-		return 0
-	}
-	if exiterr, ok := err.(exitCoder); ok {
-		if code := exiterr.ExitCode(); code != -1 {
-			return code
-		}
-	}
-	return 127
-}
-
-type exitCoder interface {
-	ExitCode() int
-}
-
-func newSignalHandler(ctx context.Context, pid int) {
-	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt)
-
-	go func() {
-		defer signal.Stop(c)
-
-		select {
-		case <-ctx.Done():
-			return
-		case s := <-c:
-			proc, err := os.FindProcess(pid)
-			if err != nil {
-				log.Errorf("failed to find pid of 'go test': %v", err)
-				return
-			}
-			if err := proc.Signal(s); err != nil {
-				log.Errorf("failed to interrupt 'go test': %v", err)
-				return
-			}
-		}
-	}()
-}
-
-// cancelWaiter wraps a waiter to cancel the context after the wrapped
-// Wait exits.
-type cancelWaiter struct {
-	cancel  func()
-	wrapped waiter
-}
-
-func (w *cancelWaiter) Wait() error {
-	err := w.wrapped.Wait()
-	w.cancel()
-	return err
 }


Test patch:
diff --git a/flags_test.go b/cmd/flags_test.go
similarity index 98%
rename from flags_test.go
rename to cmd/flags_test.go
index 817cb9d..00f9631 100644
--- a/flags_test.go
+++ b/cmd/flags_test.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"testing"
diff --git a/handler_test.go b/cmd/handler_test.go
similarity index 91%
rename from handler_test.go
rename to cmd/handler_test.go
index d20b9bc..b8875b0 100644
--- a/handler_test.go
+++ b/cmd/handler_test.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"bytes"
@@ -35,7 +35,7 @@ func TestPostRunHook(t *testing.T) {
 
 func newExecFromTestData(t *testing.T) *testjson.Execution {
 	t.Helper()
-	f, err := os.Open("testjson/testdata/go-test-json.out")
+	f, err := os.Open("../testjson/testdata/go-test-json.out")
 	assert.NilError(t, err)
 	defer f.Close() // nolint: errcheck
 
@@ -58,7 +58,7 @@ func TestEventHandler_Event_WithMissingActionFail(t *testing.T) {
 	errBuf := new(bytes.Buffer)
 	format := testjson.NewEventFormatter(errBuf, "testname")
 
-	source := golden.Get(t, "../testjson/testdata/go-test-json-missing-test-fail.out")
+	source := golden.Get(t, "../../testjson/testdata/go-test-json-missing-test-fail.out")
 	cfg := testjson.ScanConfig{
 		Stdout:  bytes.NewReader(source),
 		Handler: &eventHandler{jsonFile: buf, formatter: format},
diff --git a/main_e2e_test.go b/cmd/main_e2e_test.go
similarity index 97%
rename from main_e2e_test.go
rename to cmd/main_e2e_test.go
index 655c866..e3f6986 100644
--- a/main_e2e_test.go
+++ b/cmd/main_e2e_test.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"bytes"
@@ -88,7 +88,7 @@ func TestE2E_RerunFails(t *testing.T) {
 			args: []string{
 				"-f=testname",
 				"--rerun-fails=2",
-				"--packages=./testjson/internal/broken",
+				"--packages=../testjson/internal/broken",
 				"--", "-count=1", "-tags=stubpkg",
 			},
 			expectedErr: "rerun aborted because previous run had errors",
@@ -182,7 +182,7 @@ func compileBinary(t *testing.T) string {
 		assert.NilError(t, err)
 
 		path := filepath.Join(tmpDir, "gotestsum")
-		result := icmd.RunCommand("go", "build", "-o", path, ".")
+		result := icmd.RunCommand("go", "build", "-o", path, "..")
 		result.Assert(t, icmd.Success)
 		return path
 	})
diff --git a/main_test.go b/cmd/main_test.go
similarity index 99%
rename from main_test.go
rename to cmd/main_test.go
index e520ae5..d260be9 100644
--- a/main_test.go
+++ b/cmd/main_test.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"bytes"
diff --git a/rerunfails_test.go b/cmd/rerunfails_test.go
similarity index 99%
rename from rerunfails_test.go
rename to cmd/rerunfails_test.go
index 242e77b..1b8e410 100644
--- a/rerunfails_test.go
+++ b/cmd/rerunfails_test.go
@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"bytes"
diff --git a/testdata/TestWriteRerunFailsReport-expected b/cmd/testdata/TestWriteRerunFailsReport-expected
similarity index 100%
rename from testdata/TestWriteRerunFailsReport-expected
rename to cmd/testdata/TestWriteRerunFailsReport-expected
diff --git a/cmd/testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun b/cmd/testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun
new file mode 100644
index 0000000..dafebe5
--- /dev/null
+++ b/cmd/testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun
@@ -0,0 +1,5 @@
+
+=== Errors
+../testjson/internal/broken/broken.go:5:21: undefined: somepackage
+
+DONE 0 tests, 1 error
diff --git a/cmd/testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun-go1.13 b/cmd/testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun-go1.13
new file mode 100644
index 0000000..dafebe5
--- /dev/null
+++ b/cmd/testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun-go1.13
@@ -0,0 +1,5 @@
+
+=== Errors
+../testjson/internal/broken/broken.go:5:21: undefined: somepackage
+
+DONE 0 tests, 1 error
diff --git a/cmd/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail b/cmd/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail
new file mode 100644
index 0000000..f4b5055
--- /dev/null
+++ b/cmd/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail
@@ -0,0 +1,84 @@
+PASS cmd/testdata/e2e/flaky.TestAlwaysPasses
+=== RUN   TestFailsRarely
+SEED:  0
+    flaky_test.go:51: not this time
+--- FAIL: TestFailsRarely
+FAIL cmd/testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  0
+    flaky_test.go:58: not this time
+--- FAIL: TestFailsSometimes
+FAIL cmd/testdata/e2e/flaky.TestFailsSometimes
+PASS cmd/testdata/e2e/flaky.TestFailsOften/subtest_always_passes
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail
+=== RUN   TestFailsOften
+SEED:  0
+--- FAIL: TestFailsOften
+FAIL cmd/testdata/e2e/flaky.TestFailsOften
+PASS cmd/testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
+PASS cmd/testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
+FAIL cmd/testdata/e2e/flaky
+
+DONE 8 tests, 4 failures
+
+PASS cmd/testdata/e2e/flaky.TestFailsRarely (re-run 1)
+PASS cmd/testdata/e2e/flaky
+PASS cmd/testdata/e2e/flaky.TestFailsSometimes (re-run 1)
+PASS cmd/testdata/e2e/flaky
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1)
+=== RUN   TestFailsOften
+SEED:  3
+--- FAIL: TestFailsOften
+FAIL cmd/testdata/e2e/flaky.TestFailsOften (re-run 1)
+FAIL cmd/testdata/e2e/flaky
+
+DONE 2 runs, 12 tests, 6 failures
+
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2)
+=== RUN   TestFailsOften
+SEED:  4
+--- FAIL: TestFailsOften
+FAIL cmd/testdata/e2e/flaky.TestFailsOften (re-run 2)
+FAIL cmd/testdata/e2e/flaky
+
+=== Failed
+=== FAIL: cmd/testdata/e2e/flaky TestFailsRarely
+SEED:  0
+    flaky_test.go:51: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsSometimes
+SEED:  0
+    flaky_test.go:58: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften
+SEED:  0
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1)
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften (re-run 1)
+SEED:  3
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2)
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften (re-run 2)
+SEED:  4
+
+
+DONE 3 runs, 14 tests, 8 failures
diff --git a/cmd/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.13 b/cmd/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.13
new file mode 100644
index 0000000..027123b
--- /dev/null
+++ b/cmd/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.13
@@ -0,0 +1,84 @@
+PASS cmd/testdata/e2e/flaky.TestAlwaysPasses
+=== RUN   TestFailsRarely
+SEED:  0
+--- FAIL: TestFailsRarely
+    flaky_test.go:51: not this time
+FAIL cmd/testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  0
+--- FAIL: TestFailsSometimes
+    flaky_test.go:58: not this time
+FAIL cmd/testdata/e2e/flaky.TestFailsSometimes
+PASS cmd/testdata/e2e/flaky.TestFailsOften/subtest_always_passes
+=== RUN   TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+FAIL cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail
+=== RUN   TestFailsOften
+SEED:  0
+--- FAIL: TestFailsOften
+FAIL cmd/testdata/e2e/flaky.TestFailsOften
+PASS cmd/testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
+PASS cmd/testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
+FAIL cmd/testdata/e2e/flaky
+
+DONE 8 tests, 4 failures
+
+PASS cmd/testdata/e2e/flaky.TestFailsRarely (re-run 1)
+PASS cmd/testdata/e2e/flaky
+PASS cmd/testdata/e2e/flaky.TestFailsSometimes (re-run 1)
+PASS cmd/testdata/e2e/flaky
+=== RUN   TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+FAIL cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1)
+=== RUN   TestFailsOften
+SEED:  3
+--- FAIL: TestFailsOften
+FAIL cmd/testdata/e2e/flaky.TestFailsOften (re-run 1)
+FAIL cmd/testdata/e2e/flaky
+
+DONE 2 runs, 12 tests, 6 failures
+
+=== RUN   TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+FAIL cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2)
+=== RUN   TestFailsOften
+SEED:  4
+--- FAIL: TestFailsOften
+FAIL cmd/testdata/e2e/flaky.TestFailsOften (re-run 2)
+FAIL cmd/testdata/e2e/flaky
+
+=== Failed
+=== FAIL: cmd/testdata/e2e/flaky TestFailsRarely
+SEED:  0
+    flaky_test.go:51: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsSometimes
+SEED:  0
+    flaky_test.go:58: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften
+SEED:  0
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1)
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften (re-run 1)
+SEED:  3
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2)
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften (re-run 2)
+SEED:  4
+
+
+DONE 3 runs, 14 tests, 8 failures
diff --git a/cmd/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success b/cmd/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success
new file mode 100644
index 0000000..1ee82e7
--- /dev/null
+++ b/cmd/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success
@@ -0,0 +1,109 @@
+PASS cmd/testdata/e2e/flaky.TestAlwaysPasses
+=== RUN   TestFailsRarely
+SEED:  0
+    flaky_test.go:51: not this time
+--- FAIL: TestFailsRarely
+FAIL cmd/testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  0
+    flaky_test.go:58: not this time
+--- FAIL: TestFailsSometimes
+FAIL cmd/testdata/e2e/flaky.TestFailsSometimes
+PASS cmd/testdata/e2e/flaky.TestFailsOften/subtest_always_passes
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail
+=== RUN   TestFailsOften
+SEED:  0
+--- FAIL: TestFailsOften
+FAIL cmd/testdata/e2e/flaky.TestFailsOften
+PASS cmd/testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
+PASS cmd/testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
+FAIL cmd/testdata/e2e/flaky
+
+DONE 8 tests, 4 failures
+
+PASS cmd/testdata/e2e/flaky.TestFailsRarely (re-run 1)
+PASS cmd/testdata/e2e/flaky
+PASS cmd/testdata/e2e/flaky.TestFailsSometimes (re-run 1)
+PASS cmd/testdata/e2e/flaky
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1)
+=== RUN   TestFailsOften
+SEED:  3
+--- FAIL: TestFailsOften
+FAIL cmd/testdata/e2e/flaky.TestFailsOften (re-run 1)
+FAIL cmd/testdata/e2e/flaky
+
+DONE 2 runs, 12 tests, 6 failures
+
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2)
+=== RUN   TestFailsOften
+SEED:  4
+--- FAIL: TestFailsOften
+FAIL cmd/testdata/e2e/flaky.TestFailsOften (re-run 2)
+FAIL cmd/testdata/e2e/flaky
+
+DONE 3 runs, 14 tests, 8 failures
+
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 3)
+=== RUN   TestFailsOften
+SEED:  5
+--- FAIL: TestFailsOften
+FAIL cmd/testdata/e2e/flaky.TestFailsOften (re-run 3)
+FAIL cmd/testdata/e2e/flaky
+
+DONE 4 runs, 16 tests, 10 failures
+
+PASS cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 4)
+PASS cmd/testdata/e2e/flaky.TestFailsOften (re-run 4)
+PASS cmd/testdata/e2e/flaky
+
+=== Failed
+=== FAIL: cmd/testdata/e2e/flaky TestFailsRarely
+SEED:  0
+    flaky_test.go:51: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsSometimes
+SEED:  0
+    flaky_test.go:58: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften
+SEED:  0
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1)
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften (re-run 1)
+SEED:  3
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2)
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften (re-run 2)
+SEED:  4
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 3)
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften (re-run 3)
+SEED:  5
+
+
+DONE 5 runs, 18 tests, 10 failures
diff --git a/cmd/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.13 b/cmd/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.13
new file mode 100644
index 0000000..229c23f
--- /dev/null
+++ b/cmd/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.13
@@ -0,0 +1,109 @@
+PASS cmd/testdata/e2e/flaky.TestAlwaysPasses
+=== RUN   TestFailsRarely
+SEED:  0
+--- FAIL: TestFailsRarely
+    flaky_test.go:51: not this time
+FAIL cmd/testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  0
+--- FAIL: TestFailsSometimes
+    flaky_test.go:58: not this time
+FAIL cmd/testdata/e2e/flaky.TestFailsSometimes
+PASS cmd/testdata/e2e/flaky.TestFailsOften/subtest_always_passes
+=== RUN   TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+FAIL cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail
+=== RUN   TestFailsOften
+SEED:  0
+--- FAIL: TestFailsOften
+FAIL cmd/testdata/e2e/flaky.TestFailsOften
+PASS cmd/testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
+PASS cmd/testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
+FAIL cmd/testdata/e2e/flaky
+
+DONE 8 tests, 4 failures
+
+PASS cmd/testdata/e2e/flaky.TestFailsRarely (re-run 1)
+PASS cmd/testdata/e2e/flaky
+PASS cmd/testdata/e2e/flaky.TestFailsSometimes (re-run 1)
+PASS cmd/testdata/e2e/flaky
+=== RUN   TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+FAIL cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1)
+=== RUN   TestFailsOften
+SEED:  3
+--- FAIL: TestFailsOften
+FAIL cmd/testdata/e2e/flaky.TestFailsOften (re-run 1)
+FAIL cmd/testdata/e2e/flaky
+
+DONE 2 runs, 12 tests, 6 failures
+
+=== RUN   TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+FAIL cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2)
+=== RUN   TestFailsOften
+SEED:  4
+--- FAIL: TestFailsOften
+FAIL cmd/testdata/e2e/flaky.TestFailsOften (re-run 2)
+FAIL cmd/testdata/e2e/flaky
+
+DONE 3 runs, 14 tests, 8 failures
+
+=== RUN   TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+FAIL cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 3)
+=== RUN   TestFailsOften
+SEED:  5
+--- FAIL: TestFailsOften
+FAIL cmd/testdata/e2e/flaky.TestFailsOften (re-run 3)
+FAIL cmd/testdata/e2e/flaky
+
+DONE 4 runs, 16 tests, 10 failures
+
+PASS cmd/testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 4)
+PASS cmd/testdata/e2e/flaky.TestFailsOften (re-run 4)
+PASS cmd/testdata/e2e/flaky
+
+=== Failed
+=== FAIL: cmd/testdata/e2e/flaky TestFailsRarely
+SEED:  0
+    flaky_test.go:51: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsSometimes
+SEED:  0
+    flaky_test.go:58: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften
+SEED:  0
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1)
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften (re-run 1)
+SEED:  3
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2)
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften (re-run 2)
+SEED:  4
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 3)
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsOften (re-run 3)
+SEED:  5
+
+
+DONE 5 runs, 18 tests, 10 failures
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun b/testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun
deleted file mode 100644
index df56b5a..0000000
--- a/testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun
+++ /dev/null
@@ -1,5 +0,0 @@
-
-=== Errors
-testjson/internal/broken/broken.go:5:21: undefined: somepackage
-
-DONE 0 tests, 1 error
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun-go1.13 b/testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun-go1.13
deleted file mode 100644
index df56b5a..0000000
--- a/testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun-go1.13
+++ /dev/null
@@ -1,5 +0,0 @@
-
-=== Errors
-testjson/internal/broken/broken.go:5:21: undefined: somepackage
-
-DONE 0 tests, 1 error
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail
deleted file mode 100644
index ff479b2..0000000
--- a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail
+++ /dev/null
@@ -1,84 +0,0 @@
-PASS testdata/e2e/flaky.TestAlwaysPasses
-=== RUN   TestFailsRarely
-SEED:  0
-    flaky_test.go:51: not this time
---- FAIL: TestFailsRarely
-FAIL testdata/e2e/flaky.TestFailsRarely
-=== RUN   TestFailsSometimes
-SEED:  0
-    flaky_test.go:58: not this time
---- FAIL: TestFailsSometimes
-FAIL testdata/e2e/flaky.TestFailsSometimes
-PASS testdata/e2e/flaky.TestFailsOften/subtest_always_passes
-=== RUN   TestFailsOften/subtest_may_fail
-    flaky_test.go:68: not this time
-    --- FAIL: TestFailsOften/subtest_may_fail
-FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail
-=== RUN   TestFailsOften
-SEED:  0
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften
-PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
-PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
-FAIL testdata/e2e/flaky
-
-DONE 8 tests, 4 failures
-
-PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
-PASS testdata/e2e/flaky
-PASS testdata/e2e/flaky.TestFailsSometimes (re-run 1)
-PASS testdata/e2e/flaky
-=== RUN   TestFailsOften/subtest_may_fail
-    flaky_test.go:68: not this time
-    --- FAIL: TestFailsOften/subtest_may_fail
-FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1)
-=== RUN   TestFailsOften
-SEED:  3
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
-FAIL testdata/e2e/flaky
-
-DONE 2 runs, 12 tests, 6 failures
-
-=== RUN   TestFailsOften/subtest_may_fail
-    flaky_test.go:68: not this time
-    --- FAIL: TestFailsOften/subtest_may_fail
-FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2)
-=== RUN   TestFailsOften
-SEED:  4
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
-FAIL testdata/e2e/flaky
-
-=== Failed
-=== FAIL: testdata/e2e/flaky TestFailsRarely
-SEED:  0
-    flaky_test.go:51: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsSometimes
-SEED:  0
-    flaky_test.go:58: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail
-    flaky_test.go:68: not this time
-    --- FAIL: TestFailsOften/subtest_may_fail
-
-=== FAIL: testdata/e2e/flaky TestFailsOften
-SEED:  0
-
-=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1)
-    flaky_test.go:68: not this time
-    --- FAIL: TestFailsOften/subtest_may_fail
-
-=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
-SEED:  3
-
-=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2)
-    flaky_test.go:68: not this time
-    --- FAIL: TestFailsOften/subtest_may_fail
-
-=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
-SEED:  4
-
-
-DONE 3 runs, 14 tests, 8 failures
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.13 b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.13
deleted file mode 100644
index b968136..0000000
--- a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.13
+++ /dev/null
@@ -1,84 +0,0 @@
-PASS testdata/e2e/flaky.TestAlwaysPasses
-=== RUN   TestFailsRarely
-SEED:  0
---- FAIL: TestFailsRarely
-    flaky_test.go:51: not this time
-FAIL testdata/e2e/flaky.TestFailsRarely
-=== RUN   TestFailsSometimes
-SEED:  0
---- FAIL: TestFailsSometimes
-    flaky_test.go:58: not this time
-FAIL testdata/e2e/flaky.TestFailsSometimes
-PASS testdata/e2e/flaky.TestFailsOften/subtest_always_passes
-=== RUN   TestFailsOften/subtest_may_fail
-    --- FAIL: TestFailsOften/subtest_may_fail
-        flaky_test.go:68: not this time
-FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail
-=== RUN   TestFailsOften
-SEED:  0
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften
-PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
-PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
-FAIL testdata/e2e/flaky
-
-DONE 8 tests, 4 failures
-
-PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
-PASS testdata/e2e/flaky
-PASS testdata/e2e/flaky.TestFailsSometimes (re-run 1)
-PASS testdata/e2e/flaky
-=== RUN   TestFailsOften/subtest_may_fail
-    --- FAIL: TestFailsOften/subtest_may_fail
-        flaky_test.go:68: not this time
-FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1)
-=== RUN   TestFailsOften
-SEED:  3
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
-FAIL testdata/e2e/flaky
-
-DONE 2 runs, 12 tests, 6 failures
-
-=== RUN   TestFailsOften/subtest_may_fail
-    --- FAIL: TestFailsOften/subtest_may_fail
-        flaky_test.go:68: not this time
-FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2)
-=== RUN   TestFailsOften
-SEED:  4
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
-FAIL testdata/e2e/flaky
-
-=== Failed
-=== FAIL: testdata/e2e/flaky TestFailsRarely
-SEED:  0
-    flaky_test.go:51: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsSometimes
-SEED:  0
-    flaky_test.go:58: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail
-    --- FAIL: TestFailsOften/subtest_may_fail
-        flaky_test.go:68: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften
-SEED:  0
-
-=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1)
-    --- FAIL: TestFailsOften/subtest_may_fail
-        flaky_test.go:68: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
-SEED:  3
-
-=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2)
-    --- FAIL: TestFailsOften/subtest_may_fail
-        flaky_test.go:68: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
-SEED:  4
-
-
-DONE 3 runs, 14 tests, 8 failures
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success
deleted file mode 100644
index 9d68fce..0000000
--- a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success
+++ /dev/null
@@ -1,109 +0,0 @@
-PASS testdata/e2e/flaky.TestAlwaysPasses
-=== RUN   TestFailsRarely
-SEED:  0
-    flaky_test.go:51: not this time
---- FAIL: TestFailsRarely
-FAIL testdata/e2e/flaky.TestFailsRarely
-=== RUN   TestFailsSometimes
-SEED:  0
-    flaky_test.go:58: not this time
---- FAIL: TestFailsSometimes
-FAIL testdata/e2e/flaky.TestFailsSometimes
-PASS testdata/e2e/flaky.TestFailsOften/subtest_always_passes
-=== RUN   TestFailsOften/subtest_may_fail
-    flaky_test.go:68: not this time
-    --- FAIL: TestFailsOften/subtest_may_fail
-FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail
-=== RUN   TestFailsOften
-SEED:  0
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften
-PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
-PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
-FAIL testdata/e2e/flaky
-
-DONE 8 tests, 4 failures
-
-PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
-PASS testdata/e2e/flaky
-PASS testdata/e2e/flaky.TestFailsSometimes (re-run 1)
-PASS testdata/e2e/flaky
-=== RUN   TestFailsOften/subtest_may_fail
-    flaky_test.go:68: not this time
-    --- FAIL: TestFailsOften/subtest_may_fail
-FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1)
-=== RUN   TestFailsOften
-SEED:  3
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
-FAIL testdata/e2e/flaky
-
-DONE 2 runs, 12 tests, 6 failures
-
-=== RUN   TestFailsOften/subtest_may_fail
-    flaky_test.go:68: not this time
-    --- FAIL: TestFailsOften/subtest_may_fail
-FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2)
-=== RUN   TestFailsOften
-SEED:  4
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
-FAIL testdata/e2e/flaky
-
-DONE 3 runs, 14 tests, 8 failures
-
-=== RUN   TestFailsOften/subtest_may_fail
-    flaky_test.go:68: not this time
-    --- FAIL: TestFailsOften/subtest_may_fail
-FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 3)
-=== RUN   TestFailsOften
-SEED:  5
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften (re-run 3)
-FAIL testdata/e2e/flaky
-
-DONE 4 runs, 16 tests, 10 failures
-
-PASS testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 4)
-PASS testdata/e2e/flaky.TestFailsOften (re-run 4)
-PASS testdata/e2e/flaky
-
-=== Failed
-=== FAIL: testdata/e2e/flaky TestFailsRarely
-SEED:  0
-    flaky_test.go:51: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsSometimes
-SEED:  0
-    flaky_test.go:58: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail
-    flaky_test.go:68: not this time
-    --- FAIL: TestFailsOften/subtest_may_fail
-
-=== FAIL: testdata/e2e/flaky TestFailsOften
-SEED:  0
-
-=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1)
-    flaky_test.go:68: not this time
-    --- FAIL: TestFailsOften/subtest_may_fail
-
-=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
-SEED:  3
-
-=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2)
-    flaky_test.go:68: not this time
-    --- FAIL: TestFailsOften/subtest_may_fail
-
-=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
-SEED:  4
-
-=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 3)
-    flaky_test.go:68: not this time
-    --- FAIL: TestFailsOften/subtest_may_fail
-
-=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 3)
-SEED:  5
-
-
-DONE 5 runs, 18 tests, 10 failures
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.13 b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.13
deleted file mode 100644
index dd6e90c..0000000
--- a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.13
+++ /dev/null
@@ -1,109 +0,0 @@
-PASS testdata/e2e/flaky.TestAlwaysPasses
-=== RUN   TestFailsRarely
-SEED:  0
---- FAIL: TestFailsRarely
-    flaky_test.go:51: not this time
-FAIL testdata/e2e/flaky.TestFailsRarely
-=== RUN   TestFailsSometimes
-SEED:  0
---- FAIL: TestFailsSometimes
-    flaky_test.go:58: not this time
-FAIL testdata/e2e/flaky.TestFailsSometimes
-PASS testdata/e2e/flaky.TestFailsOften/subtest_always_passes
-=== RUN   TestFailsOften/subtest_may_fail
-    --- FAIL: TestFailsOften/subtest_may_fail
-        flaky_test.go:68: not this time
-FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail
-=== RUN   TestFailsOften
-SEED:  0
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften
-PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
-PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
-FAIL testdata/e2e/flaky
-
-DONE 8 tests, 4 failures
-
-PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
-PASS testdata/e2e/flaky
-PASS testdata/e2e/flaky.TestFailsSometimes (re-run 1)
-PASS testdata/e2e/flaky
-=== RUN   TestFailsOften/subtest_may_fail
-    --- FAIL: TestFailsOften/subtest_may_fail
-        flaky_test.go:68: not this time
-FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1)
-=== RUN   TestFailsOften
-SEED:  3
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
-FAIL testdata/e2e/flaky
-
-DONE 2 runs, 12 tests, 6 failures
-
-=== RUN   TestFailsOften/subtest_may_fail
-    --- FAIL: TestFailsOften/subtest_may_fail
-        flaky_test.go:68: not this time
-FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2)
-=== RUN   TestFailsOften
-SEED:  4
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
-FAIL testdata/e2e/flaky
-
-DONE 3 runs, 14 tests, 8 failures
-
-=== RUN   TestFailsOften/subtest_may_fail
-    --- FAIL: TestFailsOften/subtest_may_fail
-        flaky_test.go:68: not this time
-FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 3)
-=== RUN   TestFailsOften
-SEED:  5
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften (re-run 3)
-FAIL testdata/e2e/flaky
-
-DONE 4 runs, 16 tests, 10 failures
-
-PASS testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 4)
-PASS testdata/e2e/flaky.TestFailsOften (re-run 4)
-PASS testdata/e2e/flaky
-
-=== Failed
-=== FAIL: testdata/e2e/flaky TestFailsRarely
-SEED:  0
-    flaky_test.go:51: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsSometimes
-SEED:  0
-    flaky_test.go:58: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail
-    --- FAIL: TestFailsOften/subtest_may_fail
-        flaky_test.go:68: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften
-SEED:  0
-
-=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1)
-    --- FAIL: TestFailsOften/subtest_may_fail
-        flaky_test.go:68: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
-SEED:  3
-
-=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2)
-    --- FAIL: TestFailsOften/subtest_may_fail
-        flaky_test.go:68: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
-SEED:  4
-
-=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 3)
-    --- FAIL: TestFailsOften/subtest_may_fail
-        flaky_test.go:68: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 3)
-SEED:  5
-
-
-DONE 5 runs, 18 tests, 10 failures