Repository: gotestyourself/gotestsum
Pull request #126 (2020-06-06 23:49:20)

Rerun failed tests
Adds a new `--rerun-fails[=n]` flag. When the flag is used `gotestsum` will rerun any failed tests up to `n` times until each one passes at least once.

The new feature can be used to mitigate the pain of flaky tests in CI. To avoid unnecessarily rerunning tests when there is a bug or problem with the test environment, if there are more than 10 failed tests, rerun will be skipped. The `--rerun-fails-max-failures` flag can be used to change that threshold.

Using the `--rerun-fails` flag along with `--raw-command` or with `go test` args may require special handling, which will need to be documented in #117 


Patch:
diff --git a/cmd/tool/slowest/slowest.go b/cmd/tool/slowest/slowest.go
index 635405b..5068fac 100644
--- a/cmd/tool/slowest/slowest.go
+++ b/cmd/tool/slowest/slowest.go
@@ -20,7 +20,7 @@ func Run(name string, args []string) error {
 	case err == pflag.ErrHelp:
 		return nil
 	case err != nil:
-		flags.Usage()
+		usage(os.Stderr, name, flags)
 		return err
 	}
 	return run(opts)
@@ -31,8 +31,22 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
 	flags := pflag.NewFlagSet(name, pflag.ContinueOnError)
 	flags.SetInterspersed(false)
 	flags.Usage = func() {
-		fmt.Fprintf(os.Stderr, `Usage:
-    %s [flags]
+		usage(os.Stdout, name, flags)
+	}
+	flags.StringVar(&opts.jsonfile, "jsonfile", os.Getenv("GOTESTSUM_JSONFILE"),
+		"path to test2json output, defaults to stdin")
+	flags.DurationVar(&opts.threshold, "threshold", 100*time.Millisecond,
+		"test cases with elapsed time greater than threshold are slow tests")
+	flags.StringVar(&opts.skipStatement, "skip-stmt", "",
+		"add this go statement to slow tests, instead of printing the list of slow tests")
+	flags.BoolVar(&opts.debug, "debug", false,
+		"enable debug logging.")
+	return flags, opts
+}
+
+func usage(out io.Writer, name string, flags *pflag.FlagSet) {
+	fmt.Fprintf(out, `Usage:
+    %[1]s [flags]
 
 Read a json file and print or update tests which are slower than threshold.
 The json file may be created with 'gotestsum --jsonfile' or 'go test -json'.
@@ -63,7 +77,7 @@ Alternatively, a custom --skip-stmt may be provided as a string:
             t.Skip("too slow for TEST_FAST")
         }
     '
-    go test -json -short ./... | %s --skip-stmt "$skip_stmt"
+    go test -json -short ./... | %[1]s --skip-stmt "$skip_stmt"
 
 Note that this tool does not add imports, so using a custom statement may require
 you to add imports to the file.
@@ -73,18 +87,9 @@ variable, following the same rules as the go toolchain. See
 https://golang.org/cmd/go/#hdr-Environment_variables.
 
 Flags:
-`, name, name)
-		flags.PrintDefaults()
-	}
-	flags.StringVar(&opts.jsonfile, "jsonfile", os.Getenv("GOTESTSUM_JSONFILE"),
-		"path to test2json output, defaults to stdin")
-	flags.DurationVar(&opts.threshold, "threshold", 100*time.Millisecond,
-		"test cases with elapsed time greater than threshold are slow tests")
-	flags.StringVar(&opts.skipStatement, "skip-stmt", "",
-		"add this go statement to slow tests, instead of printing the list of slow tests")
-	flags.BoolVar(&opts.debug, "debug", false,
-		"enable debug logging.")
-	return flags, opts
+`, name)
+	flags.SetOutput(out)
+	flags.PrintDefaults()
 }
 
 type options struct {
diff --git a/do b/do
index a22773c..03db7ba 100755
--- a/do
+++ b/do
@@ -8,7 +8,7 @@ binary() {
 }
 
 update-golden() {
-    gotestsum -- ./testjson ./internal/junitxml -test.update-golden
+    gotestsum -- . ./testjson ./internal/junitxml ./cmd/tool/slowest -test.update-golden
 }
 
 lint() {
diff --git a/flags.go b/flags.go
index c77920f..29afb5c 100644
--- a/flags.go
+++ b/flags.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/google/shlex"
 	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
 	"gotest.tools/gotestsum/internal/junitxml"
 	"gotest.tools/gotestsum/testjson"
 )
@@ -112,3 +113,22 @@ func (c *commandValue) Value() []string {
 	}
 	return c.command
 }
+
+var _ pflag.Value = (*stringSlice)(nil)
+
+// stringSlice is a flag.Value which populates the string slice by splitting
+// the raw flag value on spaces.
+type stringSlice []string
+
+func (s *stringSlice) String() string {
+	return strings.Join(*s, " ")
+}
+
+func (s *stringSlice) Set(raw string) error {
+	*s = append(*s, strings.Split(raw, " ")...)
+	return nil
+}
+
+func (s *stringSlice) Type() string {
+	return "list"
+}
diff --git a/handler.go b/handler.go
index df356f3..a074f2b 100644
--- a/handler.go
+++ b/handler.go
@@ -25,7 +25,8 @@ func (h *eventHandler) Err(text string) error {
 }
 
 func (h *eventHandler) Event(event testjson.TestEvent, execution *testjson.Execution) error {
-	if h.jsonFile != nil {
+	// ignore artificial events with no raw Bytes()
+	if h.jsonFile != nil && len(event.Bytes()) > 0 {
 		_, err := h.jsonFile.Write(append(event.Bytes(), '\n'))
 		if err != nil {
 			return errors.Wrap(err, "failed to write JSON file")
diff --git a/main.go b/main.go
index b180b5c..d558693 100644
--- a/main.go
+++ b/main.go
@@ -51,7 +51,7 @@ func runMain(name string, args []string) error {
 	case err == pflag.ErrHelp:
 		return nil
 	case err != nil:
-		flags.Usage()
+		usage(os.Stderr, name, flags)
 		return err
 	}
 	opts.args = flags.Args()
@@ -76,24 +76,8 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
 	flags := pflag.NewFlagSet(name, pflag.ContinueOnError)
 	flags.SetInterspersed(false)
 	flags.Usage = func() {
-		fmt.Fprintf(os.Stderr, `Usage:
-    %s [flags] [--] [go test flags]
-
-Flags:
-`, name)
-		flags.PrintDefaults()
-		fmt.Fprint(os.Stderr, `
-Formats:
-    dots                    print a character for each test
-    dots-v2                 experimental dots format, one package per line
-    pkgname                 print a line for each package
-    pkgname-and-test-fails  print a line for each package and failed test output
-    testname                print a line for each test and package
-    standard-quiet          standard go test format
-    standard-verbose        standard go test -v format
-`)
+		usage(os.Stdout, name, flags)
 	}
-	flags.BoolVar(&opts.debug, "debug", false, "enabled debug")
 	flags.StringVarP(&opts.format, "format", "f",
 		lookEnvWithDefault("GOTESTSUM_FORMAT", "short"),
 		"print format of test input")
@@ -102,22 +86,57 @@ Formats:
 	flags.StringVar(&opts.jsonFile, "jsonfile",
 		lookEnvWithDefault("GOTESTSUM_JSONFILE", ""),
 		"write all TestEvents to file")
-	flags.StringVar(&opts.junitFile, "junitfile",
-		lookEnvWithDefault("GOTESTSUM_JUNITFILE", ""),
-		"write a JUnit XML file")
 	flags.BoolVar(&opts.noColor, "no-color", color.NoColor, "disable color output")
 	flags.Var(opts.noSummary, "no-summary",
 		"do not print summary of: "+testjson.SummarizeAll.String())
+	flags.Var(opts.postRunHookCmd, "post-run-command",
+		"command to run after the tests have completed")
+
+	flags.StringVar(&opts.junitFile, "junitfile",
+		lookEnvWithDefault("GOTESTSUM_JUNITFILE", ""),
+		"write a JUnit XML file")
 	flags.Var(opts.junitTestSuiteNameFormat, "junitfile-testsuite-name",
 		"format the testsuite name field as: "+junitFieldFormatValues)
 	flags.Var(opts.junitTestCaseClassnameFormat, "junitfile-testcase-classname",
 		"format the testcase classname field as: "+junitFieldFormatValues)
-	flags.Var(opts.postRunHookCmd, "post-run-command",
-		"command to run after the tests have completed")
+
+	flags.IntVar(&opts.rerunFailsMaxAttempts, "rerun-fails", 0,
+		"rerun failed tests until they all pass, or attempts exceeds maximum. Defaults to max 2 reruns when enabled.")
+	flags.Lookup("rerun-fails").NoOptDefVal = "2"
+	flags.IntVar(&opts.rerunFailsMaxInitialFailures, "rerun-fails-max-failures", 10,
+		"do not rerun any tests if the initial run has more than this number of failures")
+	flags.Var((*stringSlice)(&opts.packages), "packages",
+		"space separated list of package to test")
+
+	flags.BoolVar(&opts.debug, "debug", false, "enabled debug logging")
 	flags.BoolVar(&opts.version, "version", false, "show version and exit")
 	return flags, opts
 }
 
+func usage(out io.Writer, name string, flags *pflag.FlagSet) {
+	fmt.Fprintf(out, `Usage:
+    %[1]s [flags] [--] [go test flags]
+    %[1]s [command]
+
+Flags:
+`, name)
+	flags.SetOutput(out)
+	flags.PrintDefaults()
+	fmt.Fprint(out, `
+Formats:
+    dots                    print a character for each test
+    dots-v2                 experimental dots format, one package per line
+    pkgname                 print a line for each package
+    pkgname-and-test-fails  print a line for each package and failed test output
+    testname                print a line for each test and package
+    standard-quiet          standard go test format
+    standard-verbose        standard go test -v format
+
+Commands:
+    tool                    tools for working with test2json output
+`)
+}
+
 func lookEnvWithDefault(key, defValue string) string {
 	if value := os.Getenv(key); value != "" {
 		return value
@@ -137,6 +156,9 @@ type options struct {
 	noSummary                    *noSummaryValue
 	junitTestSuiteNameFormat     *junitFieldFormatValue
 	junitTestCaseClassnameFormat *junitFieldFormatValue
+	rerunFailsMaxAttempts        int
+	rerunFailsMaxInitialFailures int
+	packages                     []string
 	version                      bool
 
 	// shims for testing
@@ -144,6 +166,15 @@ type options struct {
 	stderr io.Writer
 }
 
+func (o options) Validate() error {
+	if o.rerunFailsMaxAttempts > 0 && len(o.args) > 0 && !o.rawCommand && len(o.packages) == 0 {
+		return fmt.Errorf(
+			"when go test args are used with --rerun-fails-max-attempts " +
+				"the list of packages to test must be specified by the --packages flag")
+	}
+	return nil
+}
+
 func setupLogging(opts *options) {
 	if opts.debug {
 		log.SetLevel(log.DebugLevel)
@@ -152,28 +183,38 @@ func setupLogging(opts *options) {
 }
 
 func run(opts *options) error {
-	ctx := context.Background()
-	goTestProc, err := startGoTest(ctx, goTestCmdArgs(opts))
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	if err := opts.Validate(); err != nil {
+		return err
+	}
+
+	goTestProc, err := startGoTest(ctx, goTestCmdArgs(opts, rerunOpts{}))
 	if err != nil {
-		return errors.Wrapf(err, "failed to run %s %s",
-			goTestProc.cmd.Path,
-			strings.Join(goTestProc.cmd.Args, " "))
+		return errors.Wrapf(err, "failed to run %s", strings.Join(goTestProc.cmd.Args, " "))
 	}
-	defer goTestProc.cancel()
 
 	handler, err := newEventHandler(opts)
 	if err != nil {
 		return err
 	}
 	defer handler.Close() // nolint: errcheck
-	exec, err := testjson.ScanTestOutput(testjson.ScanConfig{
+	cfg := testjson.ScanConfig{
 		Stdout:  goTestProc.stdout,
 		Stderr:  goTestProc.stderr,
 		Handler: handler,
-	})
+	}
+	exec, err := testjson.ScanTestOutput(cfg)
 	if err != nil {
 		return err
 	}
+	goTestExitErr := goTestProc.cmd.Wait()
+	if opts.rerunFailsMaxAttempts > 0 {
+		cfg := testjson.ScanConfig{Execution: exec, Handler: handler}
+		goTestExitErr = rerunFailed(ctx, opts, cfg)
+	}
+
 	testjson.PrintSummary(opts.stdout, exec, opts.noSummary.value)
 	if err := writeJUnitFile(opts, exec); err != nil {
 		return err
@@ -181,44 +222,100 @@ func run(opts *options) error {
 	if err := postRunHook(opts, exec); err != nil {
 		return err
 	}
-	return goTestProc.cmd.Wait()
+	return goTestExitErr
 }
 
-func goTestCmdArgs(opts *options) []string {
+func goTestCmdArgs(opts *options, rerunOpts rerunOpts) []string {
+	if opts.rawCommand {
+		var result []string
+		result = append(result, opts.args...)
+		result = append(result, rerunOpts.Args()...)
+		return result
+	}
+
 	args := opts.args
-	defaultArgs := []string{"go", "test"}
-	switch {
-	case opts.rawCommand:
-		return args
-	case len(args) == 0:
-		return append(defaultArgs, "-json", pathFromEnv("./..."))
-	case !hasJSONArg(args):
-		defaultArgs = append(defaultArgs, "-json")
+	result := []string{"go", "test"}
+
+	if len(args) == 0 {
+		result = append(result, "-json")
+		if rerunOpts.runFlag != "" {
+			result = append(result, rerunOpts.runFlag)
+		}
+		return append(result, cmdArgPackageList(opts, rerunOpts, "./...")...)
 	}
-	if testPath := pathFromEnv(""); testPath != "" {
-		args = append(args, testPath)
+
+	if boolArgIndex("json", args) < 0 {
+		result = append(result, "-json")
+	}
+
+	if rerunOpts.runFlag != "" {
+		// Remove any existing run arg, it needs to be replaced with our new one
+		// and duplicate args are not allowed by 'go test'.
+		runIndex, runIndexEnd := argIndex("run", args)
+		if runIndex >= 0 && runIndexEnd < len(args) {
+			args = append(args[:runIndex], args[runIndexEnd+1:]...)
+		}
+		result = append(result, rerunOpts.runFlag)
+	}
+
+	pkgArgIndex := findPkgArgPosition(args)
+	result = append(result, args[:pkgArgIndex]...)
+	result = append(result, cmdArgPackageList(opts, rerunOpts)...)
+	result = append(result, args[pkgArgIndex:]...)
+	return result
+}
+
+func cmdArgPackageList(opts *options, rerunOpts rerunOpts, defPkgList ...string) []string {
+	switch {
+	case rerunOpts.pkg != "":
+		return []string{rerunOpts.pkg}
+	case len(opts.packages) > 0:
+		return opts.packages
+	case os.Getenv("TEST_DIRECTORY") != "":
+		return []string{os.Getenv("TEST_DIRECTORY")}
+	default:
+		return defPkgList
 	}
-	return append(defaultArgs, args...)
 }
 
-func pathFromEnv(defaultPath string) string {
-	return lookEnvWithDefault("TEST_DIRECTORY", defaultPath)
+func boolArgIndex(flag string, args []string) int {
+	for i, arg := range args {
+		if arg == "-"+flag || arg == "--"+flag {
+			return i
+		}
+	}
+	return -1
 }
 
-func hasJSONArg(args []string) bool {
-	for _, arg := range args {
-		if arg == "-json" || arg == "--json" {
-			return true
+func argIndex(flag string, args []string) (start, end int) {
+	for i, arg := range args {
+		if arg == "-"+flag || arg == "--"+flag {
+			return i, i + 1
 		}
+		if strings.HasPrefix(arg, "-"+flag+"=") || strings.HasPrefix(arg, "--"+flag+"=") {
+			return i, i
+		}
+	}
+	return -1, -1
+}
+
+// The package list is before the -args flag, or at the end of the args list
+// if the -args flag is not in args.
+// The -args flag is a 'go test' flag that indicates that all subsequent
+// args should be passed to the test binary. It requires that the list of
+// packages comes before -args, so we re-use it as a placeholder in the case
+// where some args must be passed to the test binary.
+func findPkgArgPosition(args []string) int {
+	if i := boolArgIndex("args", args); i >= 0 {
+		return i
 	}
-	return false
+	return len(args)
 }
 
 type proc struct {
 	cmd    *exec.Cmd
 	stdout io.Reader
 	stderr io.Reader
-	cancel func()
 }
 
 func startGoTest(ctx context.Context, args []string) (proc, error) {
@@ -226,10 +323,8 @@ func startGoTest(ctx context.Context, args []string) (proc, error) {
 		return proc{}, errors.New("missing command to run")
 	}
 
-	ctx, cancel := context.WithCancel(ctx)
 	p := proc{
-		cmd:    exec.CommandContext(ctx, args[0], args[1:]...),
-		cancel: cancel,
+		cmd: exec.CommandContext(ctx, args[0], args[1:]...),
 	}
 	log.Debugf("exec: %s", p.cmd.Args)
 	var err error
diff --git a/rerunfails.go b/rerunfails.go
new file mode 100644
index 0000000..203af69
--- /dev/null
+++ b/rerunfails.go
@@ -0,0 +1,114 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+	"gotest.tools/gotestsum/testjson"
+)
+
+type rerunOpts struct {
+	runFlag string
+	pkg     string
+}
+
+func (o rerunOpts) Args() []string {
+	var result []string
+	if o.runFlag != "" {
+		result = append(result, o.runFlag)
+	}
+	if o.pkg != "" {
+		result = append(result, o.pkg)
+	}
+	return result
+}
+
+func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanConfig) error {
+	failed := len(scanConfig.Execution.Failed())
+	if failed > opts.rerunFailsMaxInitialFailures {
+		return fmt.Errorf(
+			"number of test failures (%d) exceeds maximum (%d) set by --rerun-fails-max-failures",
+			failed, opts.rerunFailsMaxInitialFailures)
+	}
+
+	rec := newFailureRecorderFromExecution(scanConfig.Execution)
+
+	var lastErr error
+	for count := 0; rec.count() > 0 && count < opts.rerunFailsMaxAttempts; count++ {
+		nextRec := newFailureRecorder(scanConfig.Handler)
+
+		for pkg, testCases := range rec.pkgFailures {
+			rerun := rerunOpts{
+				runFlag: goTestRunFlagFromTestCases(testCases),
+				pkg:     pkg,
+			}
+			goTestProc, err := startGoTest(ctx, goTestCmdArgs(opts, rerun))
+			if err != nil {
+				return errors.Wrapf(err, "failed to run %s", strings.Join(goTestProc.cmd.Args, " "))
+			}
+
+			cfg := testjson.ScanConfig{
+				Stdout:    goTestProc.stdout,
+				Stderr:    goTestProc.stderr,
+				Handler:   nextRec,
+				Execution: scanConfig.Execution,
+			}
+			if _, err := testjson.ScanTestOutput(cfg); err != nil {
+				return err
+			}
+			lastErr = goTestProc.cmd.Wait()
+			rec = nextRec
+		}
+	}
+	return lastErr
+}
+
+type failureRecorder struct {
+	testjson.EventHandler
+	pkgFailures map[string][]string
+}
+
+func newFailureRecorder(handler testjson.EventHandler) *failureRecorder {
+	return &failureRecorder{
+		EventHandler: handler,
+		pkgFailures:  make(map[string][]string),
+	}
+}
+
+func newFailureRecorderFromExecution(exec *testjson.Execution) *failureRecorder {
+	r := newFailureRecorder(nil)
+	for _, tc := range exec.Failed() {
+		r.pkgFailures[tc.Package] = append(r.pkgFailures[tc.Package], tc.Test)
+	}
+	return r
+}
+
+func (r *failureRecorder) Event(event testjson.TestEvent, execution *testjson.Execution) error {
+	if !event.PackageEvent() && event.Action == testjson.ActionFail {
+		r.pkgFailures[event.Package] = append(r.pkgFailures[event.Package], event.Test)
+	}
+	return r.EventHandler.Event(event, execution)
+}
+
+func (r *failureRecorder) count() int {
+	total := 0
+	for _, tcs := range r.pkgFailures {
+		total += len(tcs)
+	}
+	return total
+}
+
+func goTestRunFlagFromTestCases(tcs []string) string {
+	buf := new(strings.Builder)
+	buf.WriteString("-run=^(")
+	for i, tc := range tcs {
+		if i != 0 {
+			buf.WriteString("|")
+		}
+		buf.WriteString(tc)
+	}
+	buf.WriteString(")$")
+	return buf.String()
+}


Test patch:
diff --git a/cmd/tool/slowest/slowest_test.go b/cmd/tool/slowest/slowest_test.go
index 5f26263..2ef86ce 100644
--- a/cmd/tool/slowest/slowest_test.go
+++ b/cmd/tool/slowest/slowest_test.go
@@ -1,6 +1,7 @@
 package slowest
 
 import (
+	"bytes"
 	"strings"
 	"testing"
 	"time"
@@ -8,8 +9,21 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"gotest.tools/gotestsum/testjson"
 	"gotest.tools/v3/assert"
+	"gotest.tools/v3/env"
+	"gotest.tools/v3/golden"
 )
 
+func TestUsage_WithFlagsFromSetupFlags(t *testing.T) {
+	defer env.PatchAll(t, nil)()
+
+	name := "gotestsum tool slowest"
+	flags, _ := setupFlags(name)
+	buf := new(bytes.Buffer)
+	usage(buf, name, flags)
+
+	golden.Assert(t, buf.String(), "cmd-flags-help-text")
+}
+
 func TestAggregateTestCases(t *testing.T) {
 	cases := []testjson.TestCase{
 		{Test: "TestOne", Package: "pkg", Elapsed: time.Second},
diff --git a/cmd/tool/slowest/testdata/cmd-flags-help-text b/cmd/tool/slowest/testdata/cmd-flags-help-text
new file mode 100644
index 0000000..8dcbff5
--- /dev/null
+++ b/cmd/tool/slowest/testdata/cmd-flags-help-text
@@ -0,0 +1,46 @@
+Usage:
+    gotestsum tool slowest [flags]
+
+Read a json file and print or update tests which are slower than threshold.
+The json file may be created with 'gotestsum --jsonfile' or 'go test -json'.
+If a TestCase appears more than once in the json file, it will only appear once
+in the output, and the median value of all the elapsed times will be used.
+
+By default this command will print the list of tests slower than threshold to stdout.
+The list will be sorted from slowest to fastest.
+
+If --skip-stmt is set, instead of printing the list to stdout, the AST for the
+Go source code in the working directory tree will be modified. The value of
+--skip-stmt will be added to Go test files as the first statement in all the test
+functions which are slower than threshold.
+
+The --skip-stmt flag may be set to the name of a predefined statement, or to
+Go source code which will be parsed as a go/ast.Stmt. Currently there is only one
+predefined statement, --skip-stmt=testing.Short, which uses this Go statement:
+
+    if testing.Short() {
+        t.Skip("too slow for testing.Short")
+    }
+
+
+Alternatively, a custom --skip-stmt may be provided as a string:
+
+    skip_stmt='
+        if os.Getenv("TEST_FAST") {
+            t.Skip("too slow for TEST_FAST")
+        }
+    '
+    go test -json -short ./... | gotestsum tool slowest --skip-stmt "$skip_stmt"
+
+Note that this tool does not add imports, so using a custom statement may require
+you to add imports to the file.
+
+Go build flags, such as build tags, may be set using the GOFLAGS environment
+variable, following the same rules as the go toolchain. See
+https://golang.org/cmd/go/#hdr-Environment_variables.
+
+Flags:
+      --debug                enable debug logging.
+      --jsonfile string      path to test2json output, defaults to stdin
+      --skip-stmt string     add this go statement to slow tests, instead of printing the list of slow tests
+      --threshold duration   test cases with elapsed time greater than threshold are slow tests (default 100ms)
diff --git a/handler_test.go b/handler_test.go
index 068508a..d20b9bc 100644
--- a/handler_test.go
+++ b/handler_test.go
@@ -46,3 +46,28 @@ func newExecFromTestData(t *testing.T) *testjson.Execution {
 	assert.NilError(t, err)
 	return exec
 }
+
+type bufferCloser struct {
+	bytes.Buffer
+}
+
+func (bufferCloser) Close() error { return nil }
+
+func TestEventHandler_Event_WithMissingActionFail(t *testing.T) {
+	buf := new(bufferCloser)
+	errBuf := new(bytes.Buffer)
+	format := testjson.NewEventFormatter(errBuf, "testname")
+
+	source := golden.Get(t, "../testjson/testdata/go-test-json-missing-test-fail.out")
+	cfg := testjson.ScanConfig{
+		Stdout:  bytes.NewReader(source),
+		Handler: &eventHandler{jsonFile: buf, formatter: format},
+	}
+	_, err := testjson.ScanTestOutput(cfg)
+	assert.NilError(t, err)
+
+	assert.Equal(t, buf.String(), string(source))
+	// confirm the artificial event was sent to the handler by checking the output
+	// of the formatter.
+	golden.Assert(t, errBuf.String(), "event-handler-missing-test-fail-expected")
+}
diff --git a/internal/text/testing.go b/internal/text/testing.go
new file mode 100644
index 0000000..2a13901
--- /dev/null
+++ b/internal/text/testing.go
@@ -0,0 +1,42 @@
+package text
+
+import (
+	"bufio"
+	"io"
+	"strings"
+	"testing"
+
+	"gotest.tools/v3/assert"
+)
+
+// ProcessLines from the Reader by passing each one to ops. The output of each
+// op is passed to the next. Returns the string created by joining all the
+// processed lines.
+func ProcessLines(t *testing.T, r io.Reader, ops ...func(string) string) string {
+	t.Helper()
+	out := new(strings.Builder)
+	scan := bufio.NewScanner(r)
+	for scan.Scan() {
+		line := scan.Text()
+		for _, op := range ops {
+			line = op(line)
+		}
+		out.WriteString(line + "\n")
+	}
+	assert.NilError(t, scan.Err())
+	return out.String()
+}
+
+func OpRemoveSummaryLineElapsedTime(line string) string {
+	if i := strings.Index(line, " in "); i > 0 {
+		return line[:i]
+	}
+	return line
+}
+
+func OpRemoveTestElapsedTime(line string) string {
+	if i := strings.Index(line, " (0.0"); i > 0 && i+8 == len(line) {
+		return line[:i]
+	}
+	return line
+}
diff --git a/main_e2e_test.go b/main_e2e_test.go
new file mode 100644
index 0000000..0a81b56
--- /dev/null
+++ b/main_e2e_test.go
@@ -0,0 +1,102 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"runtime"
+	"strings"
+	"testing"
+
+	"gotest.tools/gotestsum/internal/text"
+	"gotest.tools/v3/assert"
+	"gotest.tools/v3/env"
+	"gotest.tools/v3/fs"
+	"gotest.tools/v3/golden"
+)
+
+func TestE2E_RerunFails(t *testing.T) {
+	type testCase struct {
+		name        string
+		args        []string
+		expectedErr bool
+	}
+	fn := func(t *testing.T, tc testCase) {
+		tmpFile := fs.NewFile(t, t.Name()+"-seedfile", fs.WithContent("0"))
+		defer tmpFile.Remove()
+
+		envVars := osEnviron()
+		envVars["TEST_SEEDFILE"] = tmpFile.Path()
+		defer env.PatchAll(t, envVars)()
+
+		flags, opts := setupFlags("gotestsum")
+		assert.NilError(t, flags.Parse(tc.args))
+		opts.args = flags.Args()
+
+		bufStdout := new(bytes.Buffer)
+		opts.stdout = bufStdout
+		bufStderr := new(bytes.Buffer)
+		opts.stderr = bufStderr
+
+		err := run(opts)
+		if tc.expectedErr {
+			assert.Error(t, err, "exit status 1")
+		} else {
+			assert.NilError(t, err)
+		}
+		out := text.ProcessLines(t, bufStdout,
+			text.OpRemoveSummaryLineElapsedTime,
+			text.OpRemoveTestElapsedTime)
+		golden.Assert(t, out, expectedFilename(t.Name()))
+		assert.Equal(t, bufStderr.String(), "")
+	}
+	var testCases = []testCase{
+		{
+			name: "reruns until success",
+			args: []string{
+				"-f=testname",
+				"--rerun-fails=4",
+				"--packages=./testdata/e2e/flaky/",
+				"--", "-count=1", "-tags=testdata",
+			},
+		},
+		{
+			name: "reruns continues to fail",
+			args: []string{
+				"-f=testname",
+				"--rerun-fails=2",
+				"--packages=./testdata/e2e/flaky/",
+				"--", "-count=1", "-tags=testdata",
+			},
+			expectedErr: true,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			if testing.Short() {
+				t.Skip("too slow for short run")
+			}
+			fn(t, tc)
+		})
+	}
+}
+
+// osEnviron returns os.Environ() as a map, with any GOTESTSUM_ env vars removed
+// so that they do not alter the test results.
+func osEnviron() map[string]string {
+	e := env.ToMap(os.Environ())
+	for k := range e {
+		if strings.HasPrefix(k, "GOTESTSUM_") {
+			delete(e, k)
+		}
+	}
+	return e
+}
+
+func expectedFilename(name string) string {
+	// go1.14 changed how it prints messages from tests. It may be changed back
+	// in go1.15, so special case this version for now.
+	if strings.HasPrefix(runtime.Version(), "go1.14.") {
+		name = name + "-go1.14"
+	}
+	return "e2e/expected/" + name
+}
diff --git a/main_test.go b/main_test.go
new file mode 100644
index 0000000..9e1b5d2
--- /dev/null
+++ b/main_test.go
@@ -0,0 +1,288 @@
+package main
+
+import (
+	"bytes"
+	"testing"
+
+	"gotest.tools/v3/assert"
+	"gotest.tools/v3/env"
+	"gotest.tools/v3/golden"
+)
+
+func TestUsage_WithFlagsFromSetupFlags(t *testing.T) {
+	defer env.PatchAll(t, nil)()
+
+	name := "gotestsum"
+	flags, _ := setupFlags(name)
+	buf := new(bytes.Buffer)
+	usage(buf, name, flags)
+
+	golden.Assert(t, buf.String(), "gotestsum-help-text")
+}
+
+func TestOptions_Validate_FromFlags(t *testing.T) {
+	type testCase struct {
+		name     string
+		args     []string
+		expected string
+	}
+	fn := func(t *testing.T, tc testCase) {
+		flags, opts := setupFlags("gotestsum")
+		err := flags.Parse(tc.args)
+		assert.NilError(t, err)
+		opts.args = flags.Args()
+
+		err = opts.Validate()
+		if tc.expected == "" {
+			assert.NilError(t, err)
+			return
+		}
+		assert.ErrorContains(t, err, tc.expected, "opts: %#v", opts)
+	}
+	var testCases = []testCase{
+		{
+			name: "no flags",
+		},
+		{
+			name: "rerun flag, raw command",
+			args: []string{"--rerun-fails", "--raw-command", "--", "./test-all"},
+		},
+		{
+			name: "rerun flag, no go-test args",
+			args: []string{"--rerun-fails", "--"},
+		},
+		{
+			name:     "rerun flag, go-test args, no packages flag",
+			args:     []string{"--rerun-fails", "--", "./..."},
+			expected: "the list of packages to test must be specified by the --packages flag",
+		},
+		{
+			name: "rerun flag, go-test args, with packages flag",
+			args: []string{"--rerun-fails", "--packages", "./...", "--", "--foo"},
+		},
+		{
+			name: "rerun flag, no go-test args, with packages flag",
+			args: []string{"--rerun-fails", "--packages", "./..."},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			fn(t, tc)
+		})
+	}
+}
+
+func TestGoTestCmdArgs(t *testing.T) {
+	type testCase struct {
+		opts      *options
+		rerunOpts rerunOpts
+		env       []string
+		expected  []string
+	}
+	fn := func(t *testing.T, tc testCase) {
+		defer env.PatchAll(t, env.ToMap(tc.env))()
+		actual := goTestCmdArgs(tc.opts, tc.rerunOpts)
+		assert.DeepEqual(t, actual, tc.expected)
+	}
+	var testcases = map[string]testCase{
+		"raw command": {
+			opts: &options{
+				rawCommand: true,
+				args:       []string{"./script", "-test.timeout=20m"},
+			},
+			expected: []string{"./script", "-test.timeout=20m"},
+		},
+		"no args": {
+			opts:     &options{},
+			expected: []string{"go", "test", "-json", "./..."},
+		},
+		"no args, with rerunPackageList arg": {
+			opts: &options{
+				packages: []string{"./pkg"},
+			},
+			expected: []string{"go", "test", "-json", "./pkg"},
+		},
+		"TEST_DIRECTORY env var no args": {
+			opts:     &options{},
+			env:      []string{"TEST_DIRECTORY=testdir"},
+			expected: []string{"go", "test", "-json", "testdir"},
+		},
+		"TEST_DIRECTORY env var with args": {
+			opts: &options{
+				args: []string{"-tags=integration"},
+			},
+			env:      []string{"TEST_DIRECTORY=testdir"},
+			expected: []string{"go", "test", "-json", "-tags=integration", "testdir"},
+		},
+		"no -json arg": {
+			opts: &options{
+				args: []string{"-timeout=2m", "./pkg"},
+			},
+			expected: []string{"go", "test", "-json", "-timeout=2m", "./pkg"},
+		},
+		"with -json arg": {
+			opts: &options{
+				args: []string{"-json", "-timeout=2m", "./pkg"},
+			},
+			expected: []string{"go", "test", "-json", "-timeout=2m", "./pkg"},
+		},
+		"raw command, with rerunOpts": {
+			opts: &options{
+				rawCommand: true,
+				args:       []string{"./script", "-test.timeout=20m"},
+			},
+			rerunOpts: rerunOpts{
+				runFlag: "-run=TestOne|TestTwo",
+				pkg:     "./fails",
+			},
+			expected: []string{"./script", "-test.timeout=20m", "-run=TestOne|TestTwo", "./fails"},
+		},
+		"no args, with rerunOpts": {
+			opts: &options{},
+			rerunOpts: rerunOpts{
+				runFlag: "-run=TestOne|TestTwo",
+				pkg:     "./fails",
+			},
+			expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "./fails"},
+		},
+		"TEST_DIRECTORY env var, no args, with rerunOpts": {
+			opts: &options{},
+			rerunOpts: rerunOpts{
+				runFlag: "-run=TestOne|TestTwo",
+				pkg:     "./fails",
+			},
+			env: []string{"TEST_DIRECTORY=testdir"},
+			// TEST_DIRECTORY should be overridden by rerun opts
+			expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "./fails"},
+		},
+		"TEST_DIRECTORY env var, with args, with rerunOpts": {
+			opts: &options{
+				args: []string{"-tags=integration"},
+			},
+			rerunOpts: rerunOpts{
+				runFlag: "-run=TestOne|TestTwo",
+				pkg:     "./fails",
+			},
+			env:      []string{"TEST_DIRECTORY=testdir"},
+			expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "-tags=integration", "./fails"},
+		},
+		"no -json arg, with rerunOpts": {
+			opts: &options{
+				args:     []string{"-timeout=2m"},
+				packages: []string{"./pkg"},
+			},
+			rerunOpts: rerunOpts{
+				runFlag: "-run=TestOne|TestTwo",
+				pkg:     "./fails",
+			},
+			expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "-timeout=2m", "./fails"},
+		},
+		"with -json arg, with rerunOpts": {
+			opts: &options{
+				args:     []string{"-json", "-timeout=2m"},
+				packages: []string{"./pkg"},
+			},
+			rerunOpts: rerunOpts{
+				runFlag: "-run=TestOne|TestTwo",
+				pkg:     "./fails",
+			},
+			expected: []string{"go", "test", "-run=TestOne|TestTwo", "-json", "-timeout=2m", "./fails"},
+		},
+		"with args, with reunFailsPackageList args, with rerunOpts": {
+			opts: &options{
+				args:     []string{"-timeout=2m"},
+				packages: []string{"./pkg1", "./pkg2", "./pkg3"},
+			},
+			rerunOpts: rerunOpts{
+				runFlag: "-run=TestOne|TestTwo",
+				pkg:     "./fails",
+			},
+			expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "-timeout=2m", "./fails"},
+		},
+		"with args, with reunFailsPackageList": {
+			opts: &options{
+				args:     []string{"-timeout=2m"},
+				packages: []string{"./pkg1", "./pkg2", "./pkg3"},
+			},
+			expected: []string{"go", "test", "-json", "-timeout=2m", "./pkg1", "./pkg2", "./pkg3"},
+		},
+		"reunFailsPackageList args, with rerunOpts ": {
+			opts: &options{
+				packages: []string{"./pkg1", "./pkg2", "./pkg3"},
+			},
+			rerunOpts: rerunOpts{
+				runFlag: "-run=TestOne|TestTwo",
+				pkg:     "./fails",
+			},
+			expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "./fails"},
+		},
+		"reunFailsPackageList args, with rerunOpts, with -args ": {
+			opts: &options{
+				args:     []string{"before", "-args", "after"},
+				packages: []string{"./pkg1"},
+			},
+			rerunOpts: rerunOpts{
+				runFlag: "-run=TestOne|TestTwo",
+				pkg:     "./fails",
+			},
+			expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "before", "./fails", "-args", "after"},
+		},
+		"reunFailsPackageList args, with rerunOpts, with -args at end": {
+			opts: &options{
+				args:     []string{"before", "-args"},
+				packages: []string{"./pkg1"},
+			},
+			rerunOpts: rerunOpts{
+				runFlag: "-run=TestOne|TestTwo",
+				pkg:     "./fails",
+			},
+			expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "before", "./fails", "-args"},
+		},
+		"reunFailsPackageList args, with -args at start": {
+			opts: &options{
+				args:     []string{"-args", "after"},
+				packages: []string{"./pkg1"},
+			},
+			expected: []string{"go", "test", "-json", "./pkg1", "-args", "after"},
+		},
+		"-run arg at start, with rerunOpts ": {
+			opts: &options{
+				args:     []string{"-run=TestFoo", "-args"},
+				packages: []string{"./pkg"},
+			},
+			rerunOpts: rerunOpts{
+				runFlag: "-run=TestOne|TestTwo",
+				pkg:     "./fails",
+			},
+			expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "./fails", "-args"},
+		},
+		"-run arg in middle, with rerunOpts ": {
+			opts: &options{
+				args:     []string{"-count", "1", "--run", "TestFoo", "-args"},
+				packages: []string{"./pkg"},
+			},
+			rerunOpts: rerunOpts{
+				runFlag: "-run=TestOne|TestTwo",
+				pkg:     "./fails",
+			},
+			expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "-count", "1", "./fails", "-args"},
+		},
+		"-run arg at end with missing value, with rerunOpts ": {
+			opts: &options{
+				args:     []string{"-count", "1", "-run"},
+				packages: []string{"./pkg"},
+			},
+			rerunOpts: rerunOpts{
+				runFlag: "-run=TestOne|TestTwo",
+				pkg:     "./fails",
+			},
+			expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "-count", "1", "-run", "./fails"},
+		},
+	}
+
+	for name, tc := range testcases {
+		t.Run(name, func(t *testing.T) {
+			fn(t, tc)
+		})
+	}
+}
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail
new file mode 100644
index 0000000..c9ebffe
--- /dev/null
+++ b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail
@@ -0,0 +1,66 @@
+PASS testdata/e2e/flaky.TestAlwaysPasses
+=== RUN   TestFailsRarely
+SEED:  0
+--- FAIL: TestFailsRarely
+    flaky_test.go:51: not this time
+FAIL testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  0
+--- FAIL: TestFailsSometimes
+    flaky_test.go:58: not this time
+FAIL testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  0
+--- FAIL: TestFailsOften
+    flaky_test.go:65: not this time
+FAIL testdata/e2e/flaky.TestFailsOften
+PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
+PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  1
+--- FAIL: TestFailsSometimes
+    flaky_test.go:58: not this time
+FAIL testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  1
+--- FAIL: TestFailsOften
+    flaky_test.go:65: not this time
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  2
+--- FAIL: TestFailsOften
+    flaky_test.go:65: not this time
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+
+=== Failed
+=== FAIL: testdata/e2e/flaky TestFailsRarely
+SEED:  0
+    flaky_test.go:51: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  0
+    flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  0
+    flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  1
+    flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  1
+    flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  2
+    flaky_test.go:65: not this time
+
+
+DONE 11 tests, 6 failures
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14 b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14
new file mode 100644
index 0000000..5a6de81
--- /dev/null
+++ b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14
@@ -0,0 +1,66 @@
+PASS testdata/e2e/flaky.TestAlwaysPasses
+=== RUN   TestFailsRarely
+SEED:  0
+    TestFailsRarely: flaky_test.go:51: not this time
+--- FAIL: TestFailsRarely
+FAIL testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  0
+    TestFailsSometimes: flaky_test.go:58: not this time
+--- FAIL: TestFailsSometimes
+FAIL testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  0
+    TestFailsOften: flaky_test.go:65: not this time
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften
+PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
+PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  1
+    TestFailsSometimes: flaky_test.go:58: not this time
+--- FAIL: TestFailsSometimes
+FAIL testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  1
+    TestFailsOften: flaky_test.go:65: not this time
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  2
+    TestFailsOften: flaky_test.go:65: not this time
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+
+=== Failed
+=== FAIL: testdata/e2e/flaky TestFailsRarely
+SEED:  0
+    TestFailsRarely: flaky_test.go:51: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  0
+    TestFailsSometimes: flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  0
+    TestFailsOften: flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  1
+    TestFailsSometimes: flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  1
+    TestFailsOften: flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  2
+    TestFailsOften: flaky_test.go:65: not this time
+
+
+DONE 11 tests, 6 failures
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success
new file mode 100644
index 0000000..cecc8fb
--- /dev/null
+++ b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success
@@ -0,0 +1,78 @@
+PASS testdata/e2e/flaky.TestAlwaysPasses
+=== RUN   TestFailsRarely
+SEED:  0
+--- FAIL: TestFailsRarely
+    flaky_test.go:51: not this time
+FAIL testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  0
+--- FAIL: TestFailsSometimes
+    flaky_test.go:58: not this time
+FAIL testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  0
+--- FAIL: TestFailsOften
+    flaky_test.go:65: not this time
+FAIL testdata/e2e/flaky.TestFailsOften
+PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
+PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  1
+--- FAIL: TestFailsSometimes
+    flaky_test.go:58: not this time
+FAIL testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  1
+--- FAIL: TestFailsOften
+    flaky_test.go:65: not this time
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  2
+--- FAIL: TestFailsOften
+    flaky_test.go:65: not this time
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+=== RUN   TestFailsOften
+SEED:  3
+--- FAIL: TestFailsOften
+    flaky_test.go:65: not this time
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsOften
+PASS testdata/e2e/flaky
+
+=== Failed
+=== FAIL: testdata/e2e/flaky TestFailsRarely
+SEED:  0
+    flaky_test.go:51: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  0
+    flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  0
+    flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  1
+    flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  1
+    flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  2
+    flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  3
+    flaky_test.go:65: not this time
+
+
+DONE 13 tests, 7 failures
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.14 b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.14
new file mode 100644
index 0000000..337ba52
--- /dev/null
+++ b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.14
@@ -0,0 +1,78 @@
+PASS testdata/e2e/flaky.TestAlwaysPasses
+=== RUN   TestFailsRarely
+SEED:  0
+    TestFailsRarely: flaky_test.go:51: not this time
+--- FAIL: TestFailsRarely
+FAIL testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  0
+    TestFailsSometimes: flaky_test.go:58: not this time
+--- FAIL: TestFailsSometimes
+FAIL testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  0
+    TestFailsOften: flaky_test.go:65: not this time
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften
+PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
+PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  1
+    TestFailsSometimes: flaky_test.go:58: not this time
+--- FAIL: TestFailsSometimes
+FAIL testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  1
+    TestFailsOften: flaky_test.go:65: not this time
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  2
+    TestFailsOften: flaky_test.go:65: not this time
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+=== RUN   TestFailsOften
+SEED:  3
+    TestFailsOften: flaky_test.go:65: not this time
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsOften
+PASS testdata/e2e/flaky
+
+=== Failed
+=== FAIL: testdata/e2e/flaky TestFailsRarely
+SEED:  0
+    TestFailsRarely: flaky_test.go:51: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  0
+    TestFailsSometimes: flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  0
+    TestFailsOften: flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  1
+    TestFailsSometimes: flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  1
+    TestFailsOften: flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  2
+    TestFailsOften: flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  3
+    TestFailsOften: flaky_test.go:65: not this time
+
+
+DONE 13 tests, 7 failures
diff --git a/testdata/e2e/flaky/flaky_test.go b/testdata/e2e/flaky/flaky_test.go
new file mode 100644
index 0000000..c148624
--- /dev/null
+++ b/testdata/e2e/flaky/flaky_test.go
@@ -0,0 +1,71 @@
+// +build testdata
+
+package flaky
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"sync"
+	"testing"
+)
+
+var seed int
+var seedfile = seedFile()
+var once = new(sync.Once)
+
+func setup(t *testing.T) {
+	once.Do(func() {
+		raw, err := ioutil.ReadFile(seedfile)
+		if err != nil {
+			t.Fatalf("failed to read seed: %v", err)
+		}
+		n, err := strconv.ParseInt(string(raw), 10, 64)
+		if err != nil {
+			t.Fatalf("failed to parse seed: %v", err)
+		}
+		seed = int(n)
+
+		err = ioutil.WriteFile(seedfile, []byte(strconv.Itoa(seed+1)), 0644)
+		if err != nil {
+			t.Fatalf("failed to write seed: %v", err)
+		}
+	})
+	fmt.Fprintln(os.Stderr, "SEED: ", seed)
+}
+
+func seedFile() string {
+	if name, ok := os.LookupEnv("TEST_SEEDFILE"); ok {
+		return name
+	}
+	return "/tmp/gotestsum-flaky-seedfile"
+}
+
+func TestAlwaysPasses(t *testing.T) {
+}
+
+func TestFailsRarely(t *testing.T) {
+	setup(t)
+	if seed%10 != 1 {
+		t.Fatal("not this time")
+	}
+}
+
+func TestFailsSometimes(t *testing.T) {
+	setup(t)
+	if seed%10 != 2 {
+		t.Fatal("not this time")
+	}
+}
+
+func TestFailsOften(t *testing.T) {
+	setup(t)
+	if seed%10 != 4 {
+		t.Fatal("not this time")
+	}
+}
+
+func TestFailsOftenDoesNotPrefixMatch(t *testing.T) {}
+
+func TestFailsSometimesDoesNotPrefixMatch(t *testing.T) {}
diff --git a/testdata/event-handler-missing-test-fail-expected b/testdata/event-handler-missing-test-fail-expected
new file mode 100644
index 0000000..3b60ed3
--- /dev/null
+++ b/testdata/event-handler-missing-test-fail-expected
@@ -0,0 +1,20 @@
+FAIL gotest.tools/v3/poll
+=== RUN   TestWaitOn_WithCompare
+panic: runtime error: index out of range [1] with length 1
+
+goroutine 7 [running]:
+gotest.tools/v3/internal/assert.ArgsFromComparisonCall(0xc0000552a0, 0x1, 0x1, 0x1, 0x0, 0x0)
+	/home/daniel/pers/code/gotest.tools/internal/assert/result.go:102 +0x9f
+gotest.tools/v3/internal/assert.runComparison(0x6bcb80, 0xc00000e180, 0x67dee8, 0xc00007a9f0, 0x0, 0x0, 0x0, 0x7f7f4fb6d108)
+	/home/daniel/pers/code/gotest.tools/internal/assert/result.go:34 +0x2b1
+gotest.tools/v3/internal/assert.Eval(0x6bcb80, 0xc00000e180, 0x67dee8, 0x627660, 0xc00007a9f0, 0x0, 0x0, 0x0, 0x642c60)
+	/home/daniel/pers/code/gotest.tools/internal/assert/assert.go:56 +0x2e4
+gotest.tools/v3/poll.Compare(0xc00007a9f0, 0x6b74a0, 0x618a60)
+	/home/daniel/pers/code/gotest.tools/poll/poll.go:151 +0x81
+gotest.tools/v3/poll.TestWaitOn_WithCompare.func1(0x6be4c0, 0xc00016c240, 0xc00016c240, 0x6be4c0)
+	/home/daniel/pers/code/gotest.tools/poll/poll_test.go:81 +0x58
+gotest.tools/v3/poll.WaitOn.func1(0xc00001e3c0, 0x67df50, 0x6c1960, 0xc00016c240)
+	/home/daniel/pers/code/gotest.tools/poll/poll.go:125 +0x62
+created by gotest.tools/v3/poll.WaitOn
+	/home/daniel/pers/code/gotest.tools/poll/poll.go:124 +0x16f
+FAIL gotest.tools/v3/poll.TestWaitOn_WithCompare (-1.00s)
diff --git a/testdata/gotestsum-help-text b/testdata/gotestsum-help-text
new file mode 100644
index 0000000..71ae28e
--- /dev/null
+++ b/testdata/gotestsum-help-text
@@ -0,0 +1,31 @@
+Usage:
+    gotestsum [flags] [--] [go test flags]
+    gotestsum [command]
+
+Flags:
+      --debug                                       enabled debug logging
+  -f, --format string                               print format of test input (default "short")
+      --jsonfile string                             write all TestEvents to file
+      --junitfile string                            write a JUnit XML file
+      --junitfile-testcase-classname field-format   format the testcase classname field as: full, relative, short (default full)
+      --junitfile-testsuite-name field-format       format the testsuite name field as: full, relative, short (default full)
+      --no-color                                    disable color output (default true)
+      --no-summary summary                          do not print summary of: skipped,failed,errors,output (default none)
+      --packages list                               space separated list of package to test
+      --post-run-command command                    command to run after the tests have completed
+      --raw-command                                 don't prepend 'go test -json' to the 'go test' command
+      --rerun-fails int[=2]                         rerun failed tests until they all pass, or attempts exceeds maximum. Defaults to max 2 reruns when enabled.
+      --rerun-fails-max-failures int                do not rerun any tests if the initial run has more than this number of failures (default 10)
+      --version                                     show version and exit
+
+Formats:
+    dots                    print a character for each test
+    dots-v2                 experimental dots format, one package per line
+    pkgname                 print a line for each package
+    pkgname-and-test-fails  print a line for each package and failed test output
+    testname                print a line for each test and package
+    standard-quiet          standard go test format
+    standard-verbose        standard go test -v format
+
+Commands:
+    tool                    tools for working with test2json output
diff --git a/testjson/dotformat_test.go b/testjson/dotformat_test.go
index e7569dc..a6b6964 100644
--- a/testjson/dotformat_test.go
+++ b/testjson/dotformat_test.go
@@ -1,18 +1,16 @@
 package testjson
 
 import (
-	"bufio"
 	"bytes"
-	"io"
 	"math/rand"
 	"runtime"
-	"strings"
 	"testing"
 	"testing/quick"
 	"time"
 	"unicode/utf8"
 
 	"gotest.tools/gotestsum/internal/dotwriter"
+	"gotest.tools/gotestsum/internal/text"
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/assert/cmp"
 	"gotest.tools/v3/golden"
@@ -32,7 +30,7 @@ func TestScanTestOutput_WithDotsFormatter(t *testing.T) {
 	exec, err := ScanTestOutput(shim.Config(t))
 	assert.NilError(t, err)
 
-	actual := removeSummaryTime(t, out)
+	actual := text.ProcessLines(t, out, text.OpRemoveSummaryLineElapsedTime)
 	golden.Assert(t, actual, outFile("dots-format"))
 	golden.Assert(t, shim.err.String(), "dots-format.err")
 	assert.DeepEqual(t, exec, expectedExecution, cmpExecutionShallow)
@@ -45,22 +43,6 @@ func outFile(name string) string {
 	return name + ".out"
 }
 
-func removeSummaryTime(t *testing.T, r io.Reader) string {
-	t.Helper()
-	out := new(strings.Builder)
-	scan := bufio.NewScanner(r)
-	for scan.Scan() {
-		line := scan.Text()
-		if i := strings.Index(line, " in "); i > 0 {
-			out.WriteString(line[:i] + "\n")
-			continue
-		}
-		out.WriteString(line + "\n")
-	}
-	assert.NilError(t, scan.Err())
-	return out.String()
-}
-
 func TestFmtDotElapsed(t *testing.T) {
 	var testcases = []struct {
 		cached   bool
diff --git a/testjson/execution.go b/testjson/execution.go
index e245f7f..468dd2a 100644
--- a/testjson/execution.go
+++ b/testjson/execution.go
@@ -200,12 +200,25 @@ func (p *Package) TestMainFailed() bool {
 
 const neverFinished time.Duration = -1
 
-func (p *Package) end() {
-	// Add tests that were missing an ActionFail event to Failed
+// end adds any tests that were missing an ActionFail TestEvent to the list of
+// Failed, and returns a slice of artificial TestEvent for the missing ones.
+//
+// This is done to work around 'go test' not sending the ActionFail TestEvents
+// in some cases, when a test panics.
+func (p *Package) end() []TestEvent {
+	result := make([]TestEvent, 0, len(p.running))
 	for _, tc := range p.running {
 		tc.Elapsed = neverFinished
 		p.Failed = append(p.Failed, tc)
+
+		result = append(result, TestEvent{
+			Action:  ActionFail,
+			Package: tc.Package,
+			Test:    tc.Test,
+			Elapsed: float64(neverFinished),
+		})
 	}
+	return result
 }
 
 // TestCase stores the name and elapsed time for a test case.
@@ -371,6 +384,9 @@ func (e *Execution) Elapsed() time.Duration {
 
 // Failed returns a list of all the failed test cases.
 func (e *Execution) Failed() []TestCase {
+	if e == nil {
+		return nil
+	}
 	var failed []TestCase //nolint:prealloc
 	for _, name := range sortedKeys(e.packages) {
 		pkg := e.packages[name]
@@ -424,11 +440,13 @@ func (e *Execution) Errors() []string {
 	return e.errors
 }
 
-func (e *Execution) end() {
+func (e *Execution) end() []TestEvent {
 	e.done = true
+	var result []TestEvent // nolint: prealloc
 	for _, pkg := range e.packages {
-		pkg.end()
+		result = append(result, pkg.end()...)
 	}
+	return result
 }
 
 // newExecution returns a new Execution and records the current time as the
@@ -449,6 +467,9 @@ type ScanConfig struct {
 	Stderr io.Reader
 	// Handler is a set of callbacks for receiving TestEvents and stderr text.
 	Handler EventHandler
+	// Execution to populate while scanning. If nil a new one will be created
+	// and returned from ScanTestOutput.
+	Execution *Execution
 }
 
 // EventHandler is called by ScanTestOutput for each event and write to stderr.
@@ -461,21 +482,24 @@ type EventHandler interface {
 	Err(text string) error
 }
 
-// ScanTestOutput reads lines from config.Stdout and config.Stderr, creates an
+// ScanTestOutput reads lines from config.Stdout and config.Stderr, populates an
 // Execution, calls the Handler for each event, and returns the Execution.
 //
 // If config.Handler is nil, a default no-op handler will be used.
 func ScanTestOutput(config ScanConfig) (*Execution, error) {
+	if config.Stdout == nil {
+		return nil, fmt.Errorf("stdout reader must be non-nil")
+	}
 	if config.Handler == nil {
 		config.Handler = noopHandler{}
 	}
 	if config.Stderr == nil {
 		config.Stderr = new(bytes.Reader)
 	}
-	if config.Stdout == nil {
-		return nil, fmt.Errorf("stdout reader must be non-nil")
+	execution := config.Execution
+	if execution == nil {
+		execution = newExecution()
 	}
-	execution := newExecution()
 	var group errgroup.Group
 	group.Go(func() error {
 		return readStdout(config, execution)
@@ -483,9 +507,16 @@ func ScanTestOutput(config ScanConfig) (*Execution, error) {
 	group.Go(func() error {
 		return readStderr(config, execution)
 	})
-	err := group.Wait()
-	execution.end()
-	return execution, err
+	if err := group.Wait(); err != nil {
+		return execution, err
+	}
+	for _, event := range execution.end() {
+		if err := config.Handler.Event(event, execution); err != nil {
+			return execution, err
+		}
+	}
+
+	return execution, nil
 }
 
 func readStdout(config ScanConfig, execution *Execution) error {