Repository: gotestyourself/gotestsum
Pull request #135 (2020-06-28 21:56:54)

rerun-fails: Only rerun failed subtests
If a root test case has failed subtests, only rerun those that have failed. A new hidden flag is added to restore the old behaviour of re-running the entire root test case if any subtest fails.

Because of limitations of the 'go test -run' flag (https://github.com/golang/go/issues/39904) this required running each test in a separate 'go test' process.

Also fixes a recently introduced bug where any incomplete tests stayed in `Execution.running`, which caused `Execution.end()` to produce incorrect artificial events for those  tests on each re-run.


Patch:
diff --git a/.goreleaser.yml b/.goreleaser.yml
index 7e8097a..0c5d8cd 100644
--- a/.goreleaser.yml
+++ b/.goreleaser.yml
@@ -12,9 +12,7 @@ builds:
       - darwin
       - windows
       - linux
-    goarch:
-      - amd64
-      - arm64
+    goarch: [amd64]
     env: [CGO_ENABLED=0]
 
 checksum:
diff --git a/do b/do
index 503afd7..ed26229 100755
--- a/do
+++ b/do
@@ -7,13 +7,24 @@ binary() {
     go build -o dist/gotestsum .
 }
 
+binary-static() {
+    echo "building static binary: dist/gotestsum"
+    CGO_ENABLED=0 binary
+}
+
 update-golden() {
-    _update-golden
-    GOLANG_VERSION=1.13-alpine ./do shell bash -c 'go build; PATH=$PATH:. ./do _update-golden'
+    #_update-golden
+    if ldd ./dist/gotestsum > /dev/null 2>&1; then
+        binary-static
+    fi
+    GOLANG_VERSION=1.13-alpine ./do shell ./do _update-golden
+    GOLANG_VERSION=1.14.6-alpine ./do shell ./do _update-golden
 }
 
 _update-golden() {
-    gotestsum -- . ./testjson ./internal/junitxml ./cmd/tool/slowest -test.update-golden
+    PATH="$PWD/dist:$PATH" gotestsum -- \
+        . ./testjson ./internal/junitxml ./cmd/tool/slowest \
+        -test.update-golden
 }
 
 lint() {
diff --git a/main.go b/main.go
index 43fe7ab..c63c9bb 100644
--- a/main.go
+++ b/main.go
@@ -109,6 +109,9 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
 		"space separated list of package to test")
 	flags.StringVar(&opts.rerunFailsReportFile, "rerun-fails-report", "",
 		"write a report to the file, of the tests that were rerun")
+	flags.BoolVar(&opts.rerunFailsOnlyRootCases, "rerun-fails-only-root-testcases", false,
+		"rerun only root testcaes, instead of only subtests")
+	flags.Lookup("rerun-fails-only-root-testcases").Hidden = true
 
 	flags.BoolVar(&opts.debug, "debug", false, "enabled debug logging")
 	flags.BoolVar(&opts.version, "version", false, "show version and exit")
@@ -161,6 +164,7 @@ type options struct {
 	rerunFailsMaxAttempts        int
 	rerunFailsMaxInitialFailures int
 	rerunFailsReportFile         string
+	rerunFailsOnlyRootCases      bool
 	packages                     []string
 	version                      bool
 
diff --git a/rerunfails.go b/rerunfails.go
index c9d512c..0c06a0d 100644
--- a/rerunfails.go
+++ b/rerunfails.go
@@ -27,8 +27,27 @@ func (o rerunOpts) Args() []string {
 	return result
 }
 
+func newRerunOptsFromTestCase(tc testjson.TestCase) rerunOpts {
+	return rerunOpts{
+		runFlag: goTestRunFlagForTestCase(tc.Test),
+		pkg:     tc.Package,
+	}
+}
+
+type testCaseFilter func([]testjson.TestCase) []testjson.TestCase
+
+func rerunFailsFilter(o *options) testCaseFilter {
+	if o.rerunFailsOnlyRootCases {
+		return func(tcs []testjson.TestCase) []testjson.TestCase {
+			return tcs
+		}
+	}
+	return testjson.FilterFailedUnique
+}
+
 func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanConfig) error {
-	failed := len(scanConfig.Execution.Failed())
+	tcFilter := rerunFailsFilter(opts)
+	failed := len(tcFilter(scanConfig.Execution.Failed()))
 	if failed > opts.rerunFailsMaxInitialFailures {
 		return fmt.Errorf(
 			"number of test failures (%d) exceeds maximum (%d) set by --rerun-fails-max-failures",
@@ -42,12 +61,8 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
 		opts.stdout.Write([]byte("\n")) // nolint: errcheck
 
 		nextRec := newFailureRecorder(scanConfig.Handler)
-		for pkg, testCases := range rec.pkgFailures {
-			rerun := rerunOpts{
-				runFlag: goTestRunFlagFromTestCases(testCases),
-				pkg:     pkg,
-			}
-			goTestProc, err := startGoTest(ctx, goTestCmdArgs(opts, rerun))
+		for _, tc := range tcFilter(rec.failures) {
+			goTestProc, err := startGoTest(ctx, goTestCmdArgs(opts, newRerunOptsFromTestCase(tc)))
 			if err != nil {
 				return errors.Wrapf(err, "failed to run %s", strings.Join(goTestProc.cmd.Args, " "))
 			}
@@ -66,8 +81,8 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
 			if err := hasErrors(lastErr, scanConfig.Execution); err != nil {
 				return err
 			}
-			rec = nextRec
 		}
+		rec = nextRec
 	}
 	return lastErr
 }
@@ -86,50 +101,36 @@ func hasErrors(err error, exec *testjson.Execution) error {
 
 type failureRecorder struct {
 	testjson.EventHandler
-	pkgFailures map[string][]string
+	failures []testjson.TestCase
 }
 
 func newFailureRecorder(handler testjson.EventHandler) *failureRecorder {
-	return &failureRecorder{
-		EventHandler: handler,
-		pkgFailures:  make(map[string][]string),
-	}
+	return &failureRecorder{EventHandler: handler}
 }
 
 func newFailureRecorderFromExecution(exec *testjson.Execution) *failureRecorder {
-	r := newFailureRecorder(nil)
-	for _, tc := range exec.Failed() {
-		r.pkgFailures[tc.Package] = append(r.pkgFailures[tc.Package], tc.Test)
-	}
-	return r
+	return &failureRecorder{failures: exec.Failed()}
 }
 
 func (r *failureRecorder) Event(event testjson.TestEvent, execution *testjson.Execution) error {
 	if !event.PackageEvent() && event.Action == testjson.ActionFail {
-		r.pkgFailures[event.Package] = append(r.pkgFailures[event.Package], event.Test)
+		pkg := execution.Package(event.Package)
+		tc := pkg.LastFailedByName(event.Test)
+		r.failures = append(r.failures, tc)
 	}
 	return r.EventHandler.Event(event, execution)
 }
 
 func (r *failureRecorder) count() int {
-	total := 0
-	for _, tcs := range r.pkgFailures {
-		total += len(tcs)
-	}
-	return total
+	return len(r.failures)
 }
 
-func goTestRunFlagFromTestCases(tcs []string) string {
-	buf := new(strings.Builder)
-	buf.WriteString("-run=^(")
-	for i, tc := range tcs {
-		if i != 0 {
-			buf.WriteString("|")
-		}
-		buf.WriteString(tc)
+func goTestRunFlagForTestCase(name string) string {
+	root, sub := testjson.SplitTestName(name)
+	if sub == "" {
+		return "-run=^" + name + "$"
 	}
-	buf.WriteString(")$")
-	return buf.String()
+	return "-run=^" + root + "$/^" + sub + "$"
 }
 
 func writeRerunFailsReport(opts *options, exec *testjson.Execution) error {


Test patch:
diff --git a/main_e2e_test.go b/main_e2e_test.go
index e9859af..976221e 100644
--- a/main_e2e_test.go
+++ b/main_e2e_test.go
@@ -49,7 +49,7 @@ func TestE2E_RerunFails(t *testing.T) {
 			text.OpRemoveTestElapsedTime,
 			filepath.ToSlash, // for windows
 		)
-		golden.Assert(t, out, expectedFilename(t.Name()))
+		golden.Assert(t, out, "e2e/expected/"+expectedFilename(t.Name()))
 	}
 	var testCases = []testCase{
 		{
@@ -105,10 +105,26 @@ func osEnviron() map[string]string {
 }
 
 func expectedFilename(name string) string {
-	// go1.14 changed how it prints messages from tests. It may be changed back
-	// in go1.15, so special case this version for now.
-	if strings.HasPrefix(runtime.Version(), "go1.14.") {
-		name = name + "-go1.14"
+	ver := runtime.Version()
+	switch {
+	case isPreGo114(ver):
+		return name + "-go1.13"
+	default:
+		return name
 	}
-	return "e2e/expected/" + name
+}
+
+// go1.14.6 changed how it prints messages from tests. go1.14.{0-5} used a format
+// that was different from both go1.14.6 and previous versions of Go. These tests
+// no longer support that format.
+func isPreGo114(ver string) bool {
+	prefix := "go1.1"
+	if !strings.HasPrefix(ver, prefix) || len(ver) < len(prefix)+1 {
+		return false
+	}
+	switch ver[len(prefix)] {
+	case '0', '1', '2', '3':
+		return true
+	}
+	return false
 }
diff --git a/rerunfails_test.go b/rerunfails_test.go
index 69a2ef5..8fc6f55 100644
--- a/rerunfails_test.go
+++ b/rerunfails_test.go
@@ -32,3 +32,30 @@ func TestWriteRerunFailsReport(t *testing.T) {
 	assert.NilError(t, err)
 	golden.Assert(t, string(raw), t.Name()+"-expected")
 }
+
+func TestGoTestRunFlagFromTestCases(t *testing.T) {
+	type testCase struct {
+		input    string
+		expected string
+	}
+	fn := func(t *testing.T, tc testCase) {
+		actual := goTestRunFlagForTestCase(tc.input)
+		assert.Equal(t, actual, tc.expected)
+	}
+
+	var testCases = map[string]testCase{
+		"root test case": {
+			input:    "TestOne",
+			expected: "-run=^TestOne$",
+		},
+		"sub test case": {
+			input:    "TestOne/SubtestA",
+			expected: "-run=^TestOne$/^SubtestA$",
+		},
+	}
+	for name := range testCases {
+		t.Run(name, func(t *testing.T) {
+			fn(t, testCases[name])
+		})
+	}
+}
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun-go1.14 b/testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun-go1.13
similarity index 100%
rename from testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun-go1.14
rename to testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun-go1.13
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail
index ed4d4a8..ff479b2 100644
--- a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail
+++ b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail
@@ -1,45 +1,52 @@
 PASS testdata/e2e/flaky.TestAlwaysPasses
 === RUN   TestFailsRarely
 SEED:  0
---- FAIL: TestFailsRarely
     flaky_test.go:51: not this time
+--- FAIL: TestFailsRarely
 FAIL testdata/e2e/flaky.TestFailsRarely
 === RUN   TestFailsSometimes
 SEED:  0
---- FAIL: TestFailsSometimes
     flaky_test.go:58: not this time
+--- FAIL: TestFailsSometimes
 FAIL testdata/e2e/flaky.TestFailsSometimes
+PASS testdata/e2e/flaky.TestFailsOften/subtest_always_passes
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail
 === RUN   TestFailsOften
 SEED:  0
 --- FAIL: TestFailsOften
-    flaky_test.go:65: not this time
 FAIL testdata/e2e/flaky.TestFailsOften
 PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
 PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
 FAIL testdata/e2e/flaky
 
-DONE 6 tests, 3 failures
+DONE 8 tests, 4 failures
 
 PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
-=== RUN   TestFailsSometimes
-SEED:  1
---- FAIL: TestFailsSometimes
-    flaky_test.go:58: not this time
-FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1)
+PASS testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsSometimes (re-run 1)
+PASS testdata/e2e/flaky
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1)
 === RUN   TestFailsOften
-SEED:  1
+SEED:  3
 --- FAIL: TestFailsOften
-    flaky_test.go:65: not this time
 FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
 FAIL testdata/e2e/flaky
 
-DONE 2 runs, 9 tests, 5 failures
+DONE 2 runs, 12 tests, 6 failures
 
-PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2)
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2)
 === RUN   TestFailsOften
-SEED:  2
+SEED:  4
 --- FAIL: TestFailsOften
-    flaky_test.go:65: not this time
 FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
 FAIL testdata/e2e/flaky
 
@@ -52,21 +59,26 @@ SEED:  0
 SEED:  0
     flaky_test.go:58: not this time
 
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+
 === FAIL: testdata/e2e/flaky TestFailsOften
 SEED:  0
-    flaky_test.go:65: not this time
 
-=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1)
-SEED:  1
-    flaky_test.go:58: not this time
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1)
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
 
 === FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
-SEED:  1
-    flaky_test.go:65: not this time
+SEED:  3
+
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2)
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
 
 === FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
-SEED:  2
-    flaky_test.go:65: not this time
+SEED:  4
 
 
-DONE 3 runs, 11 tests, 6 failures
+DONE 3 runs, 14 tests, 8 failures
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.13 b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.13
new file mode 100644
index 0000000..b968136
--- /dev/null
+++ b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.13
@@ -0,0 +1,84 @@
+PASS testdata/e2e/flaky.TestAlwaysPasses
+=== RUN   TestFailsRarely
+SEED:  0
+--- FAIL: TestFailsRarely
+    flaky_test.go:51: not this time
+FAIL testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  0
+--- FAIL: TestFailsSometimes
+    flaky_test.go:58: not this time
+FAIL testdata/e2e/flaky.TestFailsSometimes
+PASS testdata/e2e/flaky.TestFailsOften/subtest_always_passes
+=== RUN   TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail
+=== RUN   TestFailsOften
+SEED:  0
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften
+PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
+PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
+FAIL testdata/e2e/flaky
+
+DONE 8 tests, 4 failures
+
+PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
+PASS testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsSometimes (re-run 1)
+PASS testdata/e2e/flaky
+=== RUN   TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1)
+=== RUN   TestFailsOften
+SEED:  3
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
+FAIL testdata/e2e/flaky
+
+DONE 2 runs, 12 tests, 6 failures
+
+=== RUN   TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2)
+=== RUN   TestFailsOften
+SEED:  4
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
+FAIL testdata/e2e/flaky
+
+=== Failed
+=== FAIL: testdata/e2e/flaky TestFailsRarely
+SEED:  0
+    flaky_test.go:51: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  0
+    flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  0
+
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1)
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
+SEED:  3
+
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2)
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
+SEED:  4
+
+
+DONE 3 runs, 14 tests, 8 failures
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14 b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14
deleted file mode 100644
index 4334fc6..0000000
--- a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14
+++ /dev/null
@@ -1,72 +0,0 @@
-PASS testdata/e2e/flaky.TestAlwaysPasses
-=== RUN   TestFailsRarely
-SEED:  0
-    TestFailsRarely: flaky_test.go:51: not this time
---- FAIL: TestFailsRarely
-FAIL testdata/e2e/flaky.TestFailsRarely
-=== RUN   TestFailsSometimes
-SEED:  0
-    TestFailsSometimes: flaky_test.go:58: not this time
---- FAIL: TestFailsSometimes
-FAIL testdata/e2e/flaky.TestFailsSometimes
-=== RUN   TestFailsOften
-SEED:  0
-    TestFailsOften: flaky_test.go:65: not this time
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften
-PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
-PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
-FAIL testdata/e2e/flaky
-
-DONE 6 tests, 3 failures
-
-PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
-=== RUN   TestFailsSometimes
-SEED:  1
-    TestFailsSometimes: flaky_test.go:58: not this time
---- FAIL: TestFailsSometimes
-FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1)
-=== RUN   TestFailsOften
-SEED:  1
-    TestFailsOften: flaky_test.go:65: not this time
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
-FAIL testdata/e2e/flaky
-
-DONE 2 runs, 9 tests, 5 failures
-
-PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2)
-=== RUN   TestFailsOften
-SEED:  2
-    TestFailsOften: flaky_test.go:65: not this time
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
-FAIL testdata/e2e/flaky
-
-=== Failed
-=== FAIL: testdata/e2e/flaky TestFailsRarely
-SEED:  0
-    TestFailsRarely: flaky_test.go:51: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsSometimes
-SEED:  0
-    TestFailsSometimes: flaky_test.go:58: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften
-SEED:  0
-    TestFailsOften: flaky_test.go:65: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1)
-SEED:  1
-    TestFailsSometimes: flaky_test.go:58: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
-SEED:  1
-    TestFailsOften: flaky_test.go:65: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
-SEED:  2
-    TestFailsOften: flaky_test.go:65: not this time
-
-
-DONE 3 runs, 11 tests, 6 failures
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success
index ebe537e..9d68fce 100644
--- a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success
+++ b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success
@@ -1,59 +1,70 @@
 PASS testdata/e2e/flaky.TestAlwaysPasses
 === RUN   TestFailsRarely
 SEED:  0
---- FAIL: TestFailsRarely
     flaky_test.go:51: not this time
+--- FAIL: TestFailsRarely
 FAIL testdata/e2e/flaky.TestFailsRarely
 === RUN   TestFailsSometimes
 SEED:  0
---- FAIL: TestFailsSometimes
     flaky_test.go:58: not this time
+--- FAIL: TestFailsSometimes
 FAIL testdata/e2e/flaky.TestFailsSometimes
+PASS testdata/e2e/flaky.TestFailsOften/subtest_always_passes
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail
 === RUN   TestFailsOften
 SEED:  0
 --- FAIL: TestFailsOften
-    flaky_test.go:65: not this time
 FAIL testdata/e2e/flaky.TestFailsOften
 PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
 PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
 FAIL testdata/e2e/flaky
 
-DONE 6 tests, 3 failures
+DONE 8 tests, 4 failures
 
 PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
-=== RUN   TestFailsSometimes
-SEED:  1
---- FAIL: TestFailsSometimes
-    flaky_test.go:58: not this time
-FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1)
+PASS testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsSometimes (re-run 1)
+PASS testdata/e2e/flaky
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1)
 === RUN   TestFailsOften
-SEED:  1
+SEED:  3
 --- FAIL: TestFailsOften
-    flaky_test.go:65: not this time
 FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
 FAIL testdata/e2e/flaky
 
-DONE 2 runs, 9 tests, 5 failures
+DONE 2 runs, 12 tests, 6 failures
 
-PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2)
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2)
 === RUN   TestFailsOften
-SEED:  2
+SEED:  4
 --- FAIL: TestFailsOften
-    flaky_test.go:65: not this time
 FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
 FAIL testdata/e2e/flaky
 
-DONE 3 runs, 11 tests, 6 failures
+DONE 3 runs, 14 tests, 8 failures
 
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 3)
 === RUN   TestFailsOften
-SEED:  3
+SEED:  5
 --- FAIL: TestFailsOften
-    flaky_test.go:65: not this time
 FAIL testdata/e2e/flaky.TestFailsOften (re-run 3)
 FAIL testdata/e2e/flaky
 
-DONE 4 runs, 12 tests, 7 failures
+DONE 4 runs, 16 tests, 10 failures
 
+PASS testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 4)
 PASS testdata/e2e/flaky.TestFailsOften (re-run 4)
 PASS testdata/e2e/flaky
 
@@ -66,25 +77,33 @@ SEED:  0
 SEED:  0
     flaky_test.go:58: not this time
 
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+
 === FAIL: testdata/e2e/flaky TestFailsOften
 SEED:  0
-    flaky_test.go:65: not this time
 
-=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1)
-SEED:  1
-    flaky_test.go:58: not this time
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1)
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
 
 === FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
-SEED:  1
-    flaky_test.go:65: not this time
+SEED:  3
+
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2)
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
 
 === FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
-SEED:  2
-    flaky_test.go:65: not this time
+SEED:  4
+
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 3)
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
 
 === FAIL: testdata/e2e/flaky TestFailsOften (re-run 3)
-SEED:  3
-    flaky_test.go:65: not this time
+SEED:  5
 
 
-DONE 5 runs, 13 tests, 7 failures
+DONE 5 runs, 18 tests, 10 failures
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.13 b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.13
new file mode 100644
index 0000000..dd6e90c
--- /dev/null
+++ b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.13
@@ -0,0 +1,109 @@
+PASS testdata/e2e/flaky.TestAlwaysPasses
+=== RUN   TestFailsRarely
+SEED:  0
+--- FAIL: TestFailsRarely
+    flaky_test.go:51: not this time
+FAIL testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  0
+--- FAIL: TestFailsSometimes
+    flaky_test.go:58: not this time
+FAIL testdata/e2e/flaky.TestFailsSometimes
+PASS testdata/e2e/flaky.TestFailsOften/subtest_always_passes
+=== RUN   TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail
+=== RUN   TestFailsOften
+SEED:  0
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften
+PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
+PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
+FAIL testdata/e2e/flaky
+
+DONE 8 tests, 4 failures
+
+PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
+PASS testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsSometimes (re-run 1)
+PASS testdata/e2e/flaky
+=== RUN   TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1)
+=== RUN   TestFailsOften
+SEED:  3
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
+FAIL testdata/e2e/flaky
+
+DONE 2 runs, 12 tests, 6 failures
+
+=== RUN   TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2)
+=== RUN   TestFailsOften
+SEED:  4
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
+FAIL testdata/e2e/flaky
+
+DONE 3 runs, 14 tests, 8 failures
+
+=== RUN   TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 3)
+=== RUN   TestFailsOften
+SEED:  5
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften (re-run 3)
+FAIL testdata/e2e/flaky
+
+DONE 4 runs, 16 tests, 10 failures
+
+PASS testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 4)
+PASS testdata/e2e/flaky.TestFailsOften (re-run 4)
+PASS testdata/e2e/flaky
+
+=== Failed
+=== FAIL: testdata/e2e/flaky TestFailsRarely
+SEED:  0
+    flaky_test.go:51: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  0
+    flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  0
+
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1)
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
+SEED:  3
+
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2)
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
+SEED:  4
+
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 3)
+    --- FAIL: TestFailsOften/subtest_may_fail
+        flaky_test.go:68: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 3)
+SEED:  5
+
+
+DONE 5 runs, 18 tests, 10 failures
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.14 b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.14
deleted file mode 100644
index 63e7612..0000000
--- a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.14
+++ /dev/null
@@ -1,90 +0,0 @@
-PASS testdata/e2e/flaky.TestAlwaysPasses
-=== RUN   TestFailsRarely
-SEED:  0
-    TestFailsRarely: flaky_test.go:51: not this time
---- FAIL: TestFailsRarely
-FAIL testdata/e2e/flaky.TestFailsRarely
-=== RUN   TestFailsSometimes
-SEED:  0
-    TestFailsSometimes: flaky_test.go:58: not this time
---- FAIL: TestFailsSometimes
-FAIL testdata/e2e/flaky.TestFailsSometimes
-=== RUN   TestFailsOften
-SEED:  0
-    TestFailsOften: flaky_test.go:65: not this time
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften
-PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
-PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
-FAIL testdata/e2e/flaky
-
-DONE 6 tests, 3 failures
-
-PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
-=== RUN   TestFailsSometimes
-SEED:  1
-    TestFailsSometimes: flaky_test.go:58: not this time
---- FAIL: TestFailsSometimes
-FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1)
-=== RUN   TestFailsOften
-SEED:  1
-    TestFailsOften: flaky_test.go:65: not this time
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
-FAIL testdata/e2e/flaky
-
-DONE 2 runs, 9 tests, 5 failures
-
-PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2)
-=== RUN   TestFailsOften
-SEED:  2
-    TestFailsOften: flaky_test.go:65: not this time
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
-FAIL testdata/e2e/flaky
-
-DONE 3 runs, 11 tests, 6 failures
-
-=== RUN   TestFailsOften
-SEED:  3
-    TestFailsOften: flaky_test.go:65: not this time
---- FAIL: TestFailsOften
-FAIL testdata/e2e/flaky.TestFailsOften (re-run 3)
-FAIL testdata/e2e/flaky
-
-DONE 4 runs, 12 tests, 7 failures
-
-PASS testdata/e2e/flaky.TestFailsOften (re-run 4)
-PASS testdata/e2e/flaky
-
-=== Failed
-=== FAIL: testdata/e2e/flaky TestFailsRarely
-SEED:  0
-    TestFailsRarely: flaky_test.go:51: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsSometimes
-SEED:  0
-    TestFailsSometimes: flaky_test.go:58: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften
-SEED:  0
-    TestFailsOften: flaky_test.go:65: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1)
-SEED:  1
-    TestFailsSometimes: flaky_test.go:58: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
-SEED:  1
-    TestFailsOften: flaky_test.go:65: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
-SEED:  2
-    TestFailsOften: flaky_test.go:65: not this time
-
-=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 3)
-SEED:  3
-    TestFailsOften: flaky_test.go:65: not this time
-
-
-DONE 5 runs, 13 tests, 7 failures
diff --git a/testdata/e2e/flaky/flaky_test.go b/testdata/e2e/flaky/flaky_test.go
index c148624..229fc6c 100644
--- a/testdata/e2e/flaky/flaky_test.go
+++ b/testdata/e2e/flaky/flaky_test.go
@@ -47,23 +47,27 @@ func TestAlwaysPasses(t *testing.T) {
 
 func TestFailsRarely(t *testing.T) {
 	setup(t)
-	if seed%10 != 1 {
+	if seed%20 != 1 {
 		t.Fatal("not this time")
 	}
 }
 
 func TestFailsSometimes(t *testing.T) {
 	setup(t)
-	if seed%10 != 2 {
+	if seed%4 != 2 {
 		t.Fatal("not this time")
 	}
 }
 
 func TestFailsOften(t *testing.T) {
 	setup(t)
-	if seed%10 != 4 {
-		t.Fatal("not this time")
-	}
+
+	t.Run("subtest always passes", func(t *testing.T) {})
+	t.Run("subtest may fail", func(t *testing.T) {
+		if seed%20 != 6 {
+			t.Fatal("not this time")
+		}
+	})
 }
 
 func TestFailsOftenDoesNotPrefixMatch(t *testing.T) {}
diff --git a/testjson/execution.go b/testjson/execution.go
index 71f0a43..5ce1989 100644
--- a/testjson/execution.go
+++ b/testjson/execution.go
@@ -143,7 +143,7 @@ func (p *Package) Output(id int) string {
 // then all output for every subtest under the root test is returned.
 // See https://github.com/golang/go/issues/29755.
 func (p *Package) OutputLines(tc TestCase) []string {
-	_, sub := splitTestName(tc.Test)
+	_, sub := SplitTestName(tc.Test)
 	lines := p.output[tc.ID]
 
 	// If this is a subtest, or a root test case with subtest failures the
@@ -166,8 +166,8 @@ func (p *Package) addOutput(id int, output string) {
 	p.output[id] = append(p.output[id], output)
 }
 
-// splitTestName into root test name and any subtest names.
-func splitTestName(name string) (root, sub string) {
+// SplitTestName into root test name and any subtest names.
+func SplitTestName(name string) (root, sub string) {
 	parts := strings.SplitN(name, "/", 2)
 	if len(parts) < 2 {
 		return name, ""
@@ -209,7 +209,7 @@ const neverFinished time.Duration = -1
 // in some cases, when a test panics.
 func (p *Package) end() []TestEvent {
 	result := make([]TestEvent, 0, len(p.running))
-	for _, tc := range p.running {
+	for k, tc := range p.running {
 		tc.Elapsed = neverFinished
 		p.Failed = append(p.Failed, tc)
 
@@ -219,6 +219,7 @@ func (p *Package) end() []TestEvent {
 			Test:    tc.Test,
 			Elapsed: float64(neverFinished),
 		})
+		delete(p.running, k)
 	}
 	return result
 }
@@ -287,7 +288,7 @@ func (e *Execution) addPackageEvent(pkg *Package, event TestEvent) {
 
 func (p *Package) addTestEvent(event TestEvent) {
 	tc := p.running[event.Test]
-	root, subTest := splitTestName(event.Test)
+	root, subTest := SplitTestName(event.Test)
 
 	switch event.Action {
 	case ActionRun:
@@ -405,6 +406,18 @@ func (e *Execution) Failed() []TestCase {
 	return failed
 }
 
+// FilterFailedUnique filters a slice of failed TestCases by removing root test
+// case that have failed subtests.
+func FilterFailedUnique(tcs []TestCase) []TestCase {
+	var result []TestCase
+	for _, tc := range tcs {
+		if !tc.hasSubTestFailed {
+			result = append(result, tc)
+		}
+	}
+	return result
+}
+
 func sortedKeys(pkgs map[string]*Package) []string {
 	keys := make([]string, 0, len(pkgs))
 	for key := range pkgs {
diff --git a/testjson/summary_test.go b/testjson/summary_test.go
index 655be34..b51cf43 100644
--- a/testjson/summary_test.go
+++ b/testjson/summary_test.go
@@ -241,6 +241,10 @@ func TestPrintSummary_MissingTestFailEvent(t *testing.T) {
 	buf := new(bytes.Buffer)
 	PrintSummary(buf, exec, SummarizeAll)
 	golden.Assert(t, buf.String(), "summary-missing-test-fail-event")
+
+	for name, pkg := range exec.packages {
+		assert.Equal(t, len(pkg.running), 0, "package %v still had tests in running", name)
+	}
 }
 
 func TestPrintSummary_WithMisattributedOutput(t *testing.T) {