Repository: gotestyourself/gotestsum
Pull request #134 (2020-06-21 00:52:44)

Rerun fails report
The report can be used in a CI job to report which tests flaked, so that flaky tests are still visible.


Patch:
diff --git a/.golangci.yml b/.golangci.yml
index 4a7ef42..2663423 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -1,6 +1,6 @@
 linters-settings:
   gocyclo:
-    min-complexity: 10
+    min-complexity: 12
   goconst:
     min-len: 2
     min-occurrences: 4
diff --git a/main.go b/main.go
index ddd4d0e..43fe7ab 100644
--- a/main.go
+++ b/main.go
@@ -107,6 +107,8 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
 		"do not rerun any tests if the initial run has more than this number of failures")
 	flags.Var((*stringSlice)(&opts.packages), "packages",
 		"space separated list of package to test")
+	flags.StringVar(&opts.rerunFailsReportFile, "rerun-fails-report", "",
+		"write a report to the file, of the tests that were rerun")
 
 	flags.BoolVar(&opts.debug, "debug", false, "enabled debug logging")
 	flags.BoolVar(&opts.version, "version", false, "show version and exit")
@@ -158,6 +160,7 @@ type options struct {
 	junitTestCaseClassnameFormat *junitFieldFormatValue
 	rerunFailsMaxAttempts        int
 	rerunFailsMaxInitialFailures int
+	rerunFailsReportFile         string
 	packages                     []string
 	version                      bool
 
@@ -223,6 +226,9 @@ func run(opts *options) error {
 	if err := writeJUnitFile(opts, exec); err != nil {
 		return err
 	}
+	if err := writeRerunFailsReport(opts, exec); err != nil {
+		return err
+	}
 	if err := postRunHook(opts, exec); err != nil {
 		return err
 	}
diff --git a/rerunfails.go b/rerunfails.go
index 5844171..c9d512c 100644
--- a/rerunfails.go
+++ b/rerunfails.go
@@ -3,6 +3,8 @@ package main
 import (
 	"context"
 	"fmt"
+	"os"
+	"sort"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -129,3 +131,53 @@ func goTestRunFlagFromTestCases(tcs []string) string {
 	buf.WriteString(")$")
 	return buf.String()
 }
+
+func writeRerunFailsReport(opts *options, exec *testjson.Execution) error {
+	if opts.rerunFailsMaxAttempts == 0 || opts.rerunFailsReportFile == "" {
+		return nil
+	}
+
+	type testCaseCounts struct {
+		total  int
+		failed int
+	}
+
+	names := []string{}
+	results := map[string]testCaseCounts{}
+	for _, failure := range exec.Failed() {
+		name := failure.Package + "." + failure.Test
+		if _, ok := results[name]; ok {
+			continue
+		}
+		names = append(names, name)
+
+		pkg := exec.Package(failure.Package)
+		counts := testCaseCounts{}
+
+		for _, tc := range pkg.Failed {
+			if tc.Test == failure.Test {
+				counts.total++
+				counts.failed++
+			}
+		}
+		for _, tc := range pkg.Passed {
+			if tc.Test == failure.Test {
+				counts.total++
+			}
+		}
+		// Skipped tests are not counted, but presumably skipped tests can not fail
+		results[name] = counts
+	}
+
+	fh, err := os.Create(opts.rerunFailsReportFile)
+	if err != nil {
+		return err
+	}
+
+	sort.Strings(names)
+	for _, name := range names {
+		counts := results[name]
+		fmt.Fprintf(fh, "%s: %d runs, %d failures\n", name, counts.total, counts.failed)
+	}
+	return nil
+}


Test patch:
diff --git a/rerunfails_test.go b/rerunfails_test.go
new file mode 100644
index 0000000..69a2ef5
--- /dev/null
+++ b/rerunfails_test.go
@@ -0,0 +1,34 @@
+package main
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"gotest.tools/gotestsum/testjson"
+	"gotest.tools/v3/assert"
+	"gotest.tools/v3/fs"
+	"gotest.tools/v3/golden"
+)
+
+func TestWriteRerunFailsReport(t *testing.T) {
+	reportFile := fs.NewFile(t, t.Name())
+	defer reportFile.Remove()
+
+	opts := &options{
+		rerunFailsReportFile:  reportFile.Path(),
+		rerunFailsMaxAttempts: 4,
+	}
+
+	exec, err := testjson.ScanTestOutput(testjson.ScanConfig{
+		Stdout: bytes.NewReader(golden.Get(t, "go-test-json-flaky-rerun.out")),
+	})
+	assert.NilError(t, err)
+
+	err = writeRerunFailsReport(opts, exec)
+	assert.NilError(t, err)
+
+	raw, err := ioutil.ReadFile(reportFile.Path())
+	assert.NilError(t, err)
+	golden.Assert(t, string(raw), t.Name()+"-expected")
+}
diff --git a/testdata/TestWriteRerunFailsReport-expected b/testdata/TestWriteRerunFailsReport-expected
new file mode 100644
index 0000000..04e5fc9
--- /dev/null
+++ b/testdata/TestWriteRerunFailsReport-expected
@@ -0,0 +1,3 @@
+gotest.tools/gotestsum/testdata/e2e/flaky.TestFailsOften: 4 runs, 3 failures
+gotest.tools/gotestsum/testdata/e2e/flaky.TestFailsRarely: 2 runs, 1 failures
+gotest.tools/gotestsum/testdata/e2e/flaky.TestFailsSometimes: 3 runs, 2 failures
diff --git a/testdata/go-test-json-flaky-rerun.out b/testdata/go-test-json-flaky-rerun.out
new file mode 100644
index 0000000..2216325
--- /dev/null
+++ b/testdata/go-test-json-flaky-rerun.out
@@ -0,0 +1,75 @@
+{"Time":"2020-06-21T21:12:10.815884042-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestAlwaysPasses"}
+{"Time":"2020-06-21T21:12:10.816009964-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestAlwaysPasses","Output":"=== RUN   TestAlwaysPasses\n"}
+{"Time":"2020-06-21T21:12:10.816024167-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestAlwaysPasses","Output":"--- PASS: TestAlwaysPasses (0.00s)\n"}
+{"Time":"2020-06-21T21:12:10.81603019-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestAlwaysPasses","Elapsed":0}
+{"Time":"2020-06-21T21:12:10.81604135-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely"}
+{"Time":"2020-06-21T21:12:10.816045457-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"=== RUN   TestFailsRarely\n"}
+{"Time":"2020-06-21T21:12:10.816049373-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"SEED:  0\n"}
+{"Time":"2020-06-21T21:12:10.816063218-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"    TestFailsRarely: flaky_test.go:51: not this time\n"}
+{"Time":"2020-06-21T21:12:10.816068373-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"--- FAIL: TestFailsRarely (0.00s)\n"}
+{"Time":"2020-06-21T21:12:10.816072877-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Elapsed":0}
+{"Time":"2020-06-21T21:12:10.81607599-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes"}
+{"Time":"2020-06-21T21:12:10.81607897-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"=== RUN   TestFailsSometimes\n"}
+{"Time":"2020-06-21T21:12:10.816082575-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"SEED:  0\n"}
+{"Time":"2020-06-21T21:12:10.816086165-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"    TestFailsSometimes: flaky_test.go:58: not this time\n"}
+{"Time":"2020-06-21T21:12:10.8160903-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"--- FAIL: TestFailsSometimes (0.00s)\n"}
+{"Time":"2020-06-21T21:12:10.816094667-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Elapsed":0}
+{"Time":"2020-06-21T21:12:10.816098149-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften"}
+{"Time":"2020-06-21T21:12:10.816101281-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"=== RUN   TestFailsOften\n"}
+{"Time":"2020-06-21T21:12:10.81610482-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"SEED:  0\n"}
+{"Time":"2020-06-21T21:12:10.816108597-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"    TestFailsOften: flaky_test.go:65: not this time\n"}
+{"Time":"2020-06-21T21:12:10.816112698-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"--- FAIL: TestFailsOften (0.00s)\n"}
+{"Time":"2020-06-21T21:12:10.816116235-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Elapsed":0}
+{"Time":"2020-06-21T21:12:10.816119536-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOftenDoesNotPrefixMatch"}
+{"Time":"2020-06-21T21:12:10.816122935-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOftenDoesNotPrefixMatch","Output":"=== RUN   TestFailsOftenDoesNotPrefixMatch\n"}
+{"Time":"2020-06-21T21:12:10.816127081-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOftenDoesNotPrefixMatch","Output":"--- PASS: TestFailsOftenDoesNotPrefixMatch (0.00s)\n"}
+{"Time":"2020-06-21T21:12:10.816131578-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOftenDoesNotPrefixMatch","Elapsed":0}
+{"Time":"2020-06-21T21:12:10.816135262-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimesDoesNotPrefixMatch"}
+{"Time":"2020-06-21T21:12:10.816139844-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimesDoesNotPrefixMatch","Output":"=== RUN   TestFailsSometimesDoesNotPrefixMatch\n"}
+{"Time":"2020-06-21T21:12:10.816149774-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimesDoesNotPrefixMatch","Output":"--- PASS: TestFailsSometimesDoesNotPrefixMatch (0.00s)\n"}
+{"Time":"2020-06-21T21:12:10.816155804-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimesDoesNotPrefixMatch","Elapsed":0}
+{"Time":"2020-06-21T21:12:10.816159612-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\n"}
+{"Time":"2020-06-21T21:12:10.816211999-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\tgotest.tools/gotestsum/testdata/e2e/flaky\t0.001s\n"}
+{"Time":"2020-06-21T21:12:10.816224311-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Elapsed":0.001}
+{"Time":"2020-06-21T21:12:10.985779906-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely"}
+{"Time":"2020-06-21T21:12:10.985890459-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"=== RUN   TestFailsRarely\n"}
+{"Time":"2020-06-21T21:12:10.985902826-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"SEED:  1\n"}
+{"Time":"2020-06-21T21:12:10.985911982-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"--- PASS: TestFailsRarely (0.00s)\n"}
+{"Time":"2020-06-21T21:12:10.985916034-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Elapsed":0}
+{"Time":"2020-06-21T21:12:10.985923087-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes"}
+{"Time":"2020-06-21T21:12:10.985926869-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"=== RUN   TestFailsSometimes\n"}
+{"Time":"2020-06-21T21:12:10.985930857-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"SEED:  1\n"}
+{"Time":"2020-06-21T21:12:10.985934726-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"    TestFailsSometimes: flaky_test.go:58: not this time\n"}
+{"Time":"2020-06-21T21:12:10.985939499-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"--- FAIL: TestFailsSometimes (0.00s)\n"}
+{"Time":"2020-06-21T21:12:10.985943642-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Elapsed":0}
+{"Time":"2020-06-21T21:12:10.985945782-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften"}
+{"Time":"2020-06-21T21:12:10.985947733-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"=== RUN   TestFailsOften\n"}
+{"Time":"2020-06-21T21:12:10.985949865-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"SEED:  1\n"}
+{"Time":"2020-06-21T21:12:10.985952001-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"    TestFailsOften: flaky_test.go:65: not this time\n"}
+{"Time":"2020-06-21T21:12:10.985954479-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"--- FAIL: TestFailsOften (0.00s)\n"}
+{"Time":"2020-06-21T21:12:10.985956607-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Elapsed":0}
+{"Time":"2020-06-21T21:12:10.985958719-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\n"}
+{"Time":"2020-06-21T21:12:10.986048472-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\tgotest.tools/gotestsum/testdata/e2e/flaky\t0.001s\n"}
+{"Time":"2020-06-21T21:12:10.986068802-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Elapsed":0.001}
+{"Time":"2020-06-21T21:12:11.147554767-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes"}
+{"Time":"2020-06-21T21:12:11.147676606-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"=== RUN   TestFailsSometimes\n"}
+{"Time":"2020-06-21T21:12:11.14768948-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"SEED:  2\n"}
+{"Time":"2020-06-21T21:12:11.147700275-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"--- PASS: TestFailsSometimes (0.00s)\n"}
+{"Time":"2020-06-21T21:12:11.147705201-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Elapsed":0}
+{"Time":"2020-06-21T21:12:11.147710442-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften"}
+{"Time":"2020-06-21T21:12:11.147712827-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"=== RUN   TestFailsOften\n"}
+{"Time":"2020-06-21T21:12:11.147715318-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"SEED:  2\n"}
+{"Time":"2020-06-21T21:12:11.147717797-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"    TestFailsOften: flaky_test.go:65: not this time\n"}
+{"Time":"2020-06-21T21:12:11.147721095-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"--- FAIL: TestFailsOften (0.00s)\n"}
+{"Time":"2020-06-21T21:12:11.147723329-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Elapsed":0}
+{"Time":"2020-06-21T21:12:11.147725838-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\n"}
+{"Time":"2020-06-21T21:12:11.14783256-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\tgotest.tools/gotestsum/testdata/e2e/flaky\t0.001s\n"}
+{"Time":"2020-06-21T21:12:11.147849384-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Elapsed":0.001}
+{"Time":"2020-06-21T21:12:11.226137617-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften"}
+{"Time":"2020-06-21T21:12:11.226247164-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"=== RUN   TestFailsOften\n"}
+{"Time":"2020-06-21T21:12:11.226256816-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"SEED:  14\n"}
+{"Time":"2020-06-21T21:12:11.226264297-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"--- PASS: TestFailsOften (0.00s)\n"}
+{"Time":"2020-06-21T21:12:11.226266905-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Elapsed":0}
+{"Time":"2020-06-21T21:12:11.22627268-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"PASS\n"}
+{"Time":"2020-06-21T21:12:11.226275849-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"ok  \tgotest.tools/gotestsum/testdata/e2e/flaky\t(cached)\n"}
+{"Time":"2020-06-21T21:12:11.226279592-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Elapsed":0}
diff --git a/testdata/gotestsum-help-text b/testdata/gotestsum-help-text
index 71ae28e..07c6b95 100644
--- a/testdata/gotestsum-help-text
+++ b/testdata/gotestsum-help-text
@@ -16,6 +16,7 @@ Flags:
       --raw-command                                 don't prepend 'go test -json' to the 'go test' command
       --rerun-fails int[=2]                         rerun failed tests until they all pass, or attempts exceeds maximum. Defaults to max 2 reruns when enabled.
       --rerun-fails-max-failures int                do not rerun any tests if the initial run has more than this number of failures (default 10)
+      --rerun-fails-report string                   write a report to the file, of the tests that were rerun
       --version                                     show version and exit
 
 Formats:
diff --git a/testjson/execution.go b/testjson/execution.go
index da37815..71f0a43 100644
--- a/testjson/execution.go
+++ b/testjson/execution.go
@@ -285,7 +285,6 @@ func (e *Execution) addPackageEvent(pkg *Package, event TestEvent) {
 	}
 }
 
-// nolint: gocyclo
 func (p *Package) addTestEvent(event TestEvent) {
 	tc := p.running[event.Test]
 	root, subTest := splitTestName(event.Test)