Repository: gotestyourself/gotestsum
Pull request #107 (2020-05-02 20:28:03)

cmd/tool/slowest: Print or mark slowest tests
Example usage in the source.

Keep local development fast by identifying slow tests and automatically marking them with a `t.Skip` on some condition.


Patch:
diff --git a/cmd/cmd.go b/cmd/cmd.go
new file mode 100644
index 0000000..d539b29
--- /dev/null
+++ b/cmd/cmd.go
@@ -0,0 +1,13 @@
+package cmd
+
+// Next splits args into the next positional argument and any remaining args.
+func Next(args []string) (string, []string) {
+	switch len(args) {
+	case 0:
+		return "", nil
+	case 1:
+		return args[0], nil
+	default:
+		return args[0], args[1:]
+	}
+}
diff --git a/cmd/tool/cmd.go b/cmd/tool/cmd.go
new file mode 100644
index 0000000..1c4c146
--- /dev/null
+++ b/cmd/tool/cmd.go
@@ -0,0 +1,33 @@
+package tool
+
+import (
+	"fmt"
+	"os"
+
+	"gotest.tools/gotestsum/cmd"
+	"gotest.tools/gotestsum/cmd/tool/slowest"
+)
+
+// Run one of the tool commands.
+func Run(name string, args []string) error {
+	next, rest := cmd.Next(args)
+	switch next {
+	case "":
+		fmt.Println(usage(name))
+		return nil
+	case "slowest":
+		return slowest.Run(name+" "+next, rest)
+	default:
+		fmt.Fprintln(os.Stderr, usage(name))
+		return fmt.Errorf("invalid command: %v %v", name, next)
+	}
+}
+
+func usage(name string) string {
+	return fmt.Sprintf(`Usage: %s COMMAND [flags]
+
+Commands: slowest
+
+Use '%s COMMAND --help' for command specific help.
+`, name, name)
+}
diff --git a/cmd/tool/slowest/ast.go b/cmd/tool/slowest/ast.go
new file mode 100644
index 0000000..c4bcca5
--- /dev/null
+++ b/cmd/tool/slowest/ast.go
@@ -0,0 +1,178 @@
+package slowest
+
+import (
+	"fmt"
+	"go/ast"
+	"go/format"
+	"go/parser"
+	"go/token"
+	"os"
+	"strings"
+
+	"golang.org/x/tools/go/packages"
+	"gotest.tools/gotestsum/log"
+	"gotest.tools/gotestsum/testjson"
+)
+
+func writeTestSkip(tcs []testjson.TestCase, skipStmt ast.Stmt) error {
+	fset := token.NewFileSet()
+	cfg := packages.Config{
+		Mode:       modeAll(),
+		Tests:      true,
+		Fset:       fset,
+		BuildFlags: buildFlags(),
+	}
+	pkgNames, index := testNamesByPkgName(tcs)
+	pkgs, err := packages.Load(&cfg, pkgNames...)
+	if err != nil {
+		return fmt.Errorf("failed to load packages: %v", err)
+	}
+
+	for _, pkg := range pkgs {
+		if len(pkg.Errors) > 0 {
+			return errPkgLoad(pkg)
+		}
+		tcs, ok := index[normalizePkgName(pkg.PkgPath)]
+		if !ok {
+			log.Debugf("skipping %v, no slow tests", pkg.PkgPath)
+			continue
+		}
+
+		log.Debugf("rewriting %v for %d test cases", pkg.PkgPath, len(tcs))
+		for _, file := range pkg.Syntax {
+			path := fset.File(file.Pos()).Name()
+			log.Debugf("looking for test cases in: %v", path)
+			if !rewriteAST(file, tcs, skipStmt) {
+				continue
+			}
+			if err := writeFile(path, file, fset); err != nil {
+				return fmt.Errorf("failed to write ast to file %v: %v", path, err)
+			}
+		}
+	}
+	return errTestCasesNotFound(index)
+}
+
+// normalizePkgName removes the _test suffix from a package name. External test
+// packages (those named package_test) may contain tests, but the test2json output
+// always uses the non-external package name. The _test suffix must be removed
+// so that any slow tests in an external test package can be found.
+func normalizePkgName(name string) string {
+	return strings.TrimSuffix(name, "_test")
+}
+
+func writeFile(path string, file *ast.File, fset *token.FileSet) error {
+	fh, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := fh.Close(); err != nil {
+			log.Errorf("Failed to close file %v: %v", path, err)
+		}
+	}()
+	return format.Node(fh, fset, file)
+}
+
+func parseSkipStatement(text string) (ast.Stmt, error) {
+	switch text {
+	case "default", "testing.Short":
+		text = `
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+`
+	}
+	// Add some required boilerplate around the statement to make it a valid file
+	text = "package stub\nfunc Stub() {\n" + text + "\n}\n"
+	file, err := parser.ParseFile(token.NewFileSet(), "fragment", text, 0)
+	if err != nil {
+		return nil, err
+	}
+	stmt := file.Decls[0].(*ast.FuncDecl).Body.List[0]
+	return stmt, nil
+}
+
+func rewriteAST(file *ast.File, testNames set, skipStmt ast.Stmt) bool {
+	var modified bool
+	for _, decl := range file.Decls {
+		fd, ok := decl.(*ast.FuncDecl)
+		if !ok {
+			continue
+		}
+		name := fd.Name.Name // TODO: can this be nil?
+		if _, ok := testNames[name]; !ok {
+			continue
+		}
+
+		fd.Body.List = append([]ast.Stmt{skipStmt}, fd.Body.List...)
+		modified = true
+		delete(testNames, name)
+	}
+	return modified
+}
+
+type set map[string]struct{}
+
+// testNamesByPkgName removes subtests from the list of TestCases, then builds
+// and returns a slice of all the packages names, and a mapping of package name
+// to set of failed tests in that package.
+//
+// subtests are removed because the AST lookup currently only works for top-level
+// functions, not t.Run subtests.
+func testNamesByPkgName(tcs []testjson.TestCase) ([]string, map[string]set) {
+	var pkgs []string
+	index := make(map[string]set)
+	for _, tc := range tcs {
+		if isSubTest(tc.Test) {
+			continue
+		}
+		if len(index[tc.Package]) == 0 {
+			pkgs = append(pkgs, tc.Package)
+			index[tc.Package] = make(map[string]struct{})
+		}
+		index[tc.Package][tc.Test] = struct{}{}
+	}
+	return pkgs, index
+}
+
+func isSubTest(name string) bool {
+	return strings.Contains(name, "/")
+}
+
+func errPkgLoad(pkg *packages.Package) error {
+	buf := new(strings.Builder)
+	for _, err := range pkg.Errors {
+		buf.WriteString("\n" + err.Error())
+	}
+	return fmt.Errorf("failed to load package %v %v", pkg.PkgPath, buf.String())
+}
+
+func errTestCasesNotFound(index map[string]set) error {
+	var missed []string
+	for pkg, tcs := range index {
+		for tc := range tcs {
+			missed = append(missed, fmt.Sprintf("%v.%v", pkg, tc))
+		}
+	}
+	if len(missed) == 0 {
+		return nil
+	}
+	return fmt.Errorf("failed to find source for test cases:\n%v", strings.Join(missed, "\n"))
+}
+
+func modeAll() packages.LoadMode {
+	mode := packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles
+	mode = mode | packages.NeedImports | packages.NeedDeps
+	mode = mode | packages.NeedTypes | packages.NeedTypesSizes
+	mode = mode | packages.NeedSyntax | packages.NeedTypesInfo
+	return mode
+}
+
+func buildFlags() []string {
+	flags := os.Getenv("GOFLAGS")
+	if len(flags) == 0 {
+		return nil
+	}
+	return strings.Split(os.Getenv("GOFLAGS"), " ")
+}
diff --git a/cmd/tool/slowest/slowest.go b/cmd/tool/slowest/slowest.go
new file mode 100644
index 0000000..6d66561
--- /dev/null
+++ b/cmd/tool/slowest/slowest.go
@@ -0,0 +1,200 @@
+package slowest
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"sort"
+	"time"
+
+	"github.com/spf13/pflag"
+	"gotest.tools/gotestsum/log"
+	"gotest.tools/gotestsum/testjson"
+)
+
+// Run the command
+func Run(name string, args []string) error {
+	flags, opts := setupFlags(name)
+	switch err := flags.Parse(args); {
+	case err == pflag.ErrHelp:
+		return nil
+	case err != nil:
+		flags.Usage()
+		return err
+	}
+	return run(opts)
+}
+
+func setupFlags(name string) (*pflag.FlagSet, *options) {
+	opts := &options{}
+	flags := pflag.NewFlagSet(name, pflag.ContinueOnError)
+	flags.SetInterspersed(false)
+	flags.Usage = func() {
+		fmt.Fprintf(os.Stderr, `Usage:
+    %s [flags]
+
+Read a json file and print or update tests which are slower than threshold.
+The json file can be created with 'gotestsum --jsonfile' or 'go test -json'.
+
+By default this command will print the list of tests slower than threshold to stdout.
+If --skip-stmt is set, instead of printing the list of stdout, the AST for the
+Go source code in the working directory tree will be modified. The --skip-stmt
+will be added to Go test files as the first statement in all the test functions
+which are slower than threshold.
+
+The --skip-stmt flag may be set to the name of a predefine statement, or a
+source code which will be parsed as a go/ast.Stmt. Currently there is only one
+predefined statement: testing.Short:
+
+    if testing.Short() {
+        t.Skip("too slow for testing.Short")
+    }
+
+Example - using a custom --skip-stmt:
+
+    skip_stmt='
+        if os.Getenv("TEST_FAST") {
+            t.Skip("too slow for TEST_FAST")
+        }
+    '
+    go test -json -short ./... | %s --skip-stmt "$skip_stmt"
+
+Note that this tool does not add imports, so using a custom statement may require
+you to add any necessary imports to the file.
+
+Go build flags, such as build tags, may be set using the GOFLAGS environment
+variable, following the same rules as the go toolchain. See
+https://golang.org/cmd/go/#hdr-Environment_variables.
+
+Flags:
+`, name, name)
+		flags.PrintDefaults()
+	}
+	flags.StringVar(&opts.jsonfile, "jsonfile", "",
+		"path to test2json output, defaults to stdin")
+	flags.DurationVar(&opts.threshold, "threshold", 100*time.Millisecond,
+		"tests faster than this threshold will be omitted from the output")
+	flags.StringVar(&opts.skipStatement, "skip-stmt", "",
+		"add this go statement to slow tests, instead of printing the list of slow tests")
+	flags.BoolVar(&opts.debug, "debug", false,
+		"enable debug logging.")
+	return flags, opts
+}
+
+type options struct {
+	threshold     time.Duration
+	jsonfile      string
+	skipStatement string
+	debug         bool
+}
+
+func run(opts *options) error {
+	if opts.debug {
+		log.SetLevel(log.DebugLevel)
+	}
+	in, err := jsonfileReader(opts.jsonfile)
+	if err != nil {
+		return fmt.Errorf("failed to read jsonfile: %v", err)
+	}
+	defer func() {
+		if err := in.Close(); err != nil {
+			log.Errorf("Failed to close file %v: %v", opts.jsonfile, err)
+		}
+	}()
+
+	exec, err := testjson.ScanTestOutput(testjson.ScanConfig{
+		Stdout: in,
+		Stderr: bytes.NewReader(nil),
+	})
+	if err != nil {
+		return fmt.Errorf("failed to scan testjson: %v", err)
+	}
+
+	tcs := slowTestCases(exec, opts.threshold)
+	if opts.skipStatement != "" {
+		skipStmt, err := parseSkipStatement(opts.skipStatement)
+		if err != nil {
+			return fmt.Errorf("failed to parse skip expr: %v", err)
+		}
+		return writeTestSkip(tcs, skipStmt)
+	}
+	for _, tc := range tcs {
+		fmt.Printf("%s %s %v\n", tc.Package, tc.Test, tc.Elapsed)
+	}
+
+	return nil
+}
+
+// slowTestCases returns a slice of all tests with an elapsed time greater than
+// threshold. The slice is sorted by Elapsed time in descending order (slowest
+// test first).
+//
+// If there are multiple runs of a TestCase, all of them will be represented
+// by a single TestCase with the median elapsed time in the returned slice.
+func slowTestCases(exec *testjson.Execution, threshold time.Duration) []testjson.TestCase {
+	if threshold == 0 {
+		return nil
+	}
+	pkgs := exec.Packages()
+	tests := make([]testjson.TestCase, 0, len(pkgs))
+	for _, pkg := range pkgs {
+		pkgTests := aggregateTestCases(exec.Package(pkg).TestCases())
+		tests = append(tests, pkgTests...)
+	}
+	sort.Slice(tests, func(i, j int) bool {
+		return tests[i].Elapsed > tests[j].Elapsed
+	})
+	end := sort.Search(len(tests), func(i int) bool {
+		return tests[i].Elapsed < threshold
+	})
+	return tests[:end]
+}
+
+// collectTestCases maps all test cases by name, and if there is more than one
+// instance of a TestCase, finds the median elapsed time for all the runs.
+//
+// All cases are assumed to be part of the same package.
+func aggregateTestCases(cases []testjson.TestCase) []testjson.TestCase {
+	if len(cases) < 2 {
+		return cases
+	}
+	pkg := cases[0].Package
+	// nolint: prealloc // size is not predictable
+	m := make(map[string][]time.Duration)
+	for _, tc := range cases {
+		m[tc.Test] = append(m[tc.Test], tc.Elapsed)
+	}
+	result := make([]testjson.TestCase, 0, len(m))
+	for name, timing := range m {
+		result = append(result, testjson.TestCase{
+			Package: pkg,
+			Test:    name,
+			Elapsed: median(timing),
+		})
+	}
+	return result
+}
+
+func median(times []time.Duration) time.Duration {
+	switch len(times) {
+	case 0:
+		return 0
+	case 1:
+		return times[0]
+	}
+	sort.Slice(times, func(i, j int) bool {
+		return times[i] < times[j]
+	})
+	return times[len(times)/2]
+}
+
+func jsonfileReader(v string) (io.ReadCloser, error) {
+	switch v {
+	case "", "-":
+		return ioutil.NopCloser(os.Stdin), nil
+	default:
+		return os.Open(v)
+	}
+}
diff --git a/go.mod b/go.mod
index ca284ba..d79b516 100644
--- a/go.mod
+++ b/go.mod
@@ -10,6 +10,8 @@ require (
 	github.com/spf13/pflag v1.0.3
 	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
+	golang.org/x/sys v0.0.0-20190422165155-953cdadca894 // indirect
+	golang.org/x/tools v0.0.0-20190624222133-a101b041ded4
 	gotest.tools/v3 v3.0.2
 )
 
diff --git a/go.sum b/go.sum
index d94a752..819acb6 100644
--- a/go.sum
+++ b/go.sum
@@ -20,8 +20,11 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEha
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4 h1:1mMox4TgefDwqluYCv677yNXwlfTkija4owZve/jr78=
 golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
diff --git a/log/log.go b/log/log.go
index 7f4824c..3f4a935 100644
--- a/log/log.go
+++ b/log/log.go
@@ -58,3 +58,13 @@ func Errorf(format string, args ...interface{}) {
 	out.WriteString(fmt.Sprintf(format, args...))
 	out.WriteString("\n")
 }
+
+// Error prints the message to stderr, with a red ERROR prefix.
+func Error(msg string) {
+	if level < ErrorLevel {
+		return
+	}
+	out.WriteString(color.RedString("ERROR "))
+	out.WriteString(msg)
+	out.WriteString("\n")
+}
diff --git a/main.go b/main.go
index 843d9e9..cc4d2b9 100644
--- a/main.go
+++ b/main.go
@@ -11,6 +11,8 @@ import (
 	"github.com/fatih/color"
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
+	"gotest.tools/gotestsum/cmd"
+	"gotest.tools/gotestsum/cmd/tool"
 	"gotest.tools/gotestsum/log"
 	"gotest.tools/gotestsum/testjson"
 )
@@ -18,34 +20,48 @@ import (
 var version = "master"
 
 func main() {
-	name := os.Args[0]
+	err := route(os.Args)
+	switch err.(type) {
+	case nil:
+		return
+	case *exec.ExitError:
+		// go test should already report the error to stderr, exit with
+		// the same status code
+		os.Exit(ExitCodeWithDefault(err))
+	default:
+		log.Error(err.Error())
+		os.Exit(3)
+	}
+}
+
+func route(args []string) error {
+	name := args[0]
+	next, rest := cmd.Next(args[1:])
+	switch next {
+	case "tool":
+		return tool.Run(name+" "+next, rest)
+	default:
+		return runMain(name, args[1:])
+	}
+}
+
+func runMain(name string, args []string) error {
 	flags, opts := setupFlags(name)
-	switch err := flags.Parse(os.Args[1:]); {
+	switch err := flags.Parse(args); {
 	case err == pflag.ErrHelp:
-		os.Exit(0)
+		return nil
 	case err != nil:
-		log.Errorf(err.Error())
 		flags.Usage()
-		os.Exit(1)
+		return err
 	}
 	opts.args = flags.Args()
 	setupLogging(opts)
 
 	if opts.version {
 		fmt.Fprintf(os.Stdout, "gotestsum version %s\n", version)
-		os.Exit(0)
-	}
-
-	switch err := run(opts).(type) {
-	case nil:
-	case *exec.ExitError:
-		// go test should already report the error to stderr so just exit with
-		// the same status code
-		os.Exit(ExitCodeWithDefault(err))
-	default:
-		log.Errorf(err.Error())
-		os.Exit(3)
+		return nil
 	}
+	return run(opts)
 }
 
 func setupFlags(name string) (*pflag.FlagSet, *options) {


Test patch:
diff --git a/cmd/tool/slowest/ast_test.go b/cmd/tool/slowest/ast_test.go
new file mode 100644
index 0000000..2d58b9a
--- /dev/null
+++ b/cmd/tool/slowest/ast_test.go
@@ -0,0 +1,22 @@
+package slowest
+
+import (
+	"bytes"
+	"go/format"
+	"go/token"
+	"testing"
+
+	"gotest.tools/v3/assert"
+)
+
+func TestParseSkipStatement_Preset_testingShort(t *testing.T) {
+	stmt, err := parseSkipStatement("testing.Short")
+	assert.NilError(t, err)
+	expected := `if testing.Short() {
+	t.Skip("too slow for testing.Short")
+}`
+	buf := new(bytes.Buffer)
+	err = format.Node(buf, token.NewFileSet(), stmt)
+	assert.NilError(t, err)
+	assert.DeepEqual(t, buf.String(), expected)
+}
diff --git a/cmd/tool/slowest/slowest_test.go b/cmd/tool/slowest/slowest_test.go
new file mode 100644
index 0000000..5f26263
--- /dev/null
+++ b/cmd/tool/slowest/slowest_test.go
@@ -0,0 +1,63 @@
+package slowest
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"gotest.tools/gotestsum/testjson"
+	"gotest.tools/v3/assert"
+)
+
+func TestAggregateTestCases(t *testing.T) {
+	cases := []testjson.TestCase{
+		{Test: "TestOne", Package: "pkg", Elapsed: time.Second},
+		{Test: "TestTwo", Package: "pkg", Elapsed: 2 * time.Second},
+		{Test: "TestOne", Package: "pkg", Elapsed: 3 * time.Second},
+		{Test: "TestTwo", Package: "pkg", Elapsed: 4 * time.Second},
+		{Test: "TestOne", Package: "pkg", Elapsed: 5 * time.Second},
+		{Test: "TestTwo", Package: "pkg", Elapsed: 6 * time.Second},
+	}
+	actual := aggregateTestCases(cases)
+	expected := []testjson.TestCase{
+		{Test: "TestOne", Package: "pkg", Elapsed: 3 * time.Second},
+		{Test: "TestTwo", Package: "pkg", Elapsed: 4 * time.Second},
+	}
+	assert.DeepEqual(t, actual, expected,
+		cmpopts.SortSlices(func(x, y testjson.TestCase) bool {
+			return strings.Compare(x.Test, y.Test) == -1
+		}),
+		cmpopts.IgnoreUnexported(testjson.TestCase{}))
+}
+
+func TestMedian(t *testing.T) {
+	var testcases = []struct {
+		name     string
+		times    []time.Duration
+		expected time.Duration
+	}{
+		{
+			name:     "one item slice",
+			times:    []time.Duration{time.Minute},
+			expected: time.Minute,
+		},
+		{
+			name:     "odd number of items",
+			times:    []time.Duration{time.Millisecond, time.Hour, time.Second},
+			expected: time.Second,
+		},
+		{
+			name:     "even number of items",
+			times:    []time.Duration{time.Second, time.Millisecond, time.Microsecond, time.Hour},
+			expected: time.Second,
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			actual := median(tc.times)
+			assert.Equal(t, actual, tc.expected)
+		})
+	}
+}
diff --git a/internal/junitxml/report_test.go b/internal/junitxml/report_test.go
index 679f78a..d1240dd 100644
--- a/internal/junitxml/report_test.go
+++ b/internal/junitxml/report_test.go
@@ -26,9 +26,8 @@ func TestWrite(t *testing.T) {
 
 func createExecution(t *testing.T) *testjson.Execution {
 	exec, err := testjson.ScanTestOutput(testjson.ScanConfig{
-		Stdout:  readTestData(t, "out"),
-		Stderr:  readTestData(t, "err"),
-		Handler: &noopHandler{},
+		Stdout: readTestData(t, "out"),
+		Stderr: readTestData(t, "err"),
 	})
 	assert.NilError(t, err)
 	return exec
@@ -40,16 +39,6 @@ func readTestData(t *testing.T, stream string) io.Reader {
 	return bytes.NewReader(raw)
 }
 
-type noopHandler struct{}
-
-func (s *noopHandler) Event(testjson.TestEvent, *testjson.Execution) error {
-	return nil
-}
-
-func (s *noopHandler) Err(string) error {
-	return nil
-}
-
 func TestGoVersion(t *testing.T) {
 	t.Run("unknown", func(t *testing.T) {
 		defer env.Patch(t, "PATH", "/bogus")()
diff --git a/testjson/execution.go b/testjson/execution.go
index acb21c4..f82e951 100644
--- a/testjson/execution.go
+++ b/testjson/execution.go
@@ -404,7 +404,12 @@ type EventHandler interface {
 
 // ScanTestOutput reads lines from stdout and stderr, creates an Execution,
 // calls the Handler for each event, and returns the Execution.
+//
+// If config.Handler is nil, a default no-op handler will be used.
 func ScanTestOutput(config ScanConfig) (*Execution, error) {
+	if config.Handler == nil {
+		config.Handler = noopHandler{}
+	}
 	execution := NewExecution()
 	var group errgroup.Group
 	group.Go(func() error {
@@ -473,7 +478,7 @@ func isGoModuleOutput(scannerText string) bool {
 func parseEvent(raw []byte) (TestEvent, error) {
 	// TODO: this seems to be a bug in the `go test -json` output
 	if bytes.HasPrefix(raw, []byte("FAIL")) {
-		log.Warnf(string(raw))
+		log.Warnf("invalid TestEvent: %v", string(raw))
 		return TestEvent{}, errBadEvent
 	}
 
@@ -484,3 +489,13 @@ func parseEvent(raw []byte) (TestEvent, error) {
 }
 
 var errBadEvent = errors.New("bad output from test2json")
+
+type noopHandler struct{}
+
+func (s noopHandler) Event(TestEvent, *Execution) error {
+	return nil
+}
+
+func (s noopHandler) Err(string) error {
+	return nil
+}
diff --git a/testjson/summary_test.go b/testjson/summary_test.go
index 3b3ce51..c117bfe 100644
--- a/testjson/summary_test.go
+++ b/testjson/summary_test.go
@@ -231,9 +231,8 @@ func TestPrintSummary_MissingTestFailEvent(t *testing.T) {
 	defer reset()
 
 	exec, err := ScanTestOutput(ScanConfig{
-		Stdout:  bytes.NewReader(golden.Get(t, "go-test-json-missing-test-fail.out")),
-		Stderr:  bytes.NewReader(nil),
-		Handler: noopHandler{},
+		Stdout: bytes.NewReader(golden.Get(t, "go-test-json-missing-test-fail.out")),
+		Stderr: bytes.NewReader(nil),
 	})
 	assert.NilError(t, err)
 
@@ -242,24 +241,13 @@ func TestPrintSummary_MissingTestFailEvent(t *testing.T) {
 	golden.Assert(t, buf.String(), "summary-missing-test-fail-event")
 }
 
-type noopHandler struct{}
-
-func (s noopHandler) Event(TestEvent, *Execution) error {
-	return nil
-}
-
-func (s noopHandler) Err(string) error {
-	return nil
-}
-
 func TestPrintSummary_WithMisattributedOutput(t *testing.T) {
 	_, reset := patchClock()
 	defer reset()
 
 	exec, err := ScanTestOutput(ScanConfig{
-		Stdout:  bytes.NewReader(golden.Get(t, "go-test-json-misattributed.out")),
-		Stderr:  bytes.NewBuffer(nil),
-		Handler: noopHandler{},
+		Stdout: bytes.NewReader(golden.Get(t, "go-test-json-misattributed.out")),
+		Stderr: bytes.NewBuffer(nil),
 	})
 	assert.NilError(t, err)
 
@@ -273,9 +261,8 @@ func TestPrintSummary_WithSubtestFailures(t *testing.T) {
 	defer reset()
 
 	exec, err := ScanTestOutput(ScanConfig{
-		Stdout:  bytes.NewReader(golden.Get(t, "go-test-json.out")),
-		Stderr:  bytes.NewBuffer(nil),
-		Handler: noopHandler{},
+		Stdout: bytes.NewReader(golden.Get(t, "go-test-json.out")),
+		Stderr: bytes.NewBuffer(nil),
 	})
 	assert.NilError(t, err)