Repository: pantsbuild/pants

Issue:
go: support passing flags to test runner
Go test binaries take a standard set of options. See `go help testflag` for a list of options. Pants should support passing those options through to the test binaries that it produces. It is not as simple as merely passing through arguments though because the `go` tool transforms the options before passing them to the test binary.

Taking the verbosity option `-v` as the example, while it is specified as `go test -v` usually, the test binary sees `-test.v`, not `-v`. Thus Pants will either need to support:

1. Just pass raw arguments through to the test binary with no transformation. Users will need to use the `-test.v` syntax which might look like: `-go-test-args="-test.v"` -- This is verbose and clunky.
2. Parse and transform a list of arguments passed via `--go-test-args`.
3. Define specific options under `go-test` scope and generate the appropriate test runner options. E.g., `--go-test-verbose` is a bool option that causes Pants to add the `-test.v` option to the test runner args.


Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/go/subsystems/gotest.py b/src/python/pants/backend/go/subsystems/gotest.py
new file mode 100644
--- /dev/null
+++ b/src/python/pants/backend/go/subsystems/gotest.py
@@ -0,0 +1,32 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+from pants.option.custom_types import shell_str
+from pants.option.subsystem import Subsystem
+
+
+class GoTestSubsystem(Subsystem):
+    options_scope = "go-test"
+    help = "Options for Go tests."
+
+    @classmethod
+    def register_options(cls, register):
+        super().register_options(register)
+        register(
+            "--args",
+            type=list,
+            member_type=shell_str,
+            passthrough=True,
+            help=(
+                'Arguments to pass directly to the Go test binary, e.g. `--go-test-args="-run TestFoo -v"` '
+                "Known Go test options will be transformed into the form expected by the test binary, "
+                "e.g. `-v` becomes `-test.v`. Run `go help testflag` from the Go SDK to learn more about "
+                "the options supported by Go test binaries."
+            ),
+        )
+
+    @property
+    def args(self) -> tuple[str, ...]:
+        return tuple(self.options.args)


Test patch:
diff --git a/src/python/pants/backend/go/goals/test.py b/src/python/pants/backend/go/goals/test.py
--- a/src/python/pants/backend/go/goals/test.py
+++ b/src/python/pants/backend/go/goals/test.py
@@ -4,7 +4,9 @@
 from __future__ import annotations
 
 import os
+from typing import Sequence
 
+from pants.backend.go.subsystems.gotest import GoTestSubsystem
 from pants.backend.go.target_types import GoFirstPartyPackageSourcesField, GoImportPathField
 from pants.backend.go.util_rules.build_pkg import (
     BuildGoPackageRequest,
@@ -25,6 +27,35 @@
 from pants.engine.unions import UnionRule
 from pants.util.ordered_set import FrozenOrderedSet, OrderedSet
 
+# Known options to Go test binaries. Only these options will be transformed by `transform_test_args`.
+# The bool value represents whether the option is expected to take a value or not.
+# To regenerate this list, run `go run ./gentestflags.go` and copy the output below.
+TEST_FLAGS = {
+    "bench": True,
+    "benchmem": False,
+    "benchtime": True,
+    "blockprofile": True,
+    "blockprofilerate": True,
+    "count": True,
+    "coverprofile": True,
+    "cpu": True,
+    "cpuprofile": True,
+    "failfast": False,
+    "list": True,
+    "memprofile": True,
+    "memprofilerate": True,
+    "mutexprofile": True,
+    "mutexprofilefraction": True,
+    "outputdir": True,
+    "parallel": True,
+    "run": True,
+    "short": False,
+    "shuffle": True,
+    "timeout": True,
+    "trace": True,
+    "v": False,
+}
+
 
 class GoTestFieldSet(TestFieldSet):
     required_fields = (GoFirstPartyPackageSourcesField,)
@@ -32,8 +63,56 @@ class GoTestFieldSet(TestFieldSet):
     sources: GoFirstPartyPackageSourcesField
 
 
+def transform_test_args(args: Sequence[str]) -> tuple[str, ...]:
+    result = []
+
+    i = 0
+    next_arg_is_option_value = False
+    while i < len(args):
+        arg = args[i]
+        i += 1
+
+        # If this argument is an option value, then append it to the result and continue to next argument.
+        if next_arg_is_option_value:
+            result.append(arg)
+            next_arg_is_option_value = False
+            continue
+
+        # Non-arguments stop option processing.
+        if arg[0] != "-":
+            result.append(arg)
+            break
+
+        # Stop processing since "-" is a non-argument and "--" is terminator.
+        if arg == "-" or arg == "--":
+            result.append(arg)
+            break
+
+        start_index = 2 if arg[1] == "-" else 1
+        equals_index = arg.find("=", start_index)
+        if equals_index != -1:
+            arg_name = arg[start_index:equals_index]
+            option_value = arg[equals_index:]
+        else:
+            arg_name = arg[start_index:]
+            option_value = ""
+
+        if arg_name in TEST_FLAGS:
+            rewritten_arg = f"{arg[0:start_index]}test.{arg_name}{option_value}"
+            result.append(rewritten_arg)
+            if TEST_FLAGS[arg_name] and option_value == "":
+                next_arg_is_option_value = True
+        else:
+            result.append(arg)
+
+    result.extend(args[i:])
+    return tuple(result)
+
+
 @rule
-async def run_go_tests(field_set: GoTestFieldSet, test_subsystem: TestSubsystem) -> TestResult:
+async def run_go_tests(
+    field_set: GoTestFieldSet, test_subsystem: TestSubsystem, go_test_subsystem: GoTestSubsystem
+) -> TestResult:
     pkg_info, wrapped_target = await MultiGet(
         Get(FirstPartyPkgInfo, FirstPartyPkgInfoRequest(field_set.address)),
         Get(WrappedTarget, Address, field_set.address),
@@ -140,7 +219,7 @@ async def run_go_tests(field_set: GoTestFieldSet, test_subsystem: TestSubsystem)
     result = await Get(
         FallibleProcessResult,
         Process(
-            ["./test_runner"],
+            ["./test_runner", *transform_test_args(go_test_subsystem.args)],
             input_digest=binary.digest,
             description=f"Run Go tests: {field_set.address}",
             cache_scope=cache_scope,
diff --git a/src/python/pants/backend/go/goals/test_test.py b/src/python/pants/backend/go/goals/test_test.py
--- a/src/python/pants/backend/go/goals/test_test.py
+++ b/src/python/pants/backend/go/goals/test_test.py
@@ -10,6 +10,7 @@
 from pants.backend.go import target_type_rules
 from pants.backend.go.goals.test import GoTestFieldSet
 from pants.backend.go.goals.test import rules as test_rules
+from pants.backend.go.goals.test import transform_test_args
 from pants.backend.go.target_types import GoModTarget
 from pants.backend.go.util_rules import (
     assembly,
@@ -44,10 +45,21 @@ def rule_runner() -> RuleRunner:
         ],
         target_types=[GoModTarget],
     )
-    rule_runner.set_options([], env_inherit={"PATH"})
+    rule_runner.set_options(["--go-test-args=-v -bench=."], env_inherit={"PATH"})
     return rule_runner
 
 
+def test_transform_test_args() -> None:
+    assert transform_test_args(["-v", "--", "-v"]) == ("-test.v", "--", "-v")
+    assert transform_test_args(["-run=TestFoo", "-v"]) == ("-test.run=TestFoo", "-test.v")
+    assert transform_test_args(["-run", "TestFoo", "-foo", "-v"]) == (
+        "-test.run",
+        "TestFoo",
+        "-foo",
+        "-test.v",
+    )
+
+
 def test_internal_test_success(rule_runner: RuleRunner) -> None:
     rule_runner.write_files(
         {
@@ -77,7 +89,7 @@ def test_internal_test_success(rule_runner: RuleRunner) -> None:
     tgt = rule_runner.get_target(Address("foo", generated_name="./"))
     result = rule_runner.request(TestResult, [GoTestFieldSet.create(tgt)])
     assert result.exit_code == 0
-    assert result.stdout == "PASS\n"
+    assert "PASS: TestAdd" in result.stdout
 
 
 def test_internal_test_fails(rule_runner: RuleRunner) -> None:
@@ -134,7 +146,8 @@ def test_internal_benchmark_passes(rule_runner: RuleRunner) -> None:
     tgt = rule_runner.get_target(Address("foo", generated_name="./"))
     result = rule_runner.request(TestResult, [GoTestFieldSet.create(tgt)])
     assert result.exit_code == 0
-    assert result.stdout == "PASS\n"
+    assert "BenchmarkAdd" in result.stdout
+    assert "PASS" in result.stdout
 
 
 def test_internal_example_passes(rule_runner: RuleRunner) -> None:
@@ -159,7 +172,7 @@ def test_internal_example_passes(rule_runner: RuleRunner) -> None:
     tgt = rule_runner.get_target(Address("foo", generated_name="./"))
     result = rule_runner.request(TestResult, [GoTestFieldSet.create(tgt)])
     assert result.exit_code == 0
-    assert result.stdout == "PASS\n"
+    assert "PASS: ExamplePrint" in result.stdout
 
 
 def test_internal_test_with_test_main(rule_runner: RuleRunner) -> None:
@@ -179,6 +192,7 @@ def test_internal_test_with_test_main(rule_runner: RuleRunner) -> None:
                 }
                 func TestMain(m *testing.M) {
                   fmt.Println("foo.TestMain called")
+                  m.Run()
                 }
                 """
             ),
@@ -186,8 +200,9 @@ def test_internal_test_with_test_main(rule_runner: RuleRunner) -> None:
     )
     tgt = rule_runner.get_target(Address("foo", generated_name="./"))
     result = rule_runner.request(TestResult, [GoTestFieldSet.create(tgt)])
-    assert result.exit_code == 0
+    assert result.exit_code == 1
     assert "foo.TestMain called" in result.stdout
+    assert "FAIL: TestAdd" in result.stdout
 
 
 def test_external_test_success(rule_runner: RuleRunner) -> None:
@@ -222,7 +237,7 @@ def test_external_test_success(rule_runner: RuleRunner) -> None:
     tgt = rule_runner.get_target(Address("foo", generated_name="./"))
     result = rule_runner.request(TestResult, [GoTestFieldSet.create(tgt)])
     assert result.exit_code == 0
-    assert result.stdout == "PASS\n"
+    assert "PASS: TestAdd" in result.stdout
 
 
 def test_external_test_fails(rule_runner: RuleRunner) -> None:
@@ -293,7 +308,8 @@ def test_external_benchmark_passes(rule_runner: RuleRunner) -> None:
     tgt = rule_runner.get_target(Address("foo", generated_name="./"))
     result = rule_runner.request(TestResult, [GoTestFieldSet.create(tgt)])
     assert result.exit_code == 0
-    assert result.stdout == "PASS\n"
+    assert "BenchmarkAdd" in result.stdout
+    assert "PASS" in result.stdout
 
 
 def test_external_example_passes(rule_runner: RuleRunner) -> None:
@@ -327,7 +343,7 @@ def test_external_example_passes(rule_runner: RuleRunner) -> None:
     tgt = rule_runner.get_target(Address("foo", generated_name="./"))
     result = rule_runner.request(TestResult, [GoTestFieldSet.create(tgt)])
     assert result.exit_code == 0
-    assert result.stdout == "PASS\n"
+    assert "PASS: ExamplePrint" in result.stdout
 
 
 def test_external_test_with_test_main(rule_runner: RuleRunner) -> None: