Repository: pantsbuild/pants

Issue:
Test Pants against public repositories
As discussed in:

https://groups.google.com/forum/#!topic/pants-devel/MhpY34sGx-g

We'd like to beef up the integration tests that ci.sh performs, to catch more breaking changes earlier. One idea is to have ci.sh build actual open source projects. It seems like twitter-commons and aurora are likely candidates:

https://reviews.apache.org/r/22662/



Discussion:
This came up recently. It would still be great to do, and there are some examples of projects like Rust doing this.

Resolving patch:
diff --git a/src/python/pants_release/generate_github_workflows.py b/src/python/pants_release/generate_github_workflows.py
--- a/src/python/pants_release/generate_github_workflows.py
+++ b/src/python/pants_release/generate_github_workflows.py
@@ -5,7 +5,8 @@
 
 import argparse
 import os
-from dataclasses import dataclass
+import re
+from dataclasses import dataclass, field
 from enum import Enum
 from pathlib import Path
 from textwrap import dedent  # noqa: PNT20
@@ -167,7 +168,11 @@ def ensure_category_label() -> Sequence[Step]:
 
 
 def checkout(
-    *, fetch_depth: int = 10, containerized: bool = False, ref: str | None = None
+    *,
+    fetch_depth: int = 10,
+    containerized: bool = False,
+    ref: str | None = None,
+    **extra_opts: object,
 ) -> Sequence[Step]:
     """Get prior commits and the commit message."""
     fetch_depth_opt: dict[str, Any] = {"fetch-depth": fetch_depth}
@@ -181,6 +186,7 @@ def checkout(
             "with": {
                 **fetch_depth_opt,
                 **({"ref": ref} if ref else {}),
+                **extra_opts,
             },
         },
     ]
@@ -275,6 +281,22 @@ def install_rustup() -> Step:
     }
 
 
+def install_python(version: str) -> Step:
+    return {
+        "name": f"Set up Python {version}",
+        "uses": "actions/setup-python@v4",
+        "with": {"python-version": version},
+    }
+
+
+def install_node(version: str) -> Step:
+    return {
+        "name": f"Set up Node {version}",
+        "uses": "actions/setup-node@v3",
+        "with": {"node-version": version},
+    }
+
+
 def install_jdk() -> Step:
     return {
         "name": "Install AdoptJDK",
@@ -467,13 +489,7 @@ def setup_primary_python(self) -> Sequence[Step]:
         # We pre-install Python on our self-hosted platforms.
         # We must set it up on Github-hosted platforms.
         if self.platform in GITHUB_HOSTED:
-            ret.append(
-                {
-                    "name": f"Set up Python {PYTHON_VERSION}",
-                    "uses": "actions/setup-python@v4",
-                    "with": {"python-version": PYTHON_VERSION},
-                }
-            )
+            ret.append(install_python(PYTHON_VERSION))
         return ret
 
     def expose_all_pythons(self) -> Sequence[Step]:
@@ -913,6 +929,7 @@ class WorkflowInput:
     name: str
     type_str: str
     default: str | int | None = None
+    description: None | str = None
 
 
 def workflow_dispatch_inputs(
@@ -923,6 +940,7 @@ def workflow_dispatch_inputs(
         wi.name.lower(): {
             "required": (wi.default is None),
             "type": wi.type_str,
+            **({} if wi.description is None else {"description": wi.description}),
             **({} if wi.default is None else {"default": wi.default}),
         }
         for wi in workflow_inputs
@@ -1101,6 +1119,293 @@ def release_jobs_and_inputs() -> tuple[Jobs, dict[str, Any]]:
     return jobs, inputs
 
 
+class DefaultGoals(str, Enum):
+    tailor_update_build_files = "tailor --check update-build-files --check ::"
+    lint_check = "lint check ::"
+    test = "test ::"
+    package = "package ::"
+
+
+@dataclass
+class Repo:
+    """A specification for an external public repository to run pants' testing against.
+
+    Each repository is tested in two configurations:
+
+    1. using the repository's default configuration (pants version, no additional settings), as a baseline
+
+    2. overriding the pants version and potentially setting additional `PANTS_...` environment
+       variable settings (both specified as workflow inputs)
+
+    The second is the interesting test, to validate whether a particular version/configuration of
+    Pants runs against this repository. The first/baseline is to make it obvious if the behaviour
+    _changes_, to avoid trying to analyse problems that already exist upstream.
+    """
+
+    name: str
+    """
+    `user/repo`, referring to `https://github.com/user/repo`. (This can be expanded to other services, if required.)
+    """
+
+    python_version: str = "3.10"
+    """
+    The Python version to install system-wide for user code to use.
+    """
+
+    env: dict[str, str] = field(default_factory=dict)
+    """
+    Any extra environment variables to provide to all pants steps
+    """
+
+    install_go: bool = False
+    """
+    Whether to install Go system-wide
+    """
+
+    install_thrift: bool = False
+    """
+    Whether to install Thrift system-wide
+    """
+
+    node_version: None | str = None
+    """
+    Whether to install Node/NPM system-wide, and which version if so
+    """
+
+    checkout_options: dict[str, Any] = field(default_factory=dict)
+    """
+    Any additional options to provide to actions/checkout
+    """
+
+    setup_commands: str = ""
+    """
+    Any additional set-up commands to run before pants (e.g. `sudo apt install ...`)
+    """
+
+    goals: Sequence[str] = tuple(DefaultGoals)
+    """
+    Which pants goals to run, e.g. `goals=["test some/dir::"]` would only run `pants test some/dir::`
+    """
+
+
+PUBLIC_REPOS = [
+    # pants' examples
+    Repo(
+        name="pantsbuild/example-adhoc",
+        node_version="20",
+        goals=[
+            # TODO: https://github.com/pantsbuild/pants/issues/14492 means pants can't find the
+            # `setup-node`-installed node, so we have to exclude `package ::`
+            DefaultGoals.lint_check,
+            DefaultGoals.test,
+        ],
+    ),
+    Repo(name="pantsbuild/example-codegen", install_thrift=True),
+    Repo(name="pantsbuild/example-django", python_version="3.9"),
+    Repo(
+        name="pantsbuild/example-docker",
+        python_version="3.8",
+        env={"DYNAMIC_TAG": "dynamic-tag-here"},
+    ),
+    Repo(name="pantsbuild/example-golang", install_go=True),
+    Repo(name="pantsbuild/example-jvm"),
+    Repo(name="pantsbuild/example-kotlin"),
+    Repo(name="pantsbuild/example-python", python_version="3.9"),
+    Repo(
+        name="pantsbuild/example-visibility",
+        python_version="3.9",
+        # skip check
+        goals=[DefaultGoals.tailor_update_build_files, "lint ::", DefaultGoals.test],
+    ),
+    # public repos
+    Repo(name="Ars-Linguistica/mlconjug3", goals=[DefaultGoals.package]),
+    Repo(
+        name="fucina/treb",
+        env={"GIT_COMMIT": "abcdef1234567890"},
+        goals=[
+            DefaultGoals.lint_check,
+            DefaultGoals.test,
+            DefaultGoals.package,
+        ],
+    ),
+    Repo(
+        name="ghandic/jsf",
+        goals=[
+            DefaultGoals.test,
+            DefaultGoals.package,
+        ],
+    ),
+    Repo(name="komprenilo/liga", python_version="3.9", goals=[DefaultGoals.package]),
+    Repo(
+        name="lablup/backend.ai",
+        python_version="3.11.3",
+        setup_commands="mkdir .tmp",
+        goals=[
+            DefaultGoals.tailor_update_build_files,
+            DefaultGoals.lint_check,
+            "test :: -tests/agent/docker:: -tests/client/integration:: -tests/common/redis::",
+            DefaultGoals.package,
+        ],
+    ),
+    Repo(name="mitodl/ol-infrastructure", goals=[DefaultGoals.package]),
+    Repo(
+        name="mitodl/ol-django",
+        setup_commands="sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl",
+        goals=[DefaultGoals.package],
+    ),
+    Repo(
+        name="naccdata/flywheel-gear-extensions",
+        goals=[DefaultGoals.test, "package :: -directory_pull::"],
+    ),
+    Repo(name="OpenSaMD/OpenSaMD", python_version="3.9.15"),
+    Repo(
+        name="StackStorm/st2",
+        python_version="3.8",
+        checkout_options={"submodules": "recursive"},
+        setup_commands=dedent(
+            # https://docs.stackstorm.com/development/sources.html
+            # TODO: install mongo like this doesn't work, see https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-ubuntu/
+            """
+            sudo apt-get install gcc git make screen libffi-dev libssl-dev python3.8-dev libldap2-dev libsasl2-dev
+            # sudo apt-get install mongodb mongodb-server
+            sudo apt-get install rabbitmq-server
+            """
+        ),
+        goals=[
+            DefaultGoals.tailor_update_build_files,
+            DefaultGoals.lint_check,
+            # TODO: seems like only st2client tests don't depend on mongo
+            "test st2client::",
+            DefaultGoals.package,
+        ],
+    ),
+]
+
+
+@dataclass
+class PublicReposOutput:
+    jobs: Jobs
+    inputs: dict[str, Any]
+    run_name: str
+
+
+def public_repos() -> PublicReposOutput:
+    """Run tests against public repositories, to validate new versions of Pants.
+
+    See `Repo` for more details.
+    """
+    inputs, env = workflow_dispatch_inputs(
+        [
+            WorkflowInput(
+                "PANTS_VERSION",
+                "string",
+                description="Pants version (for example, `2.16.0`, `2.18.0.dev1`)",
+            ),
+            # extra environment variables to pass when running the version under test,
+            # e.g. `PANTS_SOME_SUBSYSTEM_SOME_SETTING=abc`.  NB. we use it in a way that's vulnerable to
+            # shell injection (there's no validation that it uses A=1 B=2 syntax, it can easily contain
+            # more commands), but this whole workflow is "run untrusted code as a service", so Pants
+            # maintainers injecting things is the least of our worries
+            WorkflowInput(
+                "EXTRA_ENV",
+                "string",
+                default="",
+                description="Extra environment variables (for example: `PANTS_FOO_BAR=1 PANTS_BAZ_QUX=abc`)",
+            ),
+        ]
+    )
+
+    def sanitize_name(name: str) -> str:
+        # IDs may only contain alphanumeric characters, '_', and '-'.
+        return re.sub("[^A-Za-z0-9_-]+", "_", name)
+
+    def test_job(repo: Repo) -> object:
+        def gen_goals(use_default_version: bool) -> Sequence[object]:
+            if use_default_version:
+                name = "repo-default version (baseline)"
+                version = ""
+                env_prefix = ""
+            else:
+                name = version = env["PANTS_VERSION"]
+                env_prefix = env["EXTRA_ENV"]
+
+            return [
+                {
+                    "name": f"Run `{goal}` with {name}",
+                    # injecting the input string as just prefices is easier than turning it into
+                    # arguments for `env`
+                    "run": f"{env_prefix} pants {goal}",
+                    # run all the goals, even if there's an earlier failure, because later goals
+                    # might still be interesting (e.g. still run `test` even if `lint` fails)
+                    "if": "success() || failure()",
+                    "env": {"PANTS_VERSION": version},
+                }
+                for goal in ["version", *repo.goals]
+            ]
+
+        job_env: dict[str, str] = {
+            **repo.env,
+            "PANTS_REMOTE_CACHE_READ": "false",
+            "PANTS_REMOTE_CACHE_WRITE": "false",
+        }
+        return {
+            "name": repo.name,
+            "runs-on": "ubuntu-latest",
+            "env": job_env,
+            # we're running untrusted code, so this token shouldn't be able to do anything. We also
+            # need to be sure we don't add any secrets to the job
+            "permissions": {},
+            "steps": [
+                *checkout(repository=repo.name, **repo.checkout_options),
+                install_python(repo.python_version),
+                *([install_go()] if repo.install_go else []),
+                *([install_node(repo.node_version)] if repo.node_version else []),
+                *([download_apache_thrift()] if repo.install_thrift else []),
+                {
+                    "name": "Pants on",
+                    "run": dedent(
+                        # FIXME: save the script somewhere
+                        """
+                        curl --proto '=https' --tlsv1.2 -fsSL https://static.pantsbuild.org/setup/get-pants.sh | bash -
+                        echo "$HOME/bin" | tee -a $GITHUB_PATH
+                        """
+                    ),
+                },
+                {
+                    # the pants.ci.toml convention is strong, so check for it dynamically rather
+                    # than force each repo to mark it specifically
+                    "name": "Check for pants.ci.toml",
+                    "run": dedent(
+                        """
+                        if [[ -f pants.ci.toml ]]; then
+                            echo "PANTS_CONFIG_FILES=pants.ci.toml" | tee -a $GITHUB_ENV
+                        fi
+                        """
+                    ),
+                },
+                *(
+                    [{"name": "Run set-up", "run": repo.setup_commands}]
+                    if repo.setup_commands
+                    else []
+                ),
+                # first run with the repo's base configuration, as a reference point
+                *gen_goals(use_default_version=True),
+                # FIXME: scie-pants issue
+                {
+                    "name": "Kill pantsd",
+                    "run": "pkill -f pantsd",
+                    "if": "success() || failure()",
+                },
+                # then run with the version under test (simulates an in-place upgrade, locally, too)
+                *gen_goals(use_default_version=False),
+            ],
+        }
+
+    jobs = {sanitize_name(repo.name): test_job(repo) for repo in PUBLIC_REPOS}
+    run_name = f"Public repos test: version {env['PANTS_VERSION']} {env['EXTRA_ENV']}"
+    return PublicReposOutput(jobs=jobs, inputs=inputs, run_name=run_name)
+
+
 # ----------------------------------------------------------------------
 # Main file
 # ----------------------------------------------------------------------
@@ -1260,11 +1565,23 @@ def generate() -> dict[Path, str]:
         Dumper=NoAliasDumper,
     )
 
+    public_repos_output = public_repos()
+    public_repos_yaml = yaml.dump(
+        {
+            "name": "Public repos tests",
+            "run-name": public_repos_output.run_name,
+            "on": {"workflow_dispatch": {"inputs": public_repos_output.inputs}},
+            "jobs": public_repos_output.jobs,
+        },
+        Dumper=NoAliasDumper,
+    )
+
     return {
         Path(".github/workflows/audit.yaml"): f"{HEADER}\n\n{audit_yaml}",
         Path(".github/workflows/cache_comparison.yaml"): f"{HEADER}\n\n{cache_comparison_yaml}",
         Path(".github/workflows/test.yaml"): f"{HEADER}\n\n{test_yaml}",
         Path(".github/workflows/release.yaml"): f"{HEADER}\n\n{release_yaml}",
+        Path(".github/workflows/public_repos.yaml"): f"{HEADER}\n\n{public_repos_yaml}",
     }
 
 


Test patch: