Repository: swiftlang/swift-package-manager
Pull request #8559: pipeline: support additional args in build-using-self
Author: login: bkhouri
Created: 2025-04-25T13:46:46Z | Merged: 2025-04-30T18:23:48Z

Update the build-using-self to support accepting a triple and a build system, which some self hosted CI environments might want to set.


Diff:
diff --git a/Utilities/build-using-self b/Utilities/build-using-self
index 68bfa6433a8..7bb2b9048ee 100755
--- a/Utilities/build-using-self
+++ b/Utilities/build-using-self
@@ -12,6 +12,8 @@
 # ===----------------------------------------------------------------------===##
 
 import argparse
+import dataclasses
+import itertools
 import logging
 import os
 import pathlib
@@ -64,10 +66,22 @@ def get_arguments() -> argparse.Namespace:
         "--configuration",
         type=Configuration,
         dest="config",
-        default="debug",
-        choices=[e.value for e in Configuration],
+        default=Configuration.DEBUG,
+        choices=[e for e in Configuration],
         help="The configuraiton to use.",
     )
+    parser.add_argument(
+        "-t",
+        "--triple",
+        type=str,
+        dest="triple",
+    )
+    parser.add_argument(
+        "-b",
+        "--build-system",
+        type=str,
+        dest="build_system",
+    )
     parser.add_argument(
         "--enable-swift-testing",
         action="store_true",
@@ -99,7 +113,10 @@ def is_on_darwin() -> bool:
     return platform.uname().system == "Darwin"
 
 
-def set_environment(*, swiftpm_bin_dir: pathlib.Path,) -> None:
+def set_environment(
+    *,
+    swiftpm_bin_dir: pathlib.Path,
+) -> None:
     os.environ["SWIFTCI_IS_SELF_HOSTED"] = "1"
 
     # Set the SWIFTPM_CUSTOM_BIN_DIR path
@@ -131,39 +148,117 @@ def run_bootstrap(swiftpm_bin_dir: pathlib.Path) -> None:
     )
 
 
+GlobalArgsValueType = str
+
+
+@dataclasses.dataclass
+class GlobalArgs:
+    global_argument: str
+    value: t.Optional[GlobalArgsValueType]
+
+
+def filterNone(items: t.Iterable) -> t.Iterable:
+    return list(filter(lambda x: x is not None, items))
+
+
 def main() -> None:
     args = get_arguments()
-    ignore = "-Xlinker /ignore:4217" if os.name == "nt" else ""
     logging.getLogger().setLevel(logging.DEBUG if args.is_verbose else logging.INFO)
     logging.debug("Args: %r", args)
-
+    ignore_args = ["-Xlinker", "/ignore:4217"] if os.name == "nt" else []
+    globalArgsData = [
+        GlobalArgs(global_argument="--triple", value=args.triple),
+        GlobalArgs(global_argument="--build-system", value=args.build_system),
+    ]
+    global_args: t.Iterator[GlobalArgsValueType] = list(
+        itertools.chain.from_iterable(
+            [[arg.global_argument, arg.value] for arg in globalArgsData if arg.value]
+        )
+    )
+    logging.debug("Global Args: %r", global_args)
     with change_directory(REPO_ROOT_PATH):
         swiftpm_bin_dir = get_swiftpm_bin_dir(config=args.config)
         set_environment(swiftpm_bin_dir=swiftpm_bin_dir)
 
         call(
-            shlex.split("swift --version"),
+            filterNone(
+                [
+                    "swift",
+                    "--version",
+                ]
+            )
         )
 
         call(
-            shlex.split("swift package update"),
+            filterNone(
+                [
+                    "swift",
+                    "package",
+                    "update",
+                ]
+            )
         )
         call(
-            shlex.split(f"swift build --configuration {args.config} {ignore}"),
+            filterNone(
+                [
+                    "swift",
+                    "build",
+                    *global_args,
+                    "--configuration",
+                    args.config,
+                    *ignore_args,
+                ]
+            )
         )
 
-        swift_testing_arg= "--enable-swift-testing" if args.enable_swift_testing else ""
-        xctest_arg= "--enable-xctest" if args.enable_swift_testing else ""
+        swift_testing_arg = (
+            "--enable-swift-testing" if args.enable_swift_testing else None
+        )
+        xctest_arg = "--enable-xctest" if args.enable_swift_testing else None
         call(
-            shlex.split(f"swift run swift-test --configuration {args.config} --parallel {swift_testing_arg} {xctest_arg} --scratch-path .test {ignore}"),
+            filterNone(
+                [
+                    "swift",
+                    "run",
+                    "swift-test",
+                    *global_args,
+                    "--configuration",
+                    args.config,
+                    "--parallel",
+                    swift_testing_arg,
+                    xctest_arg,
+                    "--scratch-path",
+                    ".test",
+                    *ignore_args,
+                ]
+            )
         )
 
         integration_test_dir = (REPO_ROOT_PATH / "IntegrationTests").as_posix()
         call(
-            shlex.split(f"swift package --package-path {integration_test_dir} update"),
+            filterNone(
+                [
+                    "swift",
+                    "package",
+                    "--package-path",
+                    integration_test_dir,
+                    "update",
+                ]
+            )
         )
         call(
-            shlex.split(f"swift run swift-test --package-path {integration_test_dir} --parallel {ignore}"),
+            filterNone(
+                [
+                    "swift",
+                    "run",
+                    "swift-test",
+                    *global_args,
+                    "--package-path",
+                    integration_test_dir,
+                    "--parallel",
+                    *ignore_args,
+                ]
+            )
         )
 
     if is_on_darwin():
diff --git a/Utilities/helpers.py b/Utilities/helpers.py
index b5a4a9df9a9..8f469e8ecca 100644
--- a/Utilities/helpers.py
+++ b/Utilities/helpers.py
@@ -65,11 +65,11 @@ def mkdir_p(path):
 def call(cmd, cwd=None, verbose=False):
     """Calls a subprocess."""
     cwd = cwd or pathlib.Path.cwd()
-    logging.info("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd)
     try:
+        logging.info("executing command >>> %r with cwd %s", cmd, cwd)
         subprocess.check_call(cmd, cwd=cwd)
     except subprocess.CalledProcessError as cpe:
-        logging.debug("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd)
+        logging.debug("executing command >>> %r with cwd %s", cmd, cwd)
         logging.error(
             "\n".join([
                 "Process failure with return code %d: %s",
@@ -96,7 +96,7 @@ def call_output(cmd, cwd=None, stderr=False, verbose=False):
     """Calls a subprocess for its return data."""
     stderr = subprocess.STDOUT if stderr else False
     cwd = cwd or pathlib.Path.cwd()
-    logging.info("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd)
+    logging.info("executing command >>> %r with cwd %s", cmd, cwd)
     try:
         return subprocess.check_output(
             cmd,
@@ -105,7 +105,7 @@ def call_output(cmd, cwd=None, stderr=False, verbose=False):
             universal_newlines=True,
         ).strip()
     except subprocess.CalledProcessError as cpe:
-        logging.debug("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd)
+        logging.debug("executing command >>> %r with cwd %s", cmd, cwd)
         logging.error(
             "\n".join([
                 "Process failure with return code %d: %s",


Reviews:
user: login: MaxDesiatov
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-25T14:03:19Z
user: login: MaxDesiatov
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-25T14:03:42Z
user: login: kcieplak
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T14:39:12Z
user: login: dschaefer2
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-28T17:23:38Z
user: login: bkhouri
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T17:26:10Z
user: login: dschaefer2
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-28T17:41:38Z
user: login: dschaefer2
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-28T18:00:53Z
user: login: bkhouri
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T19:43:17Z
user: login: dschaefer2
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-30T18:23:46Z

Review comments:
diff_hunk: @@ -148,23 +159,31 @@ def main() -> None:
         call(
             shlex.split("swift package update"),
         )
+
         call(
-            shlex.split(f"swift build --configuration {args.config} {ignore}"),
+            shlex.split(f"swift build --configuration {args.config} --build-tests {extra_args}"),
         )
 
         if os.name != "nt": # turn off for Windows until we get the hang resolved
-            swift_testing_arg= "--enable-swift-testing" if args.enable_swift_testing else ""
-            xctest_arg= "--enable-xctest" if args.enable_swift_testing else ""
+            swift_testing_arg = "--enable-swift-testing" if args.enable_swift_testing else ""
+            xctest_arg = "--enable-xctest" if args.enable_swift_testing else ""
+            swift_test_executable = swiftpm_bin_dir / 'swift-test'
             call(
-                shlex.split(f"swift run swift-test --configuration {args.config} --parallel {swift_testing_arg} {xctest_arg} --scratch-path .test {ignore}"),
+                shlex.split(
+                    f"{swift_test_executable} --configuration {args.config} --parallel {swift_testing_arg} {xctest_arg} --scratch-path .test {extra_args}",
+                    posix=(os.name == "posix"), # must be set correctly, otherwhsie shlex.split("C:\\Foo\\bar") become ['CFoobar']
path: Utilities/build-using-self
user: login: MaxDesiatov
body: typo nit
```suggestion
                    posix=(os.name == "posix"), # must be set correctly, otherwise shlex.split("C:\\Foo\\bar") becomes ['CFoobar']
```
created_at: 2025-04-25T14:03:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -148,23 +159,31 @@ def main() -> None:
         call(
             shlex.split("swift package update"),
         )
+
         call(
-            shlex.split(f"swift build --configuration {args.config} {ignore}"),
+            shlex.split(f"swift build --configuration {args.config} --build-tests {extra_args}"),
         )
 
         if os.name != "nt": # turn off for Windows until we get the hang resolved
-            swift_testing_arg= "--enable-swift-testing" if args.enable_swift_testing else ""
-            xctest_arg= "--enable-xctest" if args.enable_swift_testing else ""
+            swift_testing_arg = "--enable-swift-testing" if args.enable_swift_testing else ""
+            xctest_arg = "--enable-xctest" if args.enable_swift_testing else ""
+            swift_test_executable = swiftpm_bin_dir / 'swift-test'
             call(
-                shlex.split(f"swift run swift-test --configuration {args.config} --parallel {swift_testing_arg} {xctest_arg} --scratch-path .test {ignore}"),
+                shlex.split(
+                    f"{swift_test_executable} --configuration {args.config} --parallel {swift_testing_arg} {xctest_arg} --scratch-path .test {extra_args}",
+                    posix=(os.name == "posix"), # must be set correctly, otherwhsie shlex.split("C:\\Foo\\bar") become ['CFoobar']
+                ),
             )
 
             integration_test_dir = REPO_ROOT_PATH / "IntegrationTests"
             call(
                 shlex.split(f"swift package --package-path {integration_test_dir} update"),
             )
             call(
-                shlex.split(f"swift run swift-test --package-path {integration_test_dir} --parallel {ignore}"),
+                shlex.split(
+                    f"{swift_test_executable} --package-path {integration_test_dir} --parallel {extra_args}",
+                    posix=(os.name == "posix"), # must be set correctly, otherwhsie shlex.split("C:\\Foo\\bar") become ['CFoobar']
path: Utilities/build-using-self
user: login: MaxDesiatov
body: ```suggestion
                    posix=(os.name == "posix"), # must be set correctly, otherwise shlex.split("C:\\Foo\\bar") becomes ['CFoobar']
```
created_at: 2025-04-25T14:03:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -148,22 +159,32 @@ def main() -> None:
         call(
             shlex.split("swift package update"),
         )
+
         call(
-            shlex.split(f"swift build --configuration {args.config} {ignore}"),
+            shlex.split(f"swift build --configuration {args.config} --build-tests {extra_args}"),
         )
 
-        swift_testing_arg= "--enable-swift-testing" if args.enable_swift_testing else ""
-        xctest_arg= "--enable-xctest" if args.enable_swift_testing else ""
+        swift_testing_arg = "--enable-swift-testing" if args.enable_swift_testing else ""
+        xctest_arg = "--enable-xctest" if args.enable_swift_testing else ""
+        # Do not use `swift run swift-test` to avoid having `swift run` consume arguments that are common between
+        # `swift run` and `swift-test`
path: Utilities/build-using-self
user: login: dschaefer2
body: Or put the swift run arguments before swift-test?
created_at: 2025-04-28T17:23:22Z
author_association: MEMBER
diff_hunk: @@ -148,22 +159,32 @@ def main() -> None:
         call(
             shlex.split("swift package update"),
         )
+
         call(
-            shlex.split(f"swift build --configuration {args.config} {ignore}"),
+            shlex.split(f"swift build --configuration {args.config} --build-tests {extra_args}"),
         )
 
-        swift_testing_arg= "--enable-swift-testing" if args.enable_swift_testing else ""
-        xctest_arg= "--enable-xctest" if args.enable_swift_testing else ""
+        swift_testing_arg = "--enable-swift-testing" if args.enable_swift_testing else ""
+        xctest_arg = "--enable-xctest" if args.enable_swift_testing else ""
+        # Do not use `swift run swift-test` to avoid having `swift run` consume arguments that are common between
+        # `swift run` and `swift-test`
path: Utilities/build-using-self
user: login: bkhouri
body: Sure, but looks like we are dependent on argument order, which is not very developer friendly as we may not have the behaviour we expect.  For example, I want to have `swift run swift-test --build-system swiftbuild` where I want the `--build-system` to be set on the `swift-test`, but it looks like `swift swift` may have been picking it up.
created_at: 2025-04-28T17:26:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -148,22 +159,32 @@ def main() -> None:
         call(
             shlex.split("swift package update"),
         )
+
         call(
-            shlex.split(f"swift build --configuration {args.config} {ignore}"),
+            shlex.split(f"swift build --configuration {args.config} --build-tests {extra_args}"),
         )
 
-        swift_testing_arg= "--enable-swift-testing" if args.enable_swift_testing else ""
-        xctest_arg= "--enable-xctest" if args.enable_swift_testing else ""
+        swift_testing_arg = "--enable-swift-testing" if args.enable_swift_testing else ""
+        xctest_arg = "--enable-xctest" if args.enable_swift_testing else ""
+        # Do not use `swift run swift-test` to avoid having `swift run` consume arguments that are common between
+        # `swift run` and `swift-test`
path: Utilities/build-using-self
user: login: dschaefer2
body: As an example

```
swift run swift-test --scratch-path .build/test -Xlinker /ignore:4217
```

swift run doesn't take the scratch-path or Xlinker args from swift-test. Where were you seeing swift run consuming the swift-test args?
created_at: 2025-04-28T17:41:38Z
author_association: MEMBER
diff_hunk: @@ -148,22 +159,32 @@ def main() -> None:
         call(
             shlex.split("swift package update"),
         )
+
         call(
-            shlex.split(f"swift build --configuration {args.config} {ignore}"),
+            shlex.split(f"swift build --configuration {args.config} --build-tests {extra_args}"),
         )
 
-        swift_testing_arg= "--enable-swift-testing" if args.enable_swift_testing else ""
-        xctest_arg= "--enable-xctest" if args.enable_swift_testing else ""
+        swift_testing_arg = "--enable-swift-testing" if args.enable_swift_testing else ""
+        xctest_arg = "--enable-xctest" if args.enable_swift_testing else ""
+        # Do not use `swift run swift-test` to avoid having `swift run` consume arguments that are common between
+        # `swift run` and `swift-test`
path: Utilities/build-using-self
user: login: dschaefer2
body: I just tried your example:

```
swift run swift-test --package-path ../mytest --build-system swiftbuild
```

And it worked as expected. Built SwiftPM with the native build system and ../mytest with SwiftBuild.

We tell people not to reach in and run the executable, we should probably be consistent with that and not do it ourselves.
created_at: 2025-04-28T18:00:52Z
author_association: MEMBER
diff_hunk: @@ -148,22 +159,32 @@ def main() -> None:
         call(
             shlex.split("swift package update"),
         )
+
         call(
-            shlex.split(f"swift build --configuration {args.config} {ignore}"),
+            shlex.split(f"swift build --configuration {args.config} --build-tests {extra_args}"),
         )
 
-        swift_testing_arg= "--enable-swift-testing" if args.enable_swift_testing else ""
-        xctest_arg= "--enable-xctest" if args.enable_swift_testing else ""
+        swift_testing_arg = "--enable-swift-testing" if args.enable_swift_testing else ""
+        xctest_arg = "--enable-xctest" if args.enable_swift_testing else ""
+        # Do not use `swift run swift-test` to avoid having `swift run` consume arguments that are common between
+        # `swift run` and `swift-test`
path: Utilities/build-using-self
user: login: bkhouri
body: Right.. I must have gotten confused while `swift run` was building the executable target then.  I'll update.
created_at: 2025-04-28T18:20:55Z
author_association: CONTRIBUTOR

PR comments:
user: login: bkhouri
created_at: 2025-04-25T13:46:55Z
body: @swift-ci test
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-25T14:13:29Z
body: @swift-ci test
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-25T15:10:42Z
body: @swift-ci test linux
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-25T15:12:05Z
body: @swift-ci test windows
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-28T14:49:23Z
body: @swift-ci test
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-28T14:53:25Z
body: @swift-ci test windows
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-28T15:58:19Z
body: @swift-ci test
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-28T17:16:04Z
body: @swift-ci test windows
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-28T19:43:39Z
body: @swift-ci test
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-29T01:34:27Z
body: @swift-ci test
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-29T13:05:37Z
body: @swift-ci test
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-29T13:13:31Z
body: @swift-ci test windows
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-29T15:36:31Z
body: @swift-ci test
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-29T17:22:13Z
body: @swift-ci test
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-29T17:25:20Z
body: @swift-ci test windows
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-29T17:40:04Z
body: @swift-ci test
author_association: CONTRIBUTOR
user: login: bkhouri
created_at: 2025-04-29T17:44:31Z
body: @swift-ci test windows
author_association: CONTRIBUTOR