Repository: osbuild/osbuild
Pull request #1660 (2024-03-13 11:51:16)

test: Add stage test trigger to makefile
Stage tests are located in the other place than module or run tests. For consistency separated target in makefile is added.

elkoniu: Latest change is just rebase on top of main.

Patch:
diff --git a/Makefile b/Makefile
index 1537a2d8..5a51888b 100644
--- a/Makefile
+++ b/Makefile
@@ -95,6 +95,7 @@ help:
 	@echo "    test-all:           Run all tests"
 	@echo "    test-data:          Generate test data"
 	@echo "    test-module:        Run all module unit-tests"
+	@echo "    test-stage:         Run all stage unit-tests"
 	@echo "    test-run:           Run all osbuild pipeline tests"
 
 $(BUILDDIR)/:
@@ -225,6 +226,13 @@ test-module:
 			--rootdir=$(SRCDIR) \
 			-v
 
+.PHONY: test-stage
+test-stage:
+	@$(PYTHON3) -m pytest \
+			$(SRCDIR)/stages/test \
+			--rootdir=$(SRCDIR) \
+			-v
+
 .PHONY: test-run
 test-run:
 	@[[ $${EUID} -eq 0 ]] || (echo "Error: Root privileges required!"; exit 1)
diff --git a/osbuild/util/parsing.py b/osbuild/util/parsing.py
index 303fbefa..f8fb2768 100644
--- a/osbuild/util/parsing.py
+++ b/osbuild/util/parsing.py
@@ -1,7 +1,9 @@
 """Helpers related to parsing"""
 
+import os
 import re
-from typing import Union
+from typing import Dict, Union
+from urllib.parse import ParseResult, urlparse
 
 
 def parse_size(s: str) -> Union[int, str]:
@@ -31,3 +33,75 @@ def parse_size(s: str) -> Union[int, str]:
                 return "unlimited"
 
     raise TypeError(f"invalid size value: '{s}'")
+
+
+def find_mount_root(url: ParseResult, args: Dict) -> os.PathLike:
+    """
+    Parses the mount URL to extract the root path.
+
+    Parameters:
+    - url (ParseResult): The ParseResult object obtained from urlparse.
+    - args (Dict):A dictionary containing arguments including mounts and
+    path information as passed by osbuild.api.arguments()
+    """
+    name = url.netloc
+    if name:
+        root = args["mounts"].get(name, {}).get("path")
+        if root is None:
+            raise ValueError(f"Unknown mount '{name}'")
+    else:
+        root = args["paths"]["mounts"]
+
+    return root
+
+
+def parse_input(url: ParseResult, args: Dict) -> os.PathLike:
+    """
+    Parses the input URL to extract the root path.
+
+    Parameters:
+    - url (ParseResult): The ParseResult object obtained from urlparse.
+    - args (Dict): A dictionary containing arguments including mounts and
+    path information as passed by osbuild.api.arguments()
+    """
+    name = url.netloc
+    root = args["inputs"].get(name, {}).get("path")
+    if root is None:
+        raise ValueError(f"Unknown input '{name}'")
+
+    return root
+
+
+def parse_location(location: str, args: Dict) -> str:
+    """
+    Parses the location URL to derive the corresponding file path.
+
+    Parameters:
+    - location (str): The location URL to be parsed.
+    - args (Dict): A dictionary containing arguments including mounts and
+    path information as passed by osbuild.api.arguments()
+    """
+
+    url = urlparse(location)
+
+    scheme = url.scheme
+    if scheme == "tree":
+        root = args["tree"]
+    elif scheme == "mount":
+        root = find_mount_root(url, args)
+    elif scheme == "input":
+        root = parse_input(url, args)
+    else:
+        raise ValueError(f"Unsupported scheme '{scheme}'")
+
+    if not url.path.startswith("/"):
+        raise ValueError(f"url.path from location must start with '/', got: {url.path}")
+
+    path = os.path.relpath(url.path, "/")
+    path = os.path.join(root, path)
+    path = os.path.normpath(path)
+
+    if url.path.endswith("/"):
+        path = os.path.join(path, ".")
+
+    return path
diff --git a/stages/org.osbuild.chattr b/stages/org.osbuild.chattr
index b1e6ad88..4b714c79 100755
--- a/stages/org.osbuild.chattr
+++ b/stages/org.osbuild.chattr
@@ -1,52 +1,15 @@
 #!/usr/bin/python3
-import os
 import subprocess
 import sys
-from typing import Dict
-from urllib.parse import ParseResult, urlparse
 
 import osbuild.api
-
-
-def parse_mount(url: ParseResult, args: Dict):
-    name = url.netloc
-    if name:
-        root = args["mounts"].get(name, {}).get("path")
-        if not root:
-            raise ValueError(f"Unknown mount '{name}'")
-    else:
-        root = args["paths"]["mounts"]
-
-    return root
-
-
-def parse_location(location, args):
-    url = urlparse(location)
-
-    scheme = url.scheme
-    if scheme == "tree":
-        root = args["tree"]
-    elif scheme == "mount":
-        root = parse_mount(url, args)
-    else:
-        raise ValueError(f"Unsupported scheme '{scheme}'")
-
-    assert url.path.startswith("/")
-
-    path = os.path.relpath(url.path, "/")
-    path = os.path.join(root, path)
-    path = os.path.normpath(path)
-
-    if url.path.endswith("/"):
-        path = os.path.join(path, ".")
-
-    return path
+from osbuild.util import parsing
 
 
 def main(args, options):
     for path, cmdargs in options["items"].items():
         immutable = cmdargs["immutable"]
-        dst = parse_location(path, args)
+        dst = parsing.parse_location(path, args)
         op = '+' if immutable else '-'
         subprocess.run(["chattr", f"{op}i", dst], check=True)
 
diff --git a/stages/org.osbuild.copy b/stages/org.osbuild.copy
index 90766a14..433e592f 100755
--- a/stages/org.osbuild.copy
+++ b/stages/org.osbuild.copy
@@ -1,65 +1,17 @@
 #!/usr/bin/python3
-import os
 import subprocess
 import sys
-from typing import Dict
-from urllib.parse import ParseResult, urlparse
 
 import osbuild.api
-
-
-def parse_mount(url: ParseResult, args: Dict):
-    name = url.netloc
-    if name:
-        root = args["mounts"].get(name, {}).get("path")
-        if not root:
-            raise ValueError(f"Unknown mount '{name}'")
-    else:
-        root = args["paths"]["mounts"]
-
-    return root
-
-
-def parse_input(url: ParseResult, args: Dict):
-    name = url.netloc
-    root = args["inputs"].get(name, {}).get("path")
-    if root is None:
-        raise ValueError(f"Unknown input '{name}'")
-
-    return root
-
-
-def parse_location(location, args):
-    url = urlparse(location)
-
-    scheme = url.scheme
-    if scheme == "tree":
-        root = args["tree"]
-    elif scheme == "mount":
-        root = parse_mount(url, args)
-    elif scheme == "input":
-        root = parse_input(url, args)
-    else:
-        raise ValueError(f"Unsupported scheme '{scheme}'")
-
-    assert url.path.startswith("/")
-
-    path = os.path.relpath(url.path, "/")
-    path = os.path.join(root, path)
-    path = os.path.normpath(path)
-
-    if url.path.endswith("/"):
-        path = os.path.join(path, ".")
-
-    return path
+from osbuild.util import parsing
 
 
 def main(args, options):
     items = options["paths"]
 
     for path in items:
-        src = parse_location(path["from"], args)
-        dst = parse_location(path["to"], args)
+        src = parsing.parse_location(path["from"], args)
+        dst = parsing.parse_location(path["to"], args)
         remove_destination = path.get("remove_destination", False)
 
         print(f"copying '{src}' -> '{dst}'")
diff --git a/stages/org.osbuild.kernel-cmdline.bls-append b/stages/org.osbuild.kernel-cmdline.bls-append
index eb6dcf44..72873498 100755
--- a/stages/org.osbuild.kernel-cmdline.bls-append
+++ b/stages/org.osbuild.kernel-cmdline.bls-append
@@ -1,30 +1,19 @@
 #!/usr/bin/python3
 import sys
-from urllib.parse import urlparse
 
 import osbuild.api
-from osbuild.util import bls
+from osbuild.util import bls, parsing
 
 
-def main(paths, tree, options):
+def main(args, options):
     kopts = options.get("kernel_opts", [])
     bootpath = options.get("bootpath", "tree:///boot")
 
-    url = urlparse(bootpath)
-    scheme = url.scheme
-    if scheme == "tree":
-        root = tree
-    elif scheme == "mount":
-        root = paths["mounts"]
-    else:
-        raise ValueError(f"Unsupported scheme '{scheme}'")
-
-    assert url.path.startswith("/")
-    bootroot = root + url.path
+    bootroot = parsing.parse_location(bootpath, args)
     bls.options_append(bootroot, kopts)
 
 
 if __name__ == '__main__':
-    args = osbuild.api.arguments()
-    r = main(args["paths"], args["tree"], args["options"])
+    _args = osbuild.api.arguments()
+    r = main(_args, _args["options"])
     sys.exit(r)
diff --git a/tox.ini b/tox.ini
index dbaf7a94..505719c5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -28,7 +28,7 @@ setenv =
     LINTABLES_EXCLUDES = "*.json,*.sh"
     LINTABLES_EXCLUDES_RE = ".*\.json$,.*\.sh"
     TYPEABLES = osbuild
-    TYPEABLES_STRICT = ./osbuild/main_cli.py
+    TYPEABLES_STRICT = ./osbuild/main_cli.py ./osbuild/util/parsing.py
 
 passenv =
     TEST_CATEGORY


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c2537e5a..20bcd45d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -7,8 +7,9 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  # Using 4 workers is a bit arbitrary, "auto" is probably too aggressive.
-  TEST_WORKERS: "-n 4"
+  # setting workers to 1 just to have the same syntax in all test
+  # but to disable parallelism for the majority who have concurrency issues
+  TEST_WORKERS: "-n 1"
   # Share the store between the workers speeds things up further
   OSBUILD_TEST_STORE: /var/tmp/osbuild-test-store
 
@@ -20,19 +21,25 @@ jobs:
       fail-fast: false
       matrix:
         test:
-        - "test.mod"
-        - "test.run.test_boot"
-        - "test.run.test_devices"
-        - "test.run.test_executable"
-        - "test.run.test_mount"
-        - "test.run.test_noop"
-        - "test.run.test_sources"
-        - "test.run.test_stages"
-        - "stages/test"
+          - "test.mod"
+          - "test.run.test_boot"
+          - "test.run.test_devices"
+          - "test.run.test_executable"
+          - "test.run.test_mount"
+          - "test.run.test_noop"
+          - "test.run.test_sources"
+          - "test.run.test_stages"
+          - "stages/test"
         environment:
-        - "py36"    # RH8
-        - "py39"    # RH9
-        - "py312"   # latest fedora
+          - "py36"    # RH8
+          - "py39"    # RH9
+          - "py312"   # latest fedora
+        # special keyword "include"
+        include:
+          # this test _can_ run in parallel
+          # Using 4 workers is a bit arbitrary, "auto" is probably too aggressive.
+          - test: "test.run.test_stages"
+            env_TEST_WORKERS: "-n 4"
     steps:
     - name: "Clone Repository"
       uses: actions/checkout@v3
@@ -47,6 +54,8 @@ jobs:
           sed -i 's/overlay/vfs/g' /usr/share/containers/storage.conf  # default system config
           sed -i 's/overlay/vfs/g' /etc/containers/storage.conf || true  # potential overrides
           TEST_CATEGORY="${{ matrix.test }}" \
+          TEST_WORKERS="${{ matrix.env_TEST_WORKERS || env.TEST_WORKERS }}" \
+          OSBUILD_TEST_STORE="${{ env.OSBUILD_TEST_STORE }}" \
           tox -e "${{ matrix.environment }}"
 
   v1_manifests:
@@ -57,8 +66,13 @@ jobs:
         uses: actions/checkout@v4
       - name: "Run"
         uses: osbuild/containers/src/actions/privdocker@552e30cf1b4ed19c6ddaa57f96c342b3dff4227b
+        env:
+          # Using 4 workers is a bit arbitrary, "auto" is probably too aggressive.
+          TEST_WORKERS: "-n 4"
         with:
           image: ghcr.io/osbuild/osbuild-ci:latest-202308241910
           run: |
             TEST_CATEGORY="test.run.test_assemblers" \
+            TEST_WORKERS="${{ env.TEST_WORKERS }}" \
+            OSBUILD_TEST_STORE="${{ env.OSBUILD_TEST_STORE }}" \
             tox -e "py36"
diff --git a/test/mod/test_util_parsing.py b/test/mod/test_util_parsing.py
index faaa31a5..e501310c 100644
--- a/test/mod/test_util_parsing.py
+++ b/test/mod/test_util_parsing.py
@@ -37,3 +37,41 @@ def test_parse_size():
         else:
             res = parsing.parse_size(s)
             assert res == num, f"{s} parsed as {res} (wanted {num})"
+
+
+def test_parse_location_mounts():
+    args = {
+        "paths": {
+            "mounts": "/run/osbuild/mounts",
+        },
+        "mounts": {
+            "root": {
+                "path": "/run/osbuild/mounts/.",
+            },
+        },
+    }
+    location = "mount://root/"
+    path = parsing.parse_location(location, args)
+    assert path == "/run/osbuild/mounts/."
+
+
+def test_parse_location_tree():
+    args = {
+        "tree": "/run/osbuild/tree",
+    }
+    location = "tree:///disk.img"
+    path = parsing.parse_location(location, args)
+    assert path == "/run/osbuild/tree/disk.img"
+
+
+def test_parse_location_inputs():
+    args = {
+        "inputs": {
+            "tree": {
+                "path": "/run/osbuild/inputs/tree",
+            },
+        },
+    }
+    location = "input://tree/"
+    path = parsing.parse_location(location, args)
+    assert path == "/run/osbuild/inputs/tree/."