Repository: testflows/TestFlows-GitHub-Hetzner-Runners (language: python)

Issue:
Fix startup and setup scripts being set to None
When checking if scripts exist, they are being set to None.


Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index 126d111..d10c014 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 **/__pycache__
+wiki/
 dist/
-testflows.combinatorics.egg-info/
+testflows.*.egg-info/
diff --git a/examples/config.yaml b/examples/config.yaml
index 4330db0..c51a8f2 100644
--- a/examples/config.yaml
+++ b/examples/config.yaml
@@ -12,9 +12,7 @@ config:
     default_server_type: cpx11
     #default_location:
     workers: 10
-    #setup_script:
-    #startup_x64_script:
-    #startup_arm64_script:
+    #scripts:
     max_powered_off_time: 60
     max_unused_runner_time: 120
     max_runner_registration_time: 120


Test patch:
diff --git a/testflows/github/hetzner/runners/__init__.py b/testflows/github/hetzner/runners/__init__.py
index e010d6a..34ad613 100644
--- a/testflows/github/hetzner/runners/__init__.py
+++ b/testflows/github/hetzner/runners/__init__.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 __author__ = "Vitaliy Zakaznikov"
-__version__ = "1.5.__VERSION__"
+__version__ = "1.6.__VERSION__"
 __license__ = f"""
 Copyright 2023 Katteli Inc.
 TestFlows.com Open-Source Software Testing Framework (http://testflows.com)
diff --git a/testflows/github/hetzner/runners/args.py b/testflows/github/hetzner/runners/args.py
index 390eed1..6bf8786 100644
--- a/testflows/github/hetzner/runners/args.py
+++ b/testflows/github/hetzner/runners/args.py
@@ -129,6 +129,14 @@ def server_type(v):
     return ServerType(name=v)
 
 
+def meta_labels_type(v):
+    """Meta labels type argument."""
+    try:
+        return {l[0]: set(l[1].split(",") if l[1] else []) for l in v}
+    except Exception as e:
+        raise ArgumentTypeError(str(e))
+
+
 def config_type(v):
     """Program configuration file type."""
     from .config import parse_config, default_user_config
@@ -146,6 +154,8 @@ def config_type(v):
     except Exception as e:
         if "--debug" in sys.argv:
             print_exception(e)
+        if "unexpected keyword argument" in str(e):
+            e = str(e).replace(".__init__()", "") + ", please remove it"
         raise ArgumentTypeError(str(e))
 
     return config
diff --git a/testflows/github/hetzner/runners/bin/github-hetzner-runners b/testflows/github/hetzner/runners/bin/github-hetzner-runners
index f89c517..bbc0364 100755
--- a/testflows/github/hetzner/runners/bin/github-hetzner-runners
+++ b/testflows/github/hetzner/runners/bin/github-hetzner-runners
@@ -31,7 +31,7 @@ from hcloud.ssh_keys.domain import SSHKey
 from github import Github
 from github.Repository import Repository
 
-from argparse import ArgumentParser, RawTextHelpFormatter
+from argparse import ArgumentParser, RawTextHelpFormatter, ArgumentTypeError
 
 from testflows.github.hetzner.runners import __version__, __license__
 from testflows.github.hetzner.runners.actions import Action
@@ -184,7 +184,16 @@ def argparser():
         "--label-prefix",
         metavar="prefix",
         type=str,
-        help="support type, image, and location job labels with the specified prefix",
+        help="support type, image, location, setup, and startup job labels with the specified prefix",
+    )
+
+    parser.add_argument(
+        "--meta-labels",
+        metavar="name label,... --max-labels",
+        type=str,
+        action='append',
+        nargs=2,
+        help="runner meta label that will be expanded to a list of given labels",
     )
 
     parser.add_argument(
@@ -224,24 +233,10 @@ def argparser():
     )
 
     parser.add_argument(
-        "--setup-script",
+        "--scripts",
         metavar="path",
         type=args.path_type,
-        help="path to custom server setup script",
-    )
-
-    parser.add_argument(
-        "--startup-x64-script",
-        metavar="path",
-        type=args.path_type,
-        help="path to custom x64 server startup script",
-    )
-
-    parser.add_argument(
-        "--startup-arm64-script",
-        metavar="path",
-        type=args.path_type,
-        help="path to custom ARM64 server startup script",
+        help="path to the folder that contains custom server scripts",
     )
 
     parser.add_argument(
@@ -920,16 +915,23 @@ def main(config, worker_pool: ThreadPoolExecutor, terminate_timeout=60):
 
 
 if __name__ == "__main__":
-    args = argparser().parse_args()
+    parser = argparser()
+    arguments = parser.parse_args()
 
     logging_level = logging.INFO
 
-    if args.config_file:
-        config = args.config_file
+    if arguments.config_file:
+        config = arguments.config_file
     else:
         config = Config()
 
-    config.update(args)
+    if arguments.meta_labels:
+        try:
+            arguments.meta_labels = args.meta_labels_type(arguments.meta_labels)
+        except ArgumentTypeError as e:
+            parser.error(str(e))
+
+    config.update(arguments)
 
     if config.debug:
         Action.debug = True
@@ -946,9 +948,9 @@ if __name__ == "__main__":
             sys.stdout.write(f"Using config file: {config.config_file}\n")
             sys.stdout.flush()
 
-    if hasattr(args, "func"):
+    if hasattr(arguments, "func"):
         try:
-            args.func(args=args, config=config)
+            arguments.func(args=arguments, config=config)
         except KeyboardInterrupt:
             if config.debug:
                 raise
diff --git a/testflows/github/hetzner/runners/cloud.py b/testflows/github/hetzner/runners/cloud.py
index 1ba8086..417fc28 100644
--- a/testflows/github/hetzner/runners/cloud.py
+++ b/testflows/github/hetzner/runners/cloud.py
@@ -151,45 +151,21 @@ def deploy(args, config: Config, redeploy=False):
     with Action("Copying any custom scripts"):
         ip = ip_address(server)
 
-        if config.setup_script:
-            with Action(f"Copying custom setup script {config.setup_script}"):
+        if config.scripts:
+            with Action(f"Copying custom scripts {config.scripts}"):
                 scp(
-                    source=config.setup_script,
+                    source=os.path.join(config.scripts, "*.sh"),
                     destination=f"root@{ip}:{deploy_scripts_folder}.",
                 )
-                config.setup_script = os.path.join(
-                    deploy_scripts_folder,
-                    os.path.basename(config.setup_script),
-                )
-
-        if config.startup_x64_script:
-            with Action(
-                f"Copying custom startup x64 script {config.startup_x64_script}"
-            ):
-                scp(
-                    source=config.startup_x64_script,
-                    destination=f"root@{ip}:{deploy_scripts_folder}.",
-                )
-                config.startup_x64_script = os.path.join(
-                    deploy_scripts_folder,
-                    os.path.basename(config.startup_x64_script),
-                )
-
-        if config.startup_arm64_script:
-            with Action(
-                f"Copying custom startup ARM64 script {config.startup_arm64_script}"
-            ):
-                scp(
-                    source=config.startup_arm64_script,
-                    destination=f"root@{ip}:{deploy_scripts_folder}.",
-                )
-                config.startup_arm64_script = os.path.join(
-                    deploy_scripts_folder,
-                    os.path.basename(config.startup_arm64_script),
-                )
+                config.scripts = deploy_scripts_folder
 
     with Action("Fixing ownership of any copied scripts"):
         ssh(server, f"chown -R ubuntu:ubuntu {deploy_scripts_folder}", stacklevel=4)
+        ssh(
+            server,
+            f"find {deploy_scripts_folder} -type f -exec chmod +rx {{}} \;",
+            stacklevel=4,
+        )
 
     with Action(
         f"Copying config file{(' ' + config.config_file) if config.config_file else ''}"
diff --git a/testflows/github/hetzner/runners/config/__init__.py b/testflows/github/hetzner/runners/config/__init__.py
index 74c4b9a..c3de0c3 100644
--- a/testflows/github/hetzner/runners/config/__init__.py
+++ b/testflows/github/hetzner/runners/config/__init__.py
@@ -19,6 +19,8 @@
     check_prices,
     check_server_type,
     check_ssh_key,
+    check_setup_script,
+    check_startup_script,
 )
 from .config import standby_runner
 from .config import read, write
diff --git a/testflows/github/hetzner/runners/config/config.py b/testflows/github/hetzner/runners/config/config.py
index 2a40dc1..2ca48e5 100644
--- a/testflows/github/hetzner/runners/config/config.py
+++ b/testflows/github/hetzner/runners/config/config.py
@@ -32,7 +32,12 @@
 def env_constructor(loader, node):
     value = loader.construct_scalar(node)
     for group in env_pattern.findall(value):
-        value = value.replace(f"${{{group}}}", os.environ.get(group))
+        env_value = os.environ.get(group)
+        if env_value is None:
+            assert (
+                False
+            ), f"environment variable ${group} used in the config is not defined"
+        value = value.replace(f"${{{group}}}", env_value)
     return value
 
 
@@ -48,6 +53,14 @@ class ImageError(Exception):
     pass
 
 
+class SetupScriptError(Exception):
+    pass
+
+
+class StartupScriptError(Exception):
+    pass
+
+
 class ServerTypeError(Exception):
     pass
 
@@ -62,6 +75,7 @@ class ConfigError(Exception):
 location = args.location_type
 server_type = args.server_type
 end_of_life = args.end_of_life_type
+meta_labels_type = args.meta_labels_type
 
 
 @dataclass
@@ -96,6 +110,7 @@ class Config:
     additional_ssh_keys: list[str] = None
     with_label: str = None
     label_prefix: str = ""
+    meta_labels: dict[str, set[str]] = None
     recycle: bool = True
     end_of_life: int = 50
     delete_random: bool = False
@@ -105,13 +120,7 @@ class Config:
     default_server_type: ServerType = server_type("cx11")
     default_location: Location = None
     workers: int = 10
-    setup_script: str = os.path.join(current_dir, "..", "scripts", "setup.sh")
-    startup_x64_script: str = os.path.join(
-        current_dir, "..", "scripts", "startup_x64.sh"
-    )
-    startup_arm64_script: str = os.path.join(
-        current_dir, "..", "scripts", "startup_arm64.sh"
-    )
+    scripts: str = os.path.join(current_dir, "..", "scripts")
     max_powered_off_time: int = 60
     max_unused_runner_time: int = 180
     max_runner_registration_time: int = 180
@@ -136,6 +145,9 @@ def __post_init__(self):
         if self.additional_ssh_keys is None:
             self.additional_ssh_keys = []
 
+        if self.meta_labels is None:
+            self.meta_labels = {}
+
         if self.logger_format is None:
             self.logger_format = logger_format
 
@@ -240,6 +252,23 @@ def parse_config(filename: str):
             doc["label_prefix"], str
         ), "config.label_prefix: is not a string"
 
+    if doc.get("meta_labels") is not None:
+        assert isinstance(
+            doc["meta_labels"], dict
+        ), "config.meta_labels is not a dictionary"
+        for i, meta in enumerate(doc["meta_labels"]):
+            assert isinstance(
+                meta, str
+            ), f"config.meta_labels.{meta}: name is not a string"
+            assert isinstance(
+                doc["meta_labels"][meta], list
+            ), f"config.meta_labels.{meta}: is not a list"
+            for j, v in enumerate(doc["meta_labels"][meta]):
+                assert isinstance(
+                    v, str
+                ), f"config.meta_labels.{meta}[{j}]: is not a string"
+            doc["meta_labels"][meta] = set(doc["meta_labels"][meta])
+
     if doc.get("recycle") is not None:
         assert isinstance(doc["recycle"], bool), "config.recycle: is not a boolean"
 
@@ -287,23 +316,11 @@ def parse_config(filename: str):
         v = doc["workers"]
         assert isinstance(v, int) and v > 0, "config.workers: is not an integer > 0"
 
-    if doc.get("setup_script") is not None:
-        try:
-            doc["setup_script"] = path(doc["setup_script"])
-        except Exception as e:
-            assert False, f"config.setup_script: {e}"
-
-    if doc.get("startup_x64_script") is not None:
-        try:
-            doc["startup_x64_script"] = path(doc["startup_x64_script"])
-        except Exception as e:
-            assert False, f"config.startup_x64_script: {e}"
-
-    if doc.get("startup_arm64_script") is not None:
+    if doc.get("scripts") is not None:
         try:
-            doc["startup_arm64_script"] = path(doc["startup_arm64_script"])
+            doc["scripts"] = path(doc["scripts"])
         except Exception as e:
-            assert False, f"config.startup_arm64_script: {e}"
+            assert False, f"config.scripts: {e}"
 
     if doc.get("max_powered_off_time") is not None:
         v = doc["max_powered_off_time"]
@@ -351,7 +368,7 @@ def parse_config(filename: str):
         assert (
             doc["logger_config"]["loggers"].get("testflows.github.hetzner.runners")
             is not None
-        ), 'config.logger_config.loggers."tesflows.github.hetzner.runners" is not defined'
+        ), 'config.logger_config.loggers."testflows.github.hetzner.runners" is not defined'
         assert (
             doc["logger_config"]["loggers"]["testflows.github.hetzner.runners"].get(
                 "handlers"
@@ -628,6 +645,7 @@ def check_server_type(client: Client, server_type: ServerType):
 
 
 def check_prices(client: Client):
+    """Check server prices."""
     server_types: list[ServerType] = client.server_types.get_all()
     return {
         t.name.lower(): {
@@ -636,3 +654,17 @@ def check_prices(client: Client):
         }
         for t in server_types
     }
+
+
+def check_setup_script(script: str):
+    """Check if setup script is valid."""
+    if not os.path.exists(script):
+        raise SetupScriptError(f"invalid setup script path '{script}'")
+    return script
+
+
+def check_startup_script(script: str):
+    """Check if startup script is valid."""
+    if not os.path.exists(script):
+        raise StartupScriptError(f"invalid startup script path '{script}'")
+    return script
diff --git a/testflows/github/hetzner/runners/config/schema.json b/testflows/github/hetzner/runners/config/schema.json
index 07d45c0..75fea51 100644
--- a/testflows/github/hetzner/runners/config/schema.json
+++ b/testflows/github/hetzner/runners/config/schema.json
@@ -46,6 +46,15 @@
                     "description": "Support type, image, and location job labels with the specified prefix.",
                     "type": "string"
                 },
+                "meta_labels": {
+                    "description": "Meta labels.",
+                    "type": "object",
+                    "additionalProperties": {
+                      "type": "array",
+                      "items": {
+                        "type": "string"
+                    }
+                },
                 "recycle": {
                     "description": "Turn on recycling of powered off servers, either 'true' or 'false', default: true.",
                     "type": "boolean",
@@ -91,16 +100,8 @@
                     "type": "integer",
                     "examples": [10]
                 },
-                "setup_script": {
-                    "description": "Path to custom server setup script.",
-                    "type": "string"
-                },
-                "startup_x64_script": {
-                    "description": "Path to custom x64 server startup script.",
-                    "type": "string"
-                },
-                "startup_arm64_script": {
-                    "description": "Path to custom ARM64 server startup script.",
+                "scripts": {
+                    "description": "Path to the folder that contains custom server scripts.",
                     "type": "string"
                 },
                 "max_powered_off_time": {
diff --git a/testflows/github/hetzner/runners/scale_up.py b/testflows/github/hetzner/runners/scale_up.py
index 6b271ca..185681f 100644
--- a/testflows/github/hetzner/runners/scale_up.py
+++ b/testflows/github/hetzner/runners/scale_up.py
@@ -24,7 +24,7 @@
 from .request import request
 from .args import image_type
 from .logger import logger
-from .config import Config, check_image
+from .config import Config, check_image, check_startup_script, check_setup_script
 from .config import standby_runner as StandbyRunner
 
 from .server import wait_ssh, ssh, wait_ready, age
@@ -140,11 +140,10 @@ def get_server_type(labels: set[str], default: ServerType, label_prefix: str = "
         label_prefix += "-"
     label_prefix += "type-"
 
-    if server_type is None:
-        for label in labels:
-            if label.startswith(label_prefix):
-                server_type_name = label.split(label_prefix, 1)[-1].lower()
-                server_type = ServerType(name=server_type_name)
+    for label in labels:
+        if label.startswith(label_prefix):
+            server_type_name = label.split(label_prefix, 1)[-1].lower()
+            server_type = ServerType(name=server_type_name)
 
     if server_type is None:
         server_type = default
@@ -166,11 +165,10 @@ def get_server_location(
         label_prefix += "-"
     label_prefix += "in-"
 
-    if server_location is None:
-        for label in labels:
-            if label.startswith(label_prefix):
-                server_location_name = label.split(label_prefix, 1)[-1].lower()
-                server_location = Location(name=server_location_name)
+    for label in labels:
+        if label.startswith(label_prefix):
+            server_location_name = label.split(label_prefix, 1)[-1].lower()
+            server_location = Location(name=server_location_name)
 
     if server_location is None and default:
         server_location = default
@@ -188,13 +186,12 @@ def get_server_image(
         label_prefix += "-"
     label_prefix += "image-"
 
-    if server_image is None:
-        for label in labels:
-            if label.startswith(label_prefix):
-                server_image = check_image(
-                    client,
-                    image_type(label.split(label_prefix, 1)[-1].lower(), separator="-"),
-                )
+    for label in labels:
+        if label.startswith(label_prefix):
+            server_image = check_image(
+                client,
+                image_type(label.split(label_prefix, 1)[-1].lower(), separator="-"),
+            )
 
     if server_image is None:
         server_image = default
@@ -202,15 +199,84 @@ def get_server_image(
     return server_image
 
 
-def get_startup_script(server_type: ServerType, x64, arm64):
-    """Get startup script based on the requested server type.
+def get_server_arch(server_type: ServerType):
+    """Get server architecture base on the requested server type.
     ARM64 servers type names start with "CA" prefix.
 
     For example, CAX11, CAX21, CAX31, and CAX41
     """
     if server_type.name.lower().startswith("ca"):
-        return arm64
-    return x64
+        return "arm64"
+    return "x64"
+
+
+def get_setup_script(
+    scripts: str, labels: set[str], default: str = "setup.sh", label_prefix: str = ""
+):
+    """Get setup script."""
+    script = None
+
+    if label_prefix and not label_prefix.endswith("-"):
+        label_prefix += "-"
+    label_prefix += "setup-"
+
+    for label in labels:
+        if label.startswith(label_prefix):
+            script = label.split(label_prefix, 1)[-1] + ".sh"
+
+    if script is None:
+        script = default
+
+    script = check_setup_script(os.path.join(scripts, script))
+
+    return script
+
+
+def get_startup_script(
+    scripts: str,
+    server_type: ServerType,
+    labels: set[str],
+    default: str = "startup-{arch}.sh",
+    label_prefix: str = "",
+):
+    """Get startup script based on the requested server type.
+    ARM64 servers type names start with "CA" prefix.
+
+    For example, CAX11, CAX21, CAX31, and CAX41
+    """
+    script = None
+    default = default.format(arch=get_server_arch(server_type))
+
+    if label_prefix and not label_prefix.endswith("-"):
+        label_prefix += "-"
+    label_prefix += "startup-"
+
+    for label in labels:
+        if label.startswith(label_prefix):
+            script = label.split(label_prefix, 1)[-1] + ".sh"
+
+    if script is None:
+        script = default
+
+    script = check_startup_script(os.path.join(scripts, script))
+
+    return script
+
+
+def expand_meta_labels(
+    meta_labels: dict[str, set[str]], labels: set[str], label_prefix: str = ""
+):
+    """Expand any meta labels."""
+    expanded_labels = []
+
+    for label in labels:
+        expanded_labels.append(label)
+        if label.startswith(label_prefix):
+            raw_label = label.split(label_prefix, 1)[-1] if label_prefix else label
+            if raw_label in meta_labels:
+                expanded_labels += list(meta_labels[label])
+
+    return set(expanded_labels)
 
 
 def raise_exception(exc):
@@ -439,6 +505,8 @@ def scale_up(
     recycle: bool = config.recycle
     with_label: str = config.with_label
     label_prefix: str = config.label_prefix
+    meta_labels: dict[str, set[str]] = config.meta_labels
+    scripts: str = config.scripts
     interval: int = -1
 
     with Action("Logging in to Hetzner Cloud"):
@@ -454,6 +522,8 @@ def create_runner_server(
         """Create new server that would provide a runner with given labels."""
         recyclable_servers: list[BoundServer] = []
 
+        labels = expand_meta_labels(meta_labels=meta_labels, labels=labels)
+
         server_type = get_server_type(
             labels=labels, default=default_server_type, label_prefix=label_prefix
         )
@@ -466,11 +536,16 @@ def create_runner_server(
             default=default_image,
             label_prefix=label_prefix,
         )
-        setup_script = config.setup_script
+        setup_script = get_setup_script(
+            scripts=scripts,
+            labels=labels,
+            label_prefix=label_prefix,
+        )
         startup_script = get_startup_script(
+            scripts=scripts,
             server_type=server_type,
-            x64=config.startup_x64_script,
-            arm64=config.startup_arm64_script,
+            labels=labels,
+            label_prefix=label_prefix,
         )
 
         with Action(
diff --git a/testflows/github/hetzner/runners/scripts/none.sh b/testflows/github/hetzner/runners/scripts/none.sh
new file mode 100644
index 0000000..e69de29
diff --git a/testflows/github/hetzner/runners/scripts/setup_docker.sh b/testflows/github/hetzner/runners/scripts/setup-docker.sh
similarity index 100%
rename from testflows/github/hetzner/runners/scripts/setup_docker.sh
rename to testflows/github/hetzner/runners/scripts/setup-docker.sh
diff --git a/testflows/github/hetzner/runners/scripts/startup_arm64.sh b/testflows/github/hetzner/runners/scripts/startup-arm64.sh
similarity index 100%
rename from testflows/github/hetzner/runners/scripts/startup_arm64.sh
rename to testflows/github/hetzner/runners/scripts/startup-arm64.sh
diff --git a/testflows/github/hetzner/runners/scripts/startup_x64.sh b/testflows/github/hetzner/runners/scripts/startup-x64.sh
similarity index 100%
rename from testflows/github/hetzner/runners/scripts/startup_x64.sh
rename to testflows/github/hetzner/runners/scripts/startup-x64.sh
diff --git a/testflows/github/hetzner/runners/service.py b/testflows/github/hetzner/runners/service.py
index a9ed72a..52df987 100644
--- a/testflows/github/hetzner/runners/service.py
+++ b/testflows/github/hetzner/runners/service.py
@@ -14,7 +14,6 @@
 # limitations under the License.
 import os
 import sys
-import json
 import textwrap
 
 NAME = "github-hetzner-runners"
@@ -45,6 +44,12 @@ def command_options(
     command += f" --recycle " + ("on" if config.recycle else "off")
     command += f" --end-of-life {config.end_of_life}" if config.end_of_life else ""
     command += f" --with-label {config.with_label}" if config.with_label else ""
+    for k in config.meta_labels:
+        command += (
+            f" --meta-labels \"{k}\" \"{','.join(config.meta_labels[k])}\""
+            if config.meta_labels[k]
+            else ""
+        )
     command += f" --workers {config.workers}"
     command += f" --default-type {config.default_server_type.name}"
     command += (
@@ -59,17 +64,7 @@ def command_options(
         if config.max_runners_in_workflow_run
         else ""
     )
-    command += f" --setup-script {config.setup_script}" if config.setup_script else ""
-    command += (
-        f" --startup-x64-script {config.startup_x64_script}"
-        if config.startup_x64_script
-        else ""
-    )
-    command += (
-        f" --startup-arm64-script {config.startup_arm64_script}"
-        if config.startup_arm64_script
-        else ""
-    )
+    command += f" --scripts {config.scripts}" if config.scripts else ""
     command += (
         f" --max-powered-off-time {config.max_powered_off_time}"
         f" --max-unused-runner-time {config.max_unused_runner_time}"