Repository: MatterMiners/tardis

Issue:
Add -spool option in C/T config file 
We are planning to integrate our resources using Cobald/Tardis, our resources uses HTCondor as underlying mechanism.
Currently we are not able to integrate, as our resources does not include shared file system.  We tried to submit jobs using -spool option in condor_submit to transfer jobs to worker nodes. eg condor_submit -spool test.jdl.
Using the -spool option we were able to transfer files to worker nodes.
In this configuration file of cobald.yml, could you please add this option, so that will be helpful to integrate our system.
Please let me know, if you need any extra information.

Discussion:

Patch:
diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml
index c236475..8c1f917 100644
--- a/.github/workflows/docker-builds.yml
+++ b/.github/workflows/docker-builds.yml
@@ -59,7 +59,7 @@ jobs:
             password: ${{ secrets.DOCKERHUB_TOKEN }}
         - name: Build
           id: docker_build
-          uses: docker/build-push-action@v6
+          uses: docker/build-push-action@v5
           with:
             context: containers/${{ matrix.containers }}
             push: ${{ github.repository == 'MatterMiners/tardis' && github.event_name != 'pull_request' }}
diff --git a/docs/source/adapters/site.rst b/docs/source/adapters/site.rst
index 49d44d4..79f34b8 100644
--- a/docs/source/adapters/site.rst
+++ b/docs/source/adapters/site.rst
@@ -213,9 +213,22 @@ Available adapter configuration options
     |                | Default: ShellExecutor is used!                                                   |                 |
     +----------------+-----------------------------------------------------------------------------------+-----------------+
 
-    The only available option in the `MachineTypeConfiguration` section is a template jdl used to submit drones to the
-    HTCondor batch system. The template jdl is using the `Python template string`_ syntax
-    (see example HTCondor JDL for details).
+
+Available machine type configuration options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. content-tabs:: left-col
+
+    +----------------+------------------------------------------------------------------------------+-----------------+
+    | Option         | Short Description                                                            | Requirement     |
+    +================+==============================================================================+=================+
+    | jdl            | Path to the templated jdl used to submit drones to the HTCondor batch system |  **Required**   |
+    +----------------+------------------------------------------------------------------------------+-----------------+
+    | SubmitOptions  | Options to add to the `condor_submit` command. (see example)                 |  **Optional**   |
+    +----------------+------------------------------------------------------------------------------+-----------------+
+
+    .. Note::
+        The template jdl is using the `Python template string`_ syntax (see example HTCondor JDL for details).
 
     .. Warning::
         The `$(...)` used for HTCondor variables needs to be replaced by `$$(...)` in the templated JDL.
@@ -245,14 +258,24 @@ Available adapter configuration options
           max_age: 1
           MachineTypes:
             - wholenode
+            - remotenode
           MachineTypeConfiguration:
             wholenode:
               jdl: pilot_wholenode.jdl
+            remotenode:
+              jdl: pilot_remotenode.jdl
+              SubmitOptions:
+                spool: null
+                pool: remote-pool.somewhere.de
           MachineMetaData:
             wholenode:
               Cores: 42
               Memory: 256
               Disk: 840
+            remotenode:
+              Cores: 8
+              Memory: 20
+              Disk: 100
 
     .. rubric:: Example HTCondor JDL (Vanilla Universe)
 
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 5680189..6b6b6b4 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,4 +1,4 @@
-.. Created by changelog.py at 2024-05-24, command
+.. Created by changelog.py at 2024-10-16, command
    '/Users/giffler/.cache/pre-commit/repoecmh3ah8/py_env-python3.12/bin/changelog docs/source/changes compile --categories Added Changed Fixed Security Deprecated --output=docs/source/changelog.rst'
    based on the format of 'https://keepachangelog.com/'
 
@@ -6,6 +6,14 @@
 CHANGELOG
 #########
 
+[Unreleased] - 2024-10-16
+=========================
+
+Added
+-----
+
+* Add support for submit options to HTCondor site adapter
+
 [0.8.2] - 2024-05-15
 ====================
 
diff --git a/docs/source/changes/354.add_support_for_submit_options_to_HTCondor_site_adapter.yaml b/docs/source/changes/354.add_support_for_submit_options_to_HTCondor_site_adapter.yaml
new file mode 100644
index 0000000..e1e8511
--- /dev/null
+++ b/docs/source/changes/354.add_support_for_submit_options_to_HTCondor_site_adapter.yaml
@@ -0,0 +1,9 @@
+category: added
+summary: "Add support for submit options to HTCondor site adapter"
+description: |
+  Add support for additional submit options to the `condor_submit` command used to submit HTCondor batch jobs in the 
+  HTCondor site adapter. For example: `-spool` to transfer input files to a remote HTCondor Schedd for job submission.
+pull requests:
+  - 354
+issues:
+  - 353
diff --git a/tardis/adapters/sites/htcondor.py b/tardis/adapters/sites/htcondor.py
index 4e1e6f4..d6552e3 100644
--- a/tardis/adapters/sites/htcondor.py
+++ b/tardis/adapters/sites/htcondor.py
@@ -12,6 +12,7 @@ from ...utilities.asyncbulkcall import AsyncBulkCall
 from ...utilities.utils import (
     csv_parser,
     drone_environment_to_str,
+    htcondor_cmd_option_formatter,
     machine_meta_data_translation,
 )
 
@@ -44,7 +45,9 @@ async def condor_q(
         _job_id(resource.remote_resource_uuid) for resource in resource_attributes
     )
 
-    queue_command = f"condor_q {remote_resource_ids} -af:t {attributes_string}"
+    queue_command = (
+        f"condor_q {remote_resource_ids} -af:t {attributes_string}"  # noqa E231
+    )
 
     htcondor_queue = {}
     try:
@@ -97,7 +100,9 @@ def _submit_description(resource_jdls: Tuple[JDL, ...]) -> str:
     return "\n".join(commands)
 
 
-async def condor_submit(*resource_jdls: JDL, executor: Executor) -> Iterable[str]:
+async def condor_submit(
+    *resource_jdls: JDL, executor: Executor, submit_option_string: str
+) -> Iterable[str]:
     """Submit a number of resources from their JDL, reporting the new Job ID for each"""
     # verbose submit gives an ordered listing of class ads, such as
     # ** Proc 15556.0:
@@ -110,7 +115,9 @@ async def condor_submit(*resource_jdls: JDL, executor: Executor) -> Iterable[str
     #
     # ** Proc 15556.1:
     # ...
-    command = f"condor_submit -verbose -maxjobs {len(resource_jdls)}"
+    command = (
+        f"condor_submit -verbose -maxjobs {len(resource_jdls)} {submit_option_string}"
+    )
     response = await executor.run_command(
         command,
         stdin_input=_submit_description(resource_jdls),
@@ -213,20 +220,30 @@ class HTCondorAdapter(SiteAdapter):
         self._machine_type = machine_type
         self._site_name = site_name
         self._executor = getattr(self.configuration, "executor", ShellExecutor())
+
+        submit_option_string = htcondor_cmd_option_formatter(
+            self.machine_type_configuration.get("SubmitOptions", AttributeDict())
+        )
+
         bulk_size = getattr(self.configuration, "bulk_size", 100)
         bulk_delay = getattr(self.configuration, "bulk_delay", 1.0)
-        self._condor_submit, self._condor_suspend, self._condor_rm = (
+
+        self._condor_submit = AsyncBulkCall(
+            partial(
+                condor_submit,
+                executor=self._executor,
+                submit_option_string=submit_option_string,
+            ),
+            size=bulk_size,
+            delay=bulk_delay,
+        )
+        self._condor_suspend, self._condor_rm, self._condor_q = (
             AsyncBulkCall(
                 partial(tool, executor=self._executor),
                 size=bulk_size,
                 delay=bulk_delay,
             )
-            for tool in (condor_submit, condor_suspend, condor_rm)
-        )
-        self._condor_q = AsyncBulkCall(
-            partial(condor_q, executor=self._executor),
-            size=bulk_size,
-            delay=bulk_delay,
+            for tool in (condor_suspend, condor_rm, condor_q)
         )
 
         key_translator = StaticMapping(


Test patch:
diff --git a/.github/workflows/deployment-tests.yml b/.github/workflows/deployment-tests.yml
index a5ca1fb..f88c163 100644
--- a/.github/workflows/deployment-tests.yml
+++ b/.github/workflows/deployment-tests.yml
@@ -23,7 +23,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v5
         with:
           push: true
           file: containers/cobald-tardis-deployment-test-env/Dockerfile.${{ matrix.os_container }}
diff --git a/tests/adapters_t/sites_t/test_htcondorsiteadapter.py b/tests/adapters_t/sites_t/test_htcondorsiteadapter.py
index eeda206..d44cf59 100644
--- a/tests/adapters_t/sites_t/test_htcondorsiteadapter.py
+++ b/tests/adapters_t/sites_t/test_htcondorsiteadapter.py
@@ -139,6 +139,7 @@ class TestHTCondorSiteAdapter(TestCase):
             test2large_args=AttributeDict(Cores=8, Memory=32, Disk=160),
             test2large_deprecated=AttributeDict(Cores=8, Memory=32, Disk=160),
             testunkownresource=AttributeDict(Cores=8, Memory=32, Disk=160, Foo=3),
+            testsubmitoptions=AttributeDict(Cores=8, Memory=32, Disk=160),
         )
 
     @property
@@ -148,6 +149,10 @@ class TestHTCondorSiteAdapter(TestCase):
             test2large_args=AttributeDict(jdl="tests/data/submit_per_arguments.jdl"),
             test2large_deprecated=AttributeDict(jdl="tests/data/submit_deprecated.jdl"),
             testunkownresource=AttributeDict(jdl="tests/data/submit.jdl"),
+            testsubmitoptions=AttributeDict(
+                jdl="tests/data/submit.jdl",
+                SubmitOptions=AttributeDict(pool="my_remote_pool", spool=None),
+            ),
         )
 
     @mock_executor_run_command(stdout=CONDOR_SUBMIT_OUTPUT)
@@ -231,6 +236,28 @@ class TestHTCondorSiteAdapter(TestCase):
         )
         self.mock_executor.reset()
 
+        self.adapter = HTCondorAdapter(
+            machine_type="testsubmitoptions", site_name="TestSite"
+        )
+
+        # Test support for submit options
+        run_async(
+            self.adapter.deploy_resource,
+            AttributeDict(
+                drone_uuid="test-123",
+                obs_machine_meta_data_translation_mapping=AttributeDict(
+                    Cores=1,
+                    Memory=1024,
+                    Disk=1024 * 1024,
+                ),
+            ),
+        )
+
+        args, _ = self.mock_executor.return_value.run_command.call_args
+        self.assertEqual(
+            "condor_submit -verbose -maxjobs 1 -pool my_remote_pool -spool", args[0]
+        )
+
     def test_translate_resources_raises_logs(self):
         self.adapter = HTCondorAdapter(
             machine_type="testunkownresource", site_name="TestSite"