Repository: MatterMiners/tardis

Issue:
Add -spool option in C/T config file 
We are planning to integrate our resources using Cobald/Tardis, our resources uses HTCondor as underlying mechanism.
Currently we are not able to integrate, as our resources does not include shared file system.  We tried to submit jobs using -spool option in condor_submit to transfer jobs to worker nodes. eg condor_submit -spool test.jdl.
Using the -spool option we were able to transfer files to worker nodes.
In this configuration file of cobald.yml, could you please add this option, so that will be helpful to integrate our system.
Please let me know, if you need any extra information.

Hints from issue comments:
@giffels: Thanks for the feature request. We are going to include it into our next release.

Solution patch:
diff --git a/docs/source/adapters/site.rst b/docs/source/adapters/site.rst
index 49d44d4c..79f34b8b 100644
--- a/docs/source/adapters/site.rst
+++ b/docs/source/adapters/site.rst
@@ -213,9 +213,22 @@ Available adapter configuration options
     |                | Default: ShellExecutor is used!                                                   |                 |
     +----------------+-----------------------------------------------------------------------------------+-----------------+
 
-    The only available option in the `MachineTypeConfiguration` section is a template jdl used to submit drones to the
-    HTCondor batch system. The template jdl is using the `Python template string`_ syntax
-    (see example HTCondor JDL for details).
+
+Available machine type configuration options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. content-tabs:: left-col
+
+    +----------------+------------------------------------------------------------------------------+-----------------+
+    | Option         | Short Description                                                            | Requirement     |
+    +================+==============================================================================+=================+
+    | jdl            | Path to the templated jdl used to submit drones to the HTCondor batch system |  **Required**   |
+    +----------------+------------------------------------------------------------------------------+-----------------+
+    | SubmitOptions  | Options to add to the `condor_submit` command. (see example)                 |  **Optional**   |
+    +----------------+------------------------------------------------------------------------------+-----------------+
+
+    .. Note::
+        The template jdl is using the `Python template string`_ syntax (see example HTCondor JDL for details).
 
     .. Warning::
         The `$(...)` used for HTCondor variables needs to be replaced by `$$(...)` in the templated JDL.
@@ -245,14 +258,24 @@ Available adapter configuration options
           max_age: 1
           MachineTypes:
             - wholenode
+            - remotenode
           MachineTypeConfiguration:
             wholenode:
               jdl: pilot_wholenode.jdl
+            remotenode:
+              jdl: pilot_remotenode.jdl
+              SubmitOptions:
+                spool: null
+                pool: remote-pool.somewhere.de
           MachineMetaData:
             wholenode:
               Cores: 42
               Memory: 256
               Disk: 840
+            remotenode:
+              Cores: 8
+              Memory: 20
+              Disk: 100
 
     .. rubric:: Example HTCondor JDL (Vanilla Universe)
 
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 56801890..6b6b6b40 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,4 +1,4 @@
-.. Created by changelog.py at 2024-05-24, command
+.. Created by changelog.py at 2024-10-16, command
    '/Users/giffler/.cache/pre-commit/repoecmh3ah8/py_env-python3.12/bin/changelog docs/source/changes compile --categories Added Changed Fixed Security Deprecated --output=docs/source/changelog.rst'
    based on the format of 'https://keepachangelog.com/'
 
@@ -6,6 +6,14 @@
 CHANGELOG
 #########
 
+[Unreleased] - 2024-10-16
+=========================
+
+Added
+-----
+
+* Add support for submit options to HTCondor site adapter
+
 [0.8.2] - 2024-05-15
 ====================
 
diff --git a/docs/source/changes/354.add_support_for_submit_options_to_HTCondor_site_adapter.yaml b/docs/source/changes/354.add_support_for_submit_options_to_HTCondor_site_adapter.yaml
new file mode 100644
index 00000000..e1e85118
--- /dev/null
+++ b/docs/source/changes/354.add_support_for_submit_options_to_HTCondor_site_adapter.yaml
@@ -0,0 +1,9 @@
+category: added
+summary: "Add support for submit options to HTCondor site adapter"
+description: |
+  Add support for additional submit options to the `condor_submit` command used to submit HTCondor batch jobs in the 
+  HTCondor site adapter. For example: `-spool` to transfer input files to a remote HTCondor Schedd for job submission.
+pull requests:
+  - 354
+issues:
+  - 353
diff --git a/tardis/adapters/sites/htcondor.py b/tardis/adapters/sites/htcondor.py
index 4e1e6f41..d6552e30 100644
--- a/tardis/adapters/sites/htcondor.py
+++ b/tardis/adapters/sites/htcondor.py
@@ -12,6 +12,7 @@
 from ...utilities.utils import (
     csv_parser,
     drone_environment_to_str,
+    htcondor_cmd_option_formatter,
     machine_meta_data_translation,
 )
 
@@ -44,7 +45,9 @@ async def condor_q(
         _job_id(resource.remote_resource_uuid) for resource in resource_attributes
     )
 
-    queue_command = f"condor_q {remote_resource_ids} -af:t {attributes_string}"
+    queue_command = (
+        f"condor_q {remote_resource_ids} -af:t {attributes_string}"  # noqa E231
+    )
 
     htcondor_queue = {}
     try:
@@ -97,7 +100,9 @@ def _submit_description(resource_jdls: Tuple[JDL, ...]) -> str:
     return "\n".join(commands)
 
 
-async def condor_submit(*resource_jdls: JDL, executor: Executor) -> Iterable[str]:
+async def condor_submit(
+    *resource_jdls: JDL, executor: Executor, submit_option_string: str
+) -> Iterable[str]:
     """Submit a number of resources from their JDL, reporting the new Job ID for each"""
     # verbose submit gives an ordered listing of class ads, such as
     # ** Proc 15556.0:
@@ -110,7 +115,9 @@ async def condor_submit(*resource_jdls: JDL, executor: Executor) -> Iterable[str
     #
     # ** Proc 15556.1:
     # ...
-    command = f"condor_submit -verbose -maxjobs {len(resource_jdls)}"
+    command = (
+        f"condor_submit -verbose -maxjobs {len(resource_jdls)} {submit_option_string}"
+    )
     response = await executor.run_command(
         command,
         stdin_input=_submit_description(resource_jdls),
@@ -213,20 +220,30 @@ def __init__(
         self._machine_type = machine_type
         self._site_name = site_name
         self._executor = getattr(self.configuration, "executor", ShellExecutor())
+
+        submit_option_string = htcondor_cmd_option_formatter(
+            self.machine_type_configuration.get("SubmitOptions", AttributeDict())
+        )
+
         bulk_size = getattr(self.configuration, "bulk_size", 100)
         bulk_delay = getattr(self.configuration, "bulk_delay", 1.0)
-        self._condor_submit, self._condor_suspend, self._condor_rm = (
+
+        self._condor_submit = AsyncBulkCall(
+            partial(
+                condor_submit,
+                executor=self._executor,
+                submit_option_string=submit_option_string,
+            ),
+            size=bulk_size,
+            delay=bulk_delay,
+        )
+        self._condor_suspend, self._condor_rm, self._condor_q = (
             AsyncBulkCall(
                 partial(tool, executor=self._executor),
                 size=bulk_size,
                 delay=bulk_delay,
             )
-            for tool in (condor_submit, condor_suspend, condor_rm)
-        )
-        self._condor_q = AsyncBulkCall(
-            partial(condor_q, executor=self._executor),
-            size=bulk_size,
-            delay=bulk_delay,
+            for tool in (condor_suspend, condor_rm, condor_q)
         )
 
         key_translator = StaticMapping(


Test patch:
diff --git a/tests/adapters_t/sites_t/test_htcondorsiteadapter.py b/tests/adapters_t/sites_t/test_htcondorsiteadapter.py
index eeda2062..d44cf59f 100644
--- a/tests/adapters_t/sites_t/test_htcondorsiteadapter.py
+++ b/tests/adapters_t/sites_t/test_htcondorsiteadapter.py
@@ -139,6 +139,7 @@ def machine_meta_data(self):
             test2large_args=AttributeDict(Cores=8, Memory=32, Disk=160),
             test2large_deprecated=AttributeDict(Cores=8, Memory=32, Disk=160),
             testunkownresource=AttributeDict(Cores=8, Memory=32, Disk=160, Foo=3),
+            testsubmitoptions=AttributeDict(Cores=8, Memory=32, Disk=160),
         )
 
     @property
@@ -148,6 +149,10 @@ def machine_type_configuration(self):
             test2large_args=AttributeDict(jdl="tests/data/submit_per_arguments.jdl"),
             test2large_deprecated=AttributeDict(jdl="tests/data/submit_deprecated.jdl"),
             testunkownresource=AttributeDict(jdl="tests/data/submit.jdl"),
+            testsubmitoptions=AttributeDict(
+                jdl="tests/data/submit.jdl",
+                SubmitOptions=AttributeDict(pool="my_remote_pool", spool=None),
+            ),
         )
 
     @mock_executor_run_command(stdout=CONDOR_SUBMIT_OUTPUT)
@@ -231,6 +236,28 @@ def test_deploy_resource_htcondor_obs(self):
         )
         self.mock_executor.reset()
 
+        self.adapter = HTCondorAdapter(
+            machine_type="testsubmitoptions", site_name="TestSite"
+        )
+
+        # Test support for submit options
+        run_async(
+            self.adapter.deploy_resource,
+            AttributeDict(
+                drone_uuid="test-123",
+                obs_machine_meta_data_translation_mapping=AttributeDict(
+                    Cores=1,
+                    Memory=1024,
+                    Disk=1024 * 1024,
+                ),
+            ),
+        )
+
+        args, _ = self.mock_executor.return_value.run_command.call_args
+        self.assertEqual(
+            "condor_submit -verbose -maxjobs 1 -pool my_remote_pool -spool", args[0]
+        )
+
     def test_translate_resources_raises_logs(self):
         self.adapter = HTCondorAdapter(
             machine_type="testunkownresource", site_name="TestSite"