Repository: gammasim/simtools
Pull request #1561: Add the sequential command line argument 
Author: login: orelgueta
Created: 2025-05-13T20:11:10Z | Merged: 2025-05-14T08:45:11Z

Add the sequential option for simulations using multipipe. With this option the CORSIKA and sim_telarray instances run in sequential order as far as possible. This mode is useful particularly on the grid, where typically we request a single core per job.

Closes #1552.

Linked issue #1552: Add a command line option for running sequentially
multipipe_corsika has a `--sequential` argument which tells CORSIKA/sim_telarray to run using only one core (to some extent, gzip is still running in parallel, but it's not a problem). That is the way we run on the grid at the moment, so we should have the same option in simtools. Need to implement this command line argument and propagate it to `multipipe_corsika`.

Diff:
diff --git a/docs/changes/1561.feature.md b/docs/changes/1561.feature.md
new file mode 100644
index 0000000000..d5184ea733
--- /dev/null
+++ b/docs/changes/1561.feature.md
@@ -0,0 +1 @@
+Add the sequential option to simulations using multipipe. With this option the CORSIKA and sim_telarray instances run in sequential order as far as possible. This mode is useful particularly on the grid, where typically we request a single core per job.
diff --git a/src/simtools/applications/simulate_prod.py b/src/simtools/applications/simulate_prod.py
index 7868ffc8a2..ff5a854981 100644
--- a/src/simtools/applications/simulate_prod.py
+++ b/src/simtools/applications/simulate_prod.py
@@ -122,6 +122,15 @@ def _parse(description=None):
         required=False,
         default=False,
     )
+    config.parser.add_argument(
+        "--sequential",
+        help=(
+            "Enables single-core mode (as far as possible); "
+            "otherwise, CORSIKA and sim_telarray run in parallel."
+        ),
+        action="store_true",
+        default=False,
+    )
     sim_telarray_seed_group = config.parser.add_argument_group(
         title="Random seeds for sim_telarray instrument setup",
     )
diff --git a/src/simtools/runners/corsika_simtel_runner.py b/src/simtools/runners/corsika_simtel_runner.py
index 6301b2c756..a87620f941 100644
--- a/src/simtools/runners/corsika_simtel_runner.py
+++ b/src/simtools/runners/corsika_simtel_runner.py
@@ -42,6 +42,7 @@ def __init__(
         keep_seeds=False,
         use_multipipe=False,
         sim_telarray_seeds=None,
+        sequential=False,
     ):
         self._logger = logging.getLogger(__name__)
         self.corsika_config = (
@@ -53,6 +54,7 @@ def __init__(
         self._simtel_path = simtel_path
         self.sim_telarray_seeds = sim_telarray_seeds
         self.label = label
+        self.sequential = "--sequential" if sequential else ""
 
         self.base_corsika_config.set_output_file_and_directory(use_multipipe)
         self.corsika_runner = CorsikaRunner(
@@ -167,7 +169,8 @@ def _write_multipipe_script(self, multipipe_file):
         )
         with open(multipipe_script, "w", encoding="utf-8") as file:
             multipipe_command = Path(self._simtel_path).joinpath(
-                f"sim_telarray/bin/multipipe_corsika -c {multipipe_file} || echo 'Fan-out failed'"
+                f"sim_telarray/bin/multipipe_corsika -c {multipipe_file} {self.sequential} "
+                "|| echo 'Fan-out failed'"
             )
             file.write(f"{multipipe_command}")
 
diff --git a/src/simtools/simulator.py b/src/simtools/simulator.py
index ac1389c0d5..bd20ccbcf1 100644
--- a/src/simtools/simulator.py
+++ b/src/simtools/simulator.py
@@ -301,6 +301,8 @@ def _initialize_simulation_runner(self, db_config):
             runner_args["keep_seeds"] = self.args_dict.get("corsika_test_seeds", False)
         if runner_class is not CorsikaRunner:
             runner_args["sim_telarray_seeds"] = self.sim_telarray_seeds
+        if runner_class is CorsikaSimtelRunner:
+            runner_args["sequential"] = self.args_dict.get("sequential", False)
 
         return runner_class(**runner_args)
 
diff --git a/tests/unit_tests/runners/test_corsika_simtel_runner.py b/tests/unit_tests/runners/test_corsika_simtel_runner.py
index 3a02a43917..3edd3df655 100644
--- a/tests/unit_tests/runners/test_corsika_simtel_runner.py
+++ b/tests/unit_tests/runners/test_corsika_simtel_runner.py
@@ -115,6 +115,31 @@ def test_write_multipipe_script(corsika_simtel_runner):
         assert "bin/multipipe_corsika" in script_content
         assert f"-c {multipipe_file}" in script_content
         assert "'Fan-out failed'" in script_content
+        assert "--sequential" not in script_content
+
+
+def test_write_multipipe_script_sequential(corsika_simtel_runner):
+    # Set the sequential attribute
+    corsika_simtel_runner.sequential = "--sequential"
+
+    # Export and write the multipipe script
+    corsika_simtel_runner._export_multipipe_script(run_number=1)
+    multipipe_file = Path(
+        corsika_simtel_runner.base_corsika_config.config_file_path.parent
+    ).joinpath(corsika_simtel_runner.base_corsika_config.get_corsika_config_file_name("multipipe"))
+    corsika_simtel_runner._write_multipipe_script(multipipe_file)
+    script = Path(corsika_simtel_runner.base_corsika_config.config_file_path.parent).joinpath(
+        "run_cta_multipipe"
+    )
+
+    # Assertions
+    assert script.exists()
+    with open(script) as f:
+        script_content = f.read()
+        assert "bin/multipipe_corsika" in script_content
+        assert f"-c {multipipe_file}" in script_content
+        assert "'Fan-out failed'" in script_content
+        assert "--sequential" in script_content
 
 
 def test_get_file_name(corsika_simtel_runner, simulation_file):
diff --git a/tests/unit_tests/test_simulator.py b/tests/unit_tests/test_simulator.py
index a35fd7e5e5..dd985c5850 100644
--- a/tests/unit_tests/test_simulator.py
+++ b/tests/unit_tests/test_simulator.py
@@ -20,6 +20,8 @@
 
 logger = logging.getLogger()
 
+CORSIKA_CONFIG_MOCK_PATCH = "simtools.simulator.CorsikaConfig"
+
 
 @pytest.fixture
 def input_file_list():
@@ -84,6 +86,7 @@ def shower_array_simulator(io_handler, db_config, simulations_args_dict):
     args_dict = copy.deepcopy(simulations_args_dict)
     args_dict["simulation_software"] = "corsika_sim_telarray"
     args_dict["label"] = "test-shower-array-simulator"
+    args_dict["sequential"] = True
     return Simulator(
         label=args_dict["label"],
         args_dict=args_dict,
@@ -552,3 +555,74 @@ def test_get_seed_for_random_instrument_instances(shower_simulator):
         shower_simulator.sim_telarray_seeds["seed"], model_version="6.0.1"
     )
     assert seed == 600010000000 + 1000000 + 20 * 1000 + 180
+
+
+def test_initialize_simulation_runner_with_corsika(shower_simulator, db_config, mocker):
+    # Mock CorsikaConfig to avoid actual initialization
+    mock_corsika_config = mocker.patch(CORSIKA_CONFIG_MOCK_PATCH, autospec=True)
+    mock_corsika_runner = mocker.patch("simtools.simulator.CorsikaRunner", autospec=True)
+
+    # Call the method
+    simulation_runner = shower_simulator._initialize_simulation_runner(db_config)
+
+    # Assertions
+    assert isinstance(simulation_runner, CorsikaRunner)
+    mock_corsika_config.assert_called_once_with(
+        array_model=shower_simulator.array_models[0],
+        label=shower_simulator.label,
+        args_dict=shower_simulator.args_dict,
+        db_config=db_config,
+    )
+    mock_corsika_runner.assert_called_once_with(
+        label=shower_simulator.label,
+        corsika_config=mock_corsika_config.return_value,
+        simtel_path=shower_simulator.args_dict.get("simtel_path"),
+        use_multipipe=False,
+        keep_seeds=shower_simulator.args_dict.get("corsika_test_seeds", False),
+    )
+
+
+def test_initialize_simulation_runner_with_sim_telarray(array_simulator, db_config, mocker):
+    # Mock SimulatorArray to avoid actual initialization
+    mock_simulator_array = mocker.patch("simtools.simulator.SimulatorArray", autospec=True)
+    mock_corsika_config = mocker.patch(CORSIKA_CONFIG_MOCK_PATCH, autospec=True)
+
+    # Call the method
+    simulation_runner = array_simulator._initialize_simulation_runner(db_config)
+
+    # Assertions
+    assert isinstance(simulation_runner, SimulatorArray)
+    mock_simulator_array.assert_called_once_with(
+        label=array_simulator.label,
+        corsika_config=mock_corsika_config.return_value,
+        simtel_path=array_simulator.args_dict.get("simtel_path"),
+        use_multipipe=False,
+        sim_telarray_seeds=array_simulator.sim_telarray_seeds,
+    )
+
+
+def test_initialize_simulation_runner_with_corsika_sim_telarray(
+    shower_array_simulator, db_config, mocker
+):
+    # Mock CorsikaConfig and CorsikaSimtelRunner to avoid actual initialization
+    mock_corsika_config = mocker.patch(CORSIKA_CONFIG_MOCK_PATCH, autospec=True)
+    mock_corsika_simtel_runner = mocker.patch(
+        "simtools.simulator.CorsikaSimtelRunner", autospec=True
+    )
+
+    # Call the method
+    simulation_runner = shower_array_simulator._initialize_simulation_runner(db_config)
+
+    # Assertions
+    assert isinstance(simulation_runner, CorsikaSimtelRunner)
+    mock_corsika_config.assert_called()
+    mock_corsika_simtel_runner.assert_called_once_with(
+        label=shower_array_simulator.label,
+        corsika_config=[mock_corsika_config.return_value]
+        * len(shower_array_simulator.array_models),
+        simtel_path=shower_array_simulator.args_dict.get("simtel_path"),
+        use_multipipe=True,
+        sim_telarray_seeds=shower_array_simulator.sim_telarray_seeds,
+        sequential=shower_array_simulator.args_dict.get("sequential", False),
+        keep_seeds=shower_array_simulator.args_dict.get("corsika_test_seeds", False),
+    )


Reviews:
user: login: copilot-pull-request-reviewer[bot]
body: ## Pull Request Overview

This PR adds a new "sequential" option to simulations using multipipe, enabling the CORSIKA and sim_telarray instances to run in sequential order when needed (e.g., for grid jobs using a single core).  
- Introduces a new command line argument "--sequential" in the production application.  
- Updates simulation runner logic to pass and interpret the sequential flag.  
- Enhances unit tests and documentation to validate and describe this new feature.

### Reviewed Changes

Copilot reviewed 6 out of 6 changed files in this pull request and generated no comments.

<details>
<summary>Show a summary per file</summary>

| File                                                 | Description                                                                                        |
| ---------------------------------------------------- | -------------------------------------------------------------------------------------------------- |
| tests/unit_tests/test_simulator.py                   | Adds unit tests validating the sequential option in simulation runner initialization.              |
| tests/unit_tests/runners/test_corsika_simtel_runner.py | Adds tests to verify the multipipe script generation with the sequential flag.                     |
| src/simtools/simulator.py                            | Propagates the sequential parameter when initializing the simulation runner.                     |
| src/simtools/runners/corsika_simtel_runner.py          | Updates the runner to set the sequential flag appropriately in the multipipe command.              |
| src/simtools/applications/simulate_prod.py           | Adds the "--sequential" argument to the CLI parser, with updated help text.                        |
| docs/changes/1561.feature.md                          | Documents the new sequential option for simulations using multipipe.                             |
</details>





state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-14T06:26:53Z
user: login: GernotMaier
body: Thanks! Minor comment on the help messages, otherwise good.
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-14T07:46:27Z
user: login: orelgueta
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-14T08:10:03Z
user: login: GernotMaier
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-14T08:11:57Z
user: login: orelgueta
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-14T08:12:23Z
user: login: GernotMaier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-14T08:15:19Z

Review comments:
diff_hunk: @@ -122,6 +122,17 @@ def _parse(description=None):
         required=False,
         default=False,
     )
+    config.parser.add_argument(
path: src/simtools/applications/simulate_prod.py
user: login: GernotMaier
body: Wondering if the default shouldn't be 'true'? We will run all major productions on the grid with the single core requirement.
created_at: 2025-05-14T07:42:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -122,6 +122,17 @@ def _parse(description=None):
         required=False,
         default=False,
     )
+    config.parser.add_argument(
+        "--sequential",
+        help=(
+            "If set to true, run CORSIKA and sim_telarray instances in sequential order "
path: src/simtools/applications/simulate_prod.py
user: login: GernotMaier
body: Is this actually correct? One cannot set this parameter to 'set to true', but only set the flag 'sequential'. Suggestion simply "Run CORSIKA and sim_telarray instances in ...".

Further down, I suggest to remove the sentence "This mode is useful particularly on the grid, "
            "where typically we request a single core per job.".  or replace it by a more technical description (without `we` - who is that?). e.g. `Enables single-core mode; otherwise, CORSIKA and sim_telarray run in parallel. `
created_at: 2025-05-14T07:45:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -122,6 +122,17 @@ def _parse(description=None):
         required=False,
         default=False,
     )
+    config.parser.add_argument(
+        "--sequential",
+        help=(
+            "If set to true, run CORSIKA and sim_telarray instances in sequential order "
path: src/simtools/applications/simulate_prod.py
user: login: orelgueta
body: You are right, I wrote this when the argument was different and also not in a technical enough way. I took your suggested sentence almost as is.
created_at: 2025-05-14T08:10:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -122,6 +122,17 @@ def _parse(description=None):
         required=False,
         default=False,
     )
+    config.parser.add_argument(
path: src/simtools/applications/simulate_prod.py
user: login: orelgueta
body: I don't know. On the grid we will always run with a configuration file which can always have this kind of setting explicit (which is a small bonus as well). However, when running locally or during testing, it's easier to keep the configuration short and use as many cores as possible to speed things up.

It's not a strong opinion, but I think it makes more sense this way.
created_at: 2025-05-14T08:12:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -122,6 +122,17 @@ def _parse(description=None):
         required=False,
         default=False,
     )
+    config.parser.add_argument(
+        "--sequential",
+        help=(
+            "If set to true, run CORSIKA and sim_telarray instances in sequential order "
path: src/simtools/applications/simulate_prod.py
user: login: GernotMaier
body: All good - fine with me.
created_at: 2025-05-14T08:15:19Z
author_association: CONTRIBUTOR

PR comments:
user: login: ctao-dpps-sonarqube[bot]
created_at: 2025-05-13T20:15:00Z
body: ![Passed](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/checks/QualityGateBadge/passed.svg?sanitize=true)

# Analysis Details
## 1 Issue
- ![Bug](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/common/bug.svg?sanitize=true) 0 Bugs
- ![Vulnerability](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/common/vulnerability.svg?sanitize=true) 0 Vulnerabilities
- ![Code Smell](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/common/code_smell.svg?sanitize=true) 1 Code Smell

## Coverage and Duplications
- ![Coverage](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/checks/CoverageChart/100.svg?sanitize=true) 100.00% Coverage (96.90% Estimated after merge)
- ![Duplications](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/checks/Duplications/3.svg?sanitize=true) 0.00% Duplicated Code (0.00% Estimated after merge)

**Project ID:** gammasim_simtools_AY_ssha9WiFxsX-2oy_w

[View in SonarQube](https://sonar-cta-dpps.zeuthen.desy.de/dashboard?id=gammasim_simtools_AY_ssha9WiFxsX-2oy_w&pullRequest=1561)


author_association: NONE
user: login: ctao-dpps-sonarqube[bot]
created_at: 2025-05-14T06:07:26Z
body: ![Passed](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/checks/QualityGateBadge/passed.svg?sanitize=true)

# Analysis Details
## 0 Issues
- ![Bug](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/common/bug.svg?sanitize=true) 0 Bugs
- ![Vulnerability](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/common/vulnerability.svg?sanitize=true) 0 Vulnerabilities
- ![Code Smell](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/common/code_smell.svg?sanitize=true) 0 Code Smells

## Coverage and Duplications
- ![Coverage](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/checks/CoverageChart/100.svg?sanitize=true) 100.00% Coverage (96.90% Estimated after merge)
- ![Duplications](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/checks/Duplications/3.svg?sanitize=true) 0.00% Duplicated Code (0.00% Estimated after merge)

**Project ID:** gammasim_simtools_AY_ssha9WiFxsX-2oy_w

[View in SonarQube](https://sonar-cta-dpps.zeuthen.desy.de/dashboard?id=gammasim_simtools_AY_ssha9WiFxsX-2oy_w&pullRequest=1561)


author_association: NONE
user: login: ctao-dpps-sonarqube[bot]
created_at: 2025-05-14T06:14:09Z
body: ![Passed](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/checks/QualityGateBadge/passed.svg?sanitize=true)

# Analysis Details
## 0 Issues
- ![Bug](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/common/bug.svg?sanitize=true) 0 Bugs
- ![Vulnerability](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/common/vulnerability.svg?sanitize=true) 0 Vulnerabilities
- ![Code Smell](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/common/code_smell.svg?sanitize=true) 0 Code Smells

## Coverage and Duplications
- ![Coverage](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/checks/CoverageChart/100.svg?sanitize=true) 100.00% Coverage (96.90% Estimated after merge)
- ![Duplications](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/checks/Duplications/3.svg?sanitize=true) 0.00% Duplicated Code (0.00% Estimated after merge)

**Project ID:** gammasim_simtools_AY_ssha9WiFxsX-2oy_w

[View in SonarQube](https://sonar-cta-dpps.zeuthen.desy.de/dashboard?id=gammasim_simtools_AY_ssha9WiFxsX-2oy_w&pullRequest=1561)


author_association: NONE
user: login: ctao-dpps-sonarqube[bot]
created_at: 2025-05-14T06:28:40Z
body: ![Passed](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/checks/QualityGateBadge/passed.svg?sanitize=true)

# Analysis Details
## 0 Issues
- ![Bug](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/common/bug.svg?sanitize=true) 0 Bugs
- ![Vulnerability](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/common/vulnerability.svg?sanitize=true) 0 Vulnerabilities
- ![Code Smell](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/common/code_smell.svg?sanitize=true) 0 Code Smells

## Coverage and Duplications
- ![Coverage](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/checks/CoverageChart/100.svg?sanitize=true) 100.00% Coverage (96.90% Estimated after merge)
- ![Duplications](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/checks/Duplications/3.svg?sanitize=true) 0.00% Duplicated Code (0.00% Estimated after merge)

**Project ID:** gammasim_simtools_AY_ssha9WiFxsX-2oy_w

[View in SonarQube](https://sonar-cta-dpps.zeuthen.desy.de/dashboard?id=gammasim_simtools_AY_ssha9WiFxsX-2oy_w&pullRequest=1561)


author_association: NONE
user: login: ctao-dpps-sonarqube[bot]
created_at: 2025-05-14T08:14:48Z
body: ![Passed](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/checks/QualityGateBadge/passed.svg?sanitize=true)

# Analysis Details
## 0 Issues
- ![Bug](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/common/bug.svg?sanitize=true) 0 Bugs
- ![Vulnerability](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/common/vulnerability.svg?sanitize=true) 0 Vulnerabilities
- ![Code Smell](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/common/code_smell.svg?sanitize=true) 0 Code Smells

## Coverage and Duplications
- ![Coverage](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/checks/CoverageChart/100.svg?sanitize=true) 100.00% Coverage (96.90% Estimated after merge)
- ![Duplications](https://sonar-cta-dpps.zeuthen.desy.de/static/communityBranchPlugin/checks/Duplications/3.svg?sanitize=true) 0.00% Duplicated Code (0.00% Estimated after merge)

**Project ID:** gammasim_simtools_AY_ssha9WiFxsX-2oy_w

[View in SonarQube](https://sonar-cta-dpps.zeuthen.desy.de/dashboard?id=gammasim_simtools_AY_ssha9WiFxsX-2oy_w&pullRequest=1561)


author_association: NONE