Repository: cta-observatory/lstosa

Issue:
Look for proper systematics factor file and pass it to the r0_to_dl1 script
Implement a function to look for the systematic correction file path used to produce the calibration file to be employed for data processing

Discussion:

Patch:
diff --git a/cfg/sequencer.cfg b/cfg/sequencer.cfg
index 20db5c5..c5db1f5 100644
--- a/cfg/sequencer.cfg
+++ b/cfg/sequencer.cfg
@@ -18,6 +18,7 @@ CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration/LevelA
 CALIB_DIR: %(CALIB_BASE_DIR)s/calibration
 PEDESTAL_DIR: %(CALIB_BASE_DIR)s/drs4_baseline
 TIMECALIB_DIR: %(CALIB_BASE_DIR)s/drs4_time_sampling_from_FF
+SYSTEMATIC_DIR: %(CALIB_BASE_DIR)s/ffactor_systematics
 DL1_DIR: %(BASE)s/DL1
 DL1AB_DIR: %(BASE)s/DL1
 MUON_DIR: %(BASE)s/DL1
diff --git a/osa/configs/datamodel.py b/osa/configs/datamodel.py
index 8022a70..443c5b6 100644
--- a/osa/configs/datamodel.py
+++ b/osa/configs/datamodel.py
@@ -99,6 +99,8 @@ class SequenceData(Sequence):
         self.calibration = None
         self.pedestal = None
         self.drive = None
+        self.time_calibration = None
+        self.systematic_correction = None
         self.dl1status = None
         self.dl1abstatus = None
         self.muonstatus = None
diff --git a/osa/job.py b/osa/job.py
index 2c45b99..b1d82eb 100644
--- a/osa/job.py
+++ b/osa/job.py
@@ -3,8 +3,7 @@
 import datetime
 import logging
 import shutil
-import subprocess
-import sys
+import subprocess as sp
 import time
 from io import StringIO
 from pathlib import Path
@@ -46,6 +45,7 @@ __all__ = [
     "get_time_calibration_file",
     "get_calibration_file",
     "get_drs4_pedestal_file",
+    "get_systematic_correction_file"
 ]
 
 TAB = "\t".expandtabs(4)
@@ -282,6 +282,39 @@ def get_time_calibration_file(run_id: int) -> Path:
     return time_calibration_file.resolve()
 
 
+def get_systematic_correction_file(date: str) -> Path:
+    """
+    Return the systematic correction file for a given date.
+
+    Parameters
+    ----------
+    date : str
+        Date in the format YYYYMMDD.
+
+    Notes
+    -----
+    The search for the proper systematic correction file is based on
+    lstchain/scripts/onsite/onsite_create_calibration_file.py
+    """
+    sys_dir = Path(cfg.get("LST1", "SYSTEMATIC_DIR"))
+
+    # Search for the first sys correction file before the run, if nothing before,
+    # use the first found
+    dir_list = sorted(sys_dir.rglob('*/pro/ffactor_systematics*'))
+    if not dir_list:
+        raise IOError(
+            f"No systematic correction file found for production pro in {sys_dir}\n"
+        )
+    sys_date_list = sorted([file.parts[-3] for file in dir_list], reverse=True)
+    selected_date = next(
+        (day for day in sys_date_list if day <= date), sys_date_list[-1]
+    )
+
+    return Path(
+        f"{sys_dir}/{selected_date}/pro/ffactor_systematics_{selected_date}.h5"
+    ).resolve()
+
+
 def get_drs4_pedestal_file(run_id: int) -> Path:
     """
     Return the drs4 pedestal file corresponding to a given run id
@@ -336,6 +369,7 @@ def sequence_calibration_filenames(sequence_list):
         sequence.pedestal = get_drs4_pedestal_file(drs4_pedestal_run_id)
         sequence.calibration = get_calibration_file(pedcal_run_id)
         sequence.time_calibration = get_time_calibration_file(pedcal_run_id)
+        sequence.systematic_correction = get_systematic_correction_file(nightdir)
 
 
 def plot_job_statistics(sacct_output: pd.DataFrame, directory: Path):
@@ -384,8 +418,8 @@ def scheduler_env_variables(sequence, scheduler="slurm"):
         f"--job-name={sequence.jobname}",
         "--cpus-per-task=1",
         f"--chdir={options.directory}",
-        f"--output=log/slurm_{sequence.run:05d}.%4a_%A.out",
-        f"--error=log/slurm_{sequence.run:05d}.%4a_%A.err",
+        f"--output=log/Run{sequence.run:05d}.%4a_jobid_%A.out",
+        f"--error=log/Run{sequence.run:05d}.%4a_jobid_%A.err",
     ]
 
     # Get the number of subruns. The number of subruns starts counting from 0.
@@ -509,6 +543,9 @@ def create_job_template(sequence, get_content=False):
         commandargs.append(f"--drs4-pedestal-file={sequence.pedestal.resolve()}")
         commandargs.append(f"--time-calib-file={sequence.time_calibration.resolve()}")
         commandargs.append(f"--pedcal-file={sequence.calibration.resolve()}")
+        commandargs.append(
+            f"--systematic-correction-file={sequence.systematic_correction.resolve()}"
+        )
         commandargs.append(f"--drive-file={Path(drivedir).resolve() / sequence.drive}")
         commandargs.append(f"--run-summary={run_summary_file.resolve()}")
 
@@ -540,19 +577,8 @@ def create_job_template(sequence, get_content=False):
     content += TAB + "os.environ['NUMBA_CACHE_DIR'] = tmpdirname\n"
 
     content += TAB + "proc = subprocess.run([\n"
-    for i in commandargs:
-        content += TAB * 2 + f"'{i}',\n"
-    if not options.test:
-        content += (
-            TAB * 2
-            + f"'--stderr=log/sequence_{sequence.jobname}."
-            + "{0}_{1}.err'.format(str(subruns).zfill(4), str(job_id)),\n"
-        )
-        content += (
-            TAB * 2
-            + f"'--stdout=log/sequence_{sequence.jobname}."
-            + "{0}_{1}.out'.format(str(subruns).zfill(4), str(job_id)),\n"
-        )
+    for arg in commandargs:
+        content += TAB * 2 + f"'{arg}',\n"
     if sequence.type == "DATA":
         content += (
             TAB * 2
@@ -599,10 +625,10 @@ def submit_jobs(sequence_list, batch_command="sbatch"):
             else:
                 try:
                     log.debug(f"Launching script {sequence.script}")
-                    parent_jobid = subprocess.check_output(
+                    parent_jobid = sp.check_output(
                         commandargs, universal_newlines=True, shell=False
                     ).split()[0]
-                except subprocess.CalledProcessError as error:
+                except sp.CalledProcessError as error:
                     rc = error.returncode
                     log.exception(f"Command '{batch_command}' not found, error {rc}")
 
@@ -628,13 +654,13 @@ def submit_jobs(sequence_list, batch_command="sbatch"):
                     "first subrun without scheduler"
                 )
                 commandargs = ["python", sequence.script]
-                subprocess.check_output(commandargs, shell=False)
+                sp.check_output(commandargs, shell=False)
             else:
                 log.info("Submitting jobs to the cluster.")
                 try:
                     log.debug(f"Launching script {sequence.script}")
-                    subprocess.check_output(commandargs, shell=False)
-                except subprocess.CalledProcessError as error:
+                    sp.check_output(commandargs, shell=False)
+                except sp.CalledProcessError as error:
                     log.exception(error)
 
             log.debug(stringify(commandargs))
@@ -651,7 +677,7 @@ def run_squeue() -> StringIO:
         return StringIO()
 
     out_fmt = "%i,%j,%T,%M"  # JOBID, NAME, STATE, TIME
-    return StringIO(subprocess.check_output(["squeue", "--me", "-o", out_fmt]).decode())
+    return StringIO(sp.check_output(["squeue", "--me", "-o", out_fmt]).decode())
 
 
 def get_squeue_output(squeue_output: StringIO) -> pd.DataFrame:
@@ -702,7 +728,7 @@ def run_sacct() -> StringIO:
         "-o",
         ",".join(FORMAT_SLURM),
     ]
-    return StringIO(subprocess.check_output(sacct_cmd).decode())
+    return StringIO(sp.check_output(sacct_cmd).decode())
 
 
 def get_sacct_output(sacct_output: StringIO) -> pd.DataFrame:
@@ -838,12 +864,10 @@ def run_program_with_history_logging(
     rc: int
         Return code of the program
     """
-    try:
-        log.info(f"Executing {stringify(command_args)}")
-        rc = subprocess.run(command_args, check=True).returncode
-    except subprocess.CalledProcessError as error:
-        rc = error.returncode
-        log.exception(f"Could not execute {stringify(command_args)}, error: {error}")
+    log.info(f"Executing {stringify(command_args)}")
+
+    output = sp.run(command_args, stdout=sp.PIPE, stderr=sp.STDOUT, encoding='utf-8')
+    rc = output.returncode
 
     history(
         run=run,
@@ -856,6 +880,6 @@ def run_program_with_history_logging(
     )
 
     if rc != 0:
-        sys.exit(rc)
+        raise ValueError(f"{command_args[0]} failed with output: \n {output.stdout}")
 
     return rc
diff --git a/osa/provenance/config/definition.yaml b/osa/provenance/config/definition.yaml
index 9cc4a16..8caab21 100644
--- a/osa/provenance/config/definition.yaml
+++ b/osa/provenance/config/definition.yaml
@@ -145,6 +145,11 @@ activities:
               entityName: TimeCalibrationFile
               value: TimeCalibrationFile
               # filepath: /fefs/aswg/data/real/calibration/20191124/v00/time_calibration.Run1625.0000.hdf5
+            - role: "Systematic correction factor file"
+              description: "Systematic correction factor file"
+              entityName: SystematicCorrectionFile
+              value: SystematicCorrectionFile
+              # filepath: /fefs/aswg/data/real/monitoring/PixelCalibration/LevelA/ffactor_systematics/20210514/v0.8.2/no_sys_corrected_calibration_scan_fit_20210514.0000.h5
             - role: "Pointing file"
               description: "Pointing filename for DL1"
               entityName: PointingFile
diff --git a/osa/provenance/utils.py b/osa/provenance/utils.py
index 06f559e..ed81517 100644
--- a/osa/provenance/utils.py
+++ b/osa/provenance/utils.py
@@ -95,28 +95,32 @@ def parse_variables(class_instance):
         # calibrationfile   [0] .../20200218/pro/calibration_filters_52.Run02006.0000.h5
         # pedestalfile      [1] .../20200218/pro/drs4_pedestal.Run02005.0000.h5
         # timecalibfile     [2] .../20191124/pro/time_calibration.Run01625.0000.h5
-        # drivefile         [3] .../DrivePositioning/drive_log_20_02_18.txt
-        # runsummaryfile    [4] .../RunSummary/RunSummary_20200101.ecsv
-        # run_str           [5] 02006.0000
+        # systematic_corr   [3] .../20200101/pro/no_sys_corrected_calibration_scan_fit_20210514.0000.h5
+        # drivefile         [4] .../DrivePositioning/drive_log_20_02_18.txt
+        # runsummaryfile    [5] .../RunSummary/RunSummary_20200101.ecsv
+        # run_str           [6] 02006.0000
 
-        class_instance.ObservationRun = class_instance.args[5].split(".")[0]
+        run_subrun_id = class_instance.args[6]
+        class_instance.ObservationRun = run_subrun_id.split(".")[0]
         # use realpath to resolve symbolic links and return abspath
         calibration_file = os.path.realpath(class_instance.args[0])
         pedestal_file = os.path.realpath(class_instance.args[1])
         timecalibration_file = os.path.realpath(class_instance.args[2])
+        systematic_correction_file = os.path.realpath(class_instance.args[3])
         class_instance.R0SubrunDataset = os.path.realpath(
-            f"{raw_dir}/{flat_date}/LST-1.1.Run{class_instance.args[5]}.fits.fz"
+            f"{raw_dir}/{flat_date}/LST-1.1.Run{run_subrun_id}.fits.fz"
         )
         class_instance.CoefficientsCalibrationFile = calibration_file
         class_instance.PedestalFile = pedestal_file
         class_instance.TimeCalibrationFile = timecalibration_file
+        class_instance.SystematicCorrectionFile = systematic_correction_file
         class_instance.PointingFile = os.path.realpath(class_instance.args[3])
         class_instance.RunSummaryFile = os.path.realpath(class_instance.args[4])
         class_instance.DL1SubrunDataset = os.path.realpath(
-            f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[5]}.h5"
+            f"{outdir_dl1}/dl1_LST-1.Run{run_subrun_id}.h5"
         )
         class_instance.MuonsSubrunDataset = os.path.realpath(
-            f"{muon_dir}/muons_LST-1.Run{class_instance.args[5]}.fits"
+            f"{muon_dir}/muons_LST-1.Run{run_subrun_id}.fits"
         )
 
     if class_instance.__name__ == "dl1ab":
diff --git a/osa/scripts/datasequence.py b/osa/scripts/datasequence.py
index ffd5527..01ee029 100644
--- a/osa/scripts/datasequence.py
+++ b/osa/scripts/datasequence.py
@@ -21,6 +21,7 @@ def data_sequence(
         calibration_file: Path,
         pedestal_file: Path,
         time_calibration_file: Path,
+        systematic_correction_file: Path,
         drive_file: Path,
         run_summary: Path,
         run_str: str
@@ -33,6 +34,7 @@ def data_sequence(
     calibration_file: pathlib.Path
     pedestal_file: pathlib.Path
     time_calibration_file: pathlib.Path
+    systematic_correction_file: pathlib.Path
     drive_file: pathlib.Path
     run_summary: pathlib.Path
     run_str: str
@@ -52,6 +54,7 @@ def data_sequence(
             calibration_file,
             pedestal_file,
             time_calibration_file,
+            systematic_correction_file,
             drive_file,
             run_summary,
             run_str,
@@ -99,6 +102,7 @@ def r0_to_dl1(
         calibration_file: Path,
         pedestal_file: Path,
         time_calibration_file: Path,
+        systematic_correction_file: Path,
         drive_file: Path,
         run_summary: Path,
         run_str: str,
@@ -114,6 +118,7 @@ def r0_to_dl1(
     calibration_file: pathlib.Path
     pedestal_file: pathlib.Path
     time_calibration_file: pathlib.Path
+    systematic_correction_file: pathlib.Path
     drive_file: pathlib.Path
     run_summary: : pathlib.Path
         Path to the run summary file
@@ -138,6 +143,7 @@ def r0_to_dl1(
         f"--pedestal-file={pedestal_file}",
         f"--calibration-file={calibration_file}",
         f"--time-calibration-file={time_calibration_file}",
+        f"--systematic-correction-file={systematic_correction_file}",
         f"--pointing-file={drive_file}",
         f"--run-summary-path={run_summary}",
     ]
@@ -304,6 +310,7 @@ def main():
         calibration_file,
         drs4_ped_file,
         time_calibration_file,
+        systematic_correction_file,
         drive_log_file,
         run_summary_file,
         run_number,
@@ -319,6 +326,7 @@ def main():
         calibration_file,
         drs4_ped_file,
         time_calibration_file,
+        systematic_correction_file,
         drive_log_file,
         run_summary_file,
         run_number,
diff --git a/osa/utils/cliopts.py b/osa/utils/cliopts.py
index 041587d..729ee60 100644
--- a/osa/utils/cliopts.py
+++ b/osa/utils/cliopts.py
@@ -342,6 +342,11 @@ def data_sequence_argparser():
         type=Path,
         help="Path of the time calibration file"
     )
+    parser.add_argument(
+        "--systematic-correction-file",
+        type=Path,
+        help="Path of the systematic correction factor file"
+    )
     parser.add_argument(
         "--drive-file",
         type=Path,
@@ -398,6 +403,7 @@ def data_sequence_cli_parsing():
         opts.pedcal_file,
         opts.drs4_pedestal_file,
         opts.time_calib_file,
+        opts.systematic_correction_file,
         opts.drive_file,
         opts.run_summary,
         opts.run_number,


Test patch:
diff --git a/osa/conftest.py b/osa/conftest.py
index 375c3a1..e3e29ff 100644
--- a/osa/conftest.py
+++ b/osa/conftest.py
@@ -99,6 +99,21 @@ def drs4_time_calibration_files(calibration_base_dir):
     return time_file_list
 
 
+@pytest.fixture(scope="session")
+def systematic_correction_files(calibration_base_dir):
+    directory = calibration_base_dir / "ffactor_systematics"
+    directory1 = directory / "20200725" / "pro"
+    directory2 = directory / "20201110" / "pro"
+    directory1.mkdir(parents=True, exist_ok=True)
+    directory2.mkdir(parents=True, exist_ok=True)
+    file1 = directory1 / "ffactor_systematics_20200725.h5"
+    file2 = directory2 / "ffactor_systematics_20201110.h5"
+    sys_corr_file_list = [file1, file2]
+    for file in sys_corr_file_list:
+        file.touch()
+    return sys_corr_file_list
+
+
 @pytest.fixture(scope="session")
 def running_analysis_dir(base_test_dir):
     analysis_dir = base_test_dir / "running_analysis" / nightdir / prod_id
@@ -212,14 +227,27 @@ def run_summary(run_summary_file):
 
 
 @pytest.fixture(scope="session")
-def sequence_list(running_analysis_dir, run_summary, drs4_time_calibration_files):
+def sequence_list(
+        running_analysis_dir,
+        run_summary,
+        drs4_time_calibration_files,
+        systematic_correction_files,
+        r0_data,
+):
     """Creates a sequence list from a run summary file."""
     options.directory = running_analysis_dir
     options.simulate = True
     options.test = True
+
     for file in drs4_time_calibration_files:
         assert file.exists()
 
+    for file in systematic_correction_files:
+        assert file.exists()
+
+    for file in r0_data:
+        assert file.exists()
+
     subrun_list = extractsubruns(run_summary)
     run_list = extractruns(subrun_list)
     options.test = False
@@ -232,12 +260,18 @@ def sequence_file_list(
         run_summary_file,
         run_catalog,
         drs4_time_calibration_files,
+        systematic_correction_files,
         r0_data
 ):
     for r0_file in r0_data:
         assert r0_file.exists()
+
     for file in drs4_time_calibration_files:
         assert file.exists()
+
+    for file in systematic_correction_files:
+        assert file.exists()
+
     assert run_summary_file.exists()
     assert run_catalog.exists()
 
diff --git a/osa/scripts/tests/test_osa_scripts.py b/osa/scripts/tests/test_osa_scripts.py
index e4426f3..b021a05 100644
--- a/osa/scripts/tests/test_osa_scripts.py
+++ b/osa/scripts/tests/test_osa_scripts.py
@@ -52,11 +52,19 @@ def test_all_help(script):
     run_program(script, "--help")
 
 
-def test_simulate_processing(drs4_time_calibration_files, run_summary_file, r0_data):
+def test_simulate_processing(
+        drs4_time_calibration_files,
+        systematic_correction_files,
+        run_summary_file,
+        r0_data
+):
 
     for file in drs4_time_calibration_files:
         assert file.exists()
 
+    for file in systematic_correction_files:
+        assert file.exists()
+
     for r0_file in r0_data:
         assert r0_file.exists()
 
@@ -82,16 +90,16 @@ def test_simulate_processing(drs4_time_calibration_files, run_summary_file, r0_d
 
     with open(json_file_dl1) as file:
         dl1 = yaml.safe_load(file)
-    assert len(dl1["entity"]) == 15
+    assert len(dl1["entity"]) == 26
     assert len(dl1["activity"]) == 4
-    assert len(dl1["used"]) == 12
+    assert len(dl1["used"]) == 23
     assert len(dl1["wasGeneratedBy"]) == 7
 
     with open(json_file_dl2) as file:
         dl2 = yaml.safe_load(file)
-    assert len(dl2["entity"]) == 24
+    assert len(dl2["entity"]) == 35
     assert len(dl2["activity"]) == 6
-    assert len(dl2["used"]) == 20
+    assert len(dl2["used"]) == 31
     assert len(dl2["wasGeneratedBy"]) == 12
 
     rc = run_program("simulate_processing", "-p")
@@ -104,19 +112,27 @@ def test_simulate_processing(drs4_time_calibration_files, run_summary_file, r0_d
 
 def test_simulated_sequencer(
         drs4_time_calibration_files,
+        systematic_correction_files,
         run_summary_file,
         run_catalog,
         r0_data
 ):
     assert run_summary_file.exists()
     assert run_catalog.exists()
+
     for r0_file in r0_data:
         assert r0_file.exists()
+
     for file in drs4_time_calibration_files:
         assert file.exists()
+
+    for file in systematic_correction_files:
+        assert file.exists()
+
     rc = run_program(
         "sequencer", "-c", "cfg/sequencer.cfg", "-d", "2020_01_17", "-s", "-t", "LST1"
     )
+
     assert rc.returncode == 0
     now = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M")
     assert rc.stdout == dedent(
@@ -206,6 +222,7 @@ def test_datasequence(running_analysis_dir):
     drs4_file = "drs4_pedestal.Run00001.0000.fits"
     calib_file = "calibration.Run00002.0000.hdf5"
     timecalib_file = "time_calibration.Run00002.0000.hdf5"
+    systematic_correction_file = "no_sys_corrected_calibration_scan_fit_20210514.0000.h5"
     drive_file = "drive_log_20200117.txt"
     runsummary_file = "RunSummary_20200117.ecsv"
     prod_id = "v0.1.0"
@@ -222,6 +239,7 @@ def test_datasequence(running_analysis_dir):
         f"--drs4-pedestal-file={drs4_file}",
         f"--pedcal-file={calib_file}",
         f"--time-calib-file={timecalib_file}",
+        f"--systematic-correction-file={systematic_correction_file}",
         f"--drive-file={drive_file}",
         f"--run-summary={runsummary_file}",
         run_number,
diff --git a/osa/tests/test_jobs.py b/osa/tests/test_jobs.py
index 6b13d9e..64a0ade 100644
--- a/osa/tests/test_jobs.py
+++ b/osa/tests/test_jobs.py
@@ -71,8 +71,8 @@ def test_scheduler_env_variables(sequence_list, running_analysis_dir):
         '#SBATCH --job-name=LST1_01805',
         '#SBATCH --cpus-per-task=1',
         f'#SBATCH --chdir={running_analysis_dir}',
-        '#SBATCH --output=log/slurm_01805.%4a_%A.out',
-        '#SBATCH --error=log/slurm_01805.%4a_%A.err',
+        '#SBATCH --output=log/Run01805.%4a_jobid_%A.out',
+        '#SBATCH --error=log/Run01805.%4a_jobid_%A.err',
         '#SBATCH --partition=short',
         '#SBATCH --mem-per-cpu=3GB'
     ]
@@ -83,8 +83,8 @@ def test_scheduler_env_variables(sequence_list, running_analysis_dir):
         '#SBATCH --job-name=LST1_01807',
         '#SBATCH --cpus-per-task=1',
         f'#SBATCH --chdir={running_analysis_dir}',
-        '#SBATCH --output=log/slurm_01807.%4a_%A.out',
-        '#SBATCH --error=log/slurm_01807.%4a_%A.err',
+        '#SBATCH --output=log/Run01807.%4a_jobid_%A.out',
+        '#SBATCH --error=log/Run01807.%4a_jobid_%A.err',
         '#SBATCH --array=0-10',
         '#SBATCH --partition=long',
         '#SBATCH --mem-per-cpu=16GB'
@@ -103,8 +103,8 @@ def test_job_header_template(sequence_list, running_analysis_dir):
     #SBATCH --job-name=LST1_01805
     #SBATCH --cpus-per-task=1
     #SBATCH --chdir={running_analysis_dir}
-    #SBATCH --output=log/slurm_01805.%4a_%A.out
-    #SBATCH --error=log/slurm_01805.%4a_%A.err
+    #SBATCH --output=log/Run01805.%4a_jobid_%A.out
+    #SBATCH --error=log/Run01805.%4a_jobid_%A.err
     #SBATCH --partition=short
     #SBATCH --mem-per-cpu=3GB""")
     assert header == output_string1
@@ -118,8 +118,8 @@ def test_job_header_template(sequence_list, running_analysis_dir):
     #SBATCH --job-name=LST1_01807
     #SBATCH --cpus-per-task=1
     #SBATCH --chdir={running_analysis_dir}
-    #SBATCH --output=log/slurm_01807.%4a_%A.out
-    #SBATCH --error=log/slurm_01807.%4a_%A.err
+    #SBATCH --output=log/Run01807.%4a_jobid_%A.out
+    #SBATCH --error=log/Run01807.%4a_jobid_%A.err
     #SBATCH --array=0-10
     #SBATCH --partition=long
     #SBATCH --mem-per-cpu=16GB""")
@@ -143,8 +143,8 @@ def test_create_job_template_scheduler(
     #SBATCH --job-name=LST1_01807
     #SBATCH --cpus-per-task=1
     #SBATCH --chdir={Path.cwd()}/test_osa/test_files0/running_analysis/20200117/v0.1.0
-    #SBATCH --output=log/slurm_01807.%4a_%A.out
-    #SBATCH --error=log/slurm_01807.%4a_%A.err
+    #SBATCH --output=log/Run01807.%4a_jobid_%A.out
+    #SBATCH --error=log/Run01807.%4a_jobid_%A.err
     #SBATCH --array=0-10
     #SBATCH --partition={cfg.get('SLURM', 'PARTITION_DATA')}
     #SBATCH --mem-per-cpu={cfg.get('SLURM', 'MEMSIZE_DATA')}
@@ -171,10 +171,9 @@ def test_create_job_template_scheduler(
             '--drs4-pedestal-file={drs4_baseline_file}',
             '--time-calib-file={drs4_time_calibration_files[0]}',
             '--pedcal-file={calibration_file}',
+            '--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/LevelA/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
             '--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/drive_log_20_01_17.txt',
             '--run-summary={run_summary_file}',
-            '--stderr=log/sequence_LST1_01807.{{0}}_{{1}}.err'.format(str(subruns).zfill(4), str(job_id)),
-            '--stdout=log/sequence_LST1_01807.{{0}}_{{1}}.out'.format(str(subruns).zfill(4), str(job_id)),
             '01807.{{0}}'.format(str(subruns).zfill(4)),
             'LST1'
         ])
@@ -189,10 +188,22 @@ def test_create_job_template_local(
         drs4_time_calibration_files,
         drs4_baseline_file,
         calibration_file,
-        run_summary_file
+        systematic_correction_files,
+        run_summary_file,
+        r0_data
 ):
     """Check the job file in local mode (assuming no scheduler)."""
     from osa.job import create_job_template
+
+    for file in drs4_time_calibration_files:
+        assert file.exists()
+
+    for file in systematic_correction_files:
+        assert file.exists()
+
+    for file in r0_data:
+        assert file.exists()
+
     options.test = True
     options.simulate = False
     content = create_job_template(sequence_list[1], get_content=True)
@@ -217,6 +228,7 @@ def test_create_job_template_local(
             '--drs4-pedestal-file={drs4_baseline_file}',
             '--time-calib-file={drs4_time_calibration_files[0]}',
             '--pedcal-file={calibration_file}',
+            '--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/LevelA/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
             '--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/drive_log_20_01_17.txt',
             '--run-summary={run_summary_file}',
             '01807.{{0}}'.format(str(subruns).zfill(4)),