Repository: cta-observatory/lstmcpipe (language: python)

Issue:
fix path dl2 irf



Discussion / hints:

Fix patch:
diff --git a/README.rst b/README.rst
index 6311833c..22ba20ad 100644
--- a/README.rst
+++ b/README.rst
@@ -89,8 +89,11 @@ code follows our guidelines and there is less work dealing with the code checker
 Quickstart
 ----------
 
-The way to launch either the full pipeline (r0 to IRFs) or a workflow with less stages (r0 to merging, for example) is
-the same; you should run the same command/launch the same script in any case;
+To generate your lstmcpipe configuration file, use `lstmcpipe_generate_config` command.
+If the type of production you want is not listed in the existing ones, you may create your own `PathConfig` class
+from an existing one, or generate a config from an existing prod type and edit the file manually.
+
+Once you have your configuration file, you way launch the pipeline with the described stages in the config using:
 
 .. code-block:: python
 
@@ -132,7 +135,6 @@ processing tool. These are:
 - ctapipe: ctapipe-stage1
 - rta: lstmcpipe_hiperta_r0_to_dl1lstchain (``lstmcpipe/hiperta/hiperta_r0_to_dl1lstchain.py``)
 
-Results can be found in ``running_analysis``
 
 **dl1ab**
 
@@ -147,11 +149,16 @@ For lstchain the dl1ab script is used, ctapipe can use the same script as for si
 processing. There is no support for hiperta!
 
 
-**merge_and_copy_dl1**
+**merge_dl1**
 
 In this stage the previously created dl1 files are merged so that you end up with
 train and test datesets for the next stages.
-Cleans and moves the ``running_analysis`` directory into ``DL1`` and ``analysis_logs``.
+
+
+**train_test_split**
+
+Split the dataset into training and testing datasets, performing a random selection of files with the specified ratio
+(default=0.5).
 
 **train_pipe**
 
@@ -205,7 +212,8 @@ while the second one contains the same plus all the commands passed to slurm.
 Steps explanation
 -----------------
 
-Job dependency between stages is done automatically. Also, the directory structure is created by every stage.
+The directory structure and the stages to run are determined by the config stages.
+After that, the job dependency between stages is done automatically.
     - If the full workflow is launched, directories will not be verified as containing data. Overwriting will only happen when a MC prods sharing the same ``prod_id`` and analysed the same day is run
     - If each step is launched independently (advanced users), no overwriting directory will take place prior confirmation from the user
 
diff --git a/environment.yml b/environment.yml
index 3d6679c8..d5593ec9 100644
--- a/environment.yml
+++ b/environment.yml
@@ -9,6 +9,7 @@ dependencies:
   - pip
   - pre-commit
   - ctapipe~=0.12
+  - ruamel.yaml>=0.17
   - pip:
       - ctaplot~=0.5.5
       - ctapipe_io_lst~=0.15.1
diff --git a/lstmcpipe/__init__.py b/lstmcpipe/__init__.py
index 2b8b4851..fa25ad70 100644
--- a/lstmcpipe/__init__.py
+++ b/lstmcpipe/__init__.py
@@ -1,5 +1,8 @@
+import os
 from pathlib import Path
 from .version import __version__  # noqa
 
 __version__
 lstmcpipe_root_dir = Path(__file__).absolute().parents[1]
+prod_logs = Path(os.getenv('LSTMCPIPE_PROD_LOGS')) if os.getenv('LSTMCPIPE_PROD_LOGS') is not None \
+    else Path(os.getenv('HOME')).joinpath('LSTMCPIPE_PROD_LOGS')
diff --git a/lstmcpipe/base_config_lstmcpipe.yaml b/lstmcpipe/base_config_lstmcpipe.yaml
new file mode 100644
index 00000000..3ecea5bc
--- /dev/null
+++ b/lstmcpipe/base_config_lstmcpipe.yaml
@@ -0,0 +1,16 @@
+# lstmcpipe base config #
+
+
+# workflow_kind is not expected to be changed by LST users
+workflow_kind: lstchain
+
+# prod_id ex: local_no_n_islands. Default; v00 (if key left empty or None)
+prod_id:
+
+source_environment:
+  source_file: /fefs/aswg/software/conda/etc/profile.d/conda.sh
+  conda_env: lstchain-v0.9.3
+
+slurm_config:
+  user_account: aswg
+
diff --git a/lstmcpipe/config/__init__.py b/lstmcpipe/config/__init__.py
index 7751ce93..aa571a2b 100644
--- a/lstmcpipe/config/__init__.py
+++ b/lstmcpipe/config/__init__.py
@@ -1,5 +1,8 @@
 import os
 from shutil import which
+import pkg_resources
+from ruamel.yaml import YAML
+
 from .pipeline_config import load_config
 from .dl1ab_tuning import create_dl1ab_tuned_config
 
@@ -12,3 +15,8 @@ def export_env(outdir="."):
         os.system((f"conda env export > {os.path.join(outdir, 'conda_env.yml')}"))
     elif which("pip") is not None:
         os.system(f"python -m pip freeze > {os.path.join(outdir, 'requirements.txt')}")
+
+
+def base_config():
+    base_config_path = pkg_resources.resource_filename('lstmcpipe', 'base_config_lstmcpipe.yaml')
+    return YAML().load(open(base_config_path).read())
diff --git a/lstmcpipe/config/paths_config.py b/lstmcpipe/config/paths_config.py
new file mode 100644
index 00000000..f9953c5d
--- /dev/null
+++ b/lstmcpipe/config/paths_config.py
@@ -0,0 +1,640 @@
+#!/usr/bin/env python
+
+import os
+from ruamel.yaml import YAML
+from datetime import date
+from copy import deepcopy
+import re
+
+from . import base_config
+
+
+class PathConfig:
+    """
+    Base class to generate a Path configuration for a production
+    """
+
+    def __init__(self, prod_id):
+        self.prod_id = prod_id
+        self.paths = {}
+        self.stages = []
+
+    def generate(self):
+        """
+        Generate the stages paths. They are copied into self.paths and returned.
+
+        Returns
+        -------
+        dict: paths config
+        """
+        for stage in self.stages:
+            if not hasattr(self, stage):
+                raise NotImplementedError(f"The stage {stage} is not implemented for this class")
+            self.paths[stage] = getattr(self, stage)
+        return self.paths
+
+    def save_yml(self, filename, overwrite=False, append=False):
+        """
+        Dump the path config to a file
+
+        Parameters
+        ----------
+        filename: str or Path
+        overwrite: bool
+        append: bool
+        """
+        config_to_save = base_config()
+        config_to_save.yaml_set_start_comment(f"lstmcpipe generated config from {self.__class__.__name__} "
+                                              f"- {date.today()}\n\n")
+
+        if self.paths == {}:
+            raise ValueError("Empty paths, generate first")
+
+        config_to_save['prod_type'] = self.__class__.__name__
+        config_to_save['prod_id'] = self.prod_id
+        config_to_save['stages_to_run'] = self.stages
+        config_to_save['stages'] = self.paths
+
+        if os.path.exists(filename) and not (overwrite or append):
+            raise FileExistsError(f"{filename} exists. Set overwrite=True or append=True")
+        if append and overwrite:
+            raise ValueError("Append or overwrite, not both ;-)")
+        if append:
+            config_to_save.update(YAML().load((open(filename).read())))
+
+        with open(filename, 'w') as f:
+            yaml = YAML()
+            yaml.dump(config_to_save, f)
+
+
+class PathConfigProd5Trans80(PathConfig):
+    """
+    Standard paths configuration for a prod5_trans_80 MC production
+    """
+
+    def __init__(self, prod_id, zenith='zenith_20deg'):
+        super().__init__(prod_id)
+        self.prod_id = prod_id
+        self.zenith = zenith
+        self.base_dir = \
+            '/fefs/aswg/data/mc/{data_level}/20200629_prod5_trans_80/{particle}/{zenith}/south_pointing/{prod_id}'
+        self.training_particles = ['gamma-diffuse', 'proton']
+        self.testing_particles = ['gamma', 'electron', 'proton', 'gamma-diffuse']
+        self.point_src_offsets = ['off0.0deg', 'off0.4deg']
+        self.paths = {}
+        self.stages = [
+            'r0_to_dl1',
+            'train_test_split',
+            'merge_dl1',
+            'train_pipe',
+            'dl1_to_dl2',
+            'dl2_to_sensitivity',
+            'dl2_to_irfs',
+        ]
+
+    @property
+    def particles(self):
+        return list(set(self.training_particles + self.testing_particles))
+
+    def _data_level_dir(self, prod_id, data_level, particle, gamma_src_offset='off0.4deg'):
+        """
+
+        Parameters
+        ----------
+        data_level: str
+            `DL0` or `DL1` or `DL2`
+        particle: str
+            `proton`, `gamma-diffuse`, `gamma` or `electron`
+        gamma_src_offset: str
+            for point source gammas only. `off0.0deg` or `off0.4deg`
+        Returns
+        -------
+        str: path to directory
+        """
+        base = self.base_dir.format(
+            data_level=data_level,
+            particle=particle,
+            zenith=self.zenith,
+            prod_id=prod_id
+        )
+        if particle == 'gamma':
+            base = os.path.join(base, gamma_src_offset)
+        return base
+
+    def r0_dir(self, particle, gamma_src_offset='off0.4deg'):
+        # for R0 dir there is no `prod_id` in the path
+        return os.path.realpath(self._data_level_dir(data_level='DL0',
+                                                     particle=particle,
+                                                     gamma_src_offset=gamma_src_offset,
+                                                     prod_id='')
+                                )
+
+    def dl1_dir(self, particle, gamma_src_offset='off0.4deg'):
+        return self._data_level_dir(data_level='DL1',
+                                    particle=particle,
+                                    gamma_src_offset=gamma_src_offset,
+                                    prod_id=self.prod_id)
+
+    def dl2_dir(self, particle, gamma_src_offset='off0.4deg'):
+        return self._data_level_dir(data_level='DL2',
+                                    particle=particle,
+                                    gamma_src_offset=gamma_src_offset,
+                                    prod_id=self.prod_id)
+
+    def irf_dir(self, gamma_src_offset='diffuse'):
+        return os.path.join(
+            os.path.realpath(self._data_level_dir(data_level='IRF',
+                                                  particle='',
+                                                  gamma_src_offset=gamma_src_offset,
+                                                  prod_id=self.prod_id)
+                             ),
+            gamma_src_offset
+        )
+
+    @property
+    def r0_to_dl1(self):
+        paths = []
+        for particle in self.particles:
+            if particle == 'gamma':
+                for offset in self.point_src_offsets:
+                    r0 = self.r0_dir(particle=particle, gamma_src_offset=offset)
+                    dl1 = self.dl1_dir(particle=particle, gamma_src_offset=offset)
+                    paths.append({'input': r0, 'output': dl1})
+            else:
+                r0 = self.r0_dir(particle=particle)
+                dl1 = self.dl1_dir(particle=particle)
+                paths.append({'input': r0, 'output': dl1})
+        return paths
+
+    def train_dir(self, particle, gamma_src_offset='off0.4deg'):
+        return os.path.join(
+            self.dl1_dir(particle, gamma_src_offset=gamma_src_offset),
+            'train'
+        )
+
+    def test_dir(self, particle, gamma_src_offset='off0.4deg'):
+        return os.path.join(
+            self.dl1_dir(particle, gamma_src_offset=gamma_src_offset),
+            'test'
+        )
+
+    @property
+    def train_test_split(self):
+        paths = []
+        for particle in self.particles:
+            if particle == 'gamma':
+                for offset in self.point_src_offsets:
+                    dl1 = self.dl1_dir(particle=particle, gamma_src_offset=offset)
+                    paths.append(
+                        {'input': dl1,
+                         'output': {
+                             'train': self.train_dir(particle),
+                             'test': self.test_dir(particle)}
+                         })
+            else:
+                dl1 = self.dl1_dir(particle=particle)
+                paths.append({
+                    'input': dl1,
+                    'output': {
+                        'train': self.train_dir(particle),
+                        'test': self.test_dir(particle)
+                    }
+                })
+        return paths
+
+    def merge_output_file(self, particle, step, gamma_src_offset='off0.4deg'):
+        if not step in ['train', 'test']:
+            raise ValueError("Only steps accepted: train or test")
+        dl1 = self.dl1_dir(particle=particle, gamma_src_offset=gamma_src_offset)
+        return os.path.join(dl1, f'dl1_{particle}_{self.prod_id}_{step}.h5')
+
+    @property
+    def merge_dl1(self):
+        paths = []
+        for particle in self.training_particles:
+            if particle == 'gamma':
+                for offset in self.point_src_offsets:
+                    train = self.train_dir(particle)
+                    output_file = self.merge_output_file(particle=particle, step='train', gamma_src_offset=offset)
+                    paths.append({
+                        'input': train,
+                        'output': output_file,
+                        'options': '--no-image'
+                    })
+            else:
+                train = self.train_dir(particle)
+                output_file = self.merge_output_file(particle=particle, step='train')
+                paths.append({
+                    'input': train,
+                    'output': output_file,
+                    'options': '--no-image'
+                })
+
+        for particle in self.testing_particles:
+            if particle == 'gamma':
+                for offset in self.point_src_offsets:
+                    test = self.test_dir(particle)
+                    output_file = self.merge_output_file(particle=particle, step='test', gamma_src_offset=offset)
+                    paths.append({
+                        'input': test,
+                        'output': output_file
+                    })
+            else:
+                test = self.test_dir(particle)
+                output_file = self.merge_output_file(particle=particle, step='test')
+                paths.append({
+                    'input': test,
+                    'output': output_file
+                })
+
+        self.paths['merge_dl1'] = paths
+        return paths
+
+    def models_path(self):
+        p = self.base_dir.format(
+            data_level='models',
+            particle='',
+            zenith=self.zenith,
+            prod_id=self.prod_id
+        ).replace('/mc/', '/')
+
+        return os.path.realpath(p)
+
+    @property
+    def train_pipe(self):
+        paths = [{
+            'input': {
+                'gamma': self.merge_output_file('gamma-diffuse', 'train'),
+                'proton': self.merge_output_file('proton', 'train')
+            },
+            'output': self.models_path()
+        }]
+        return paths
+
+    @property
+    def dl1_to_dl2(self):
+        paths = []
+        for particle in self.testing_particles:
+            if particle == 'gamma':
+                for offset in self.point_src_offsets:
+                    dl1 = self.merge_output_file(particle=particle, step='test', gamma_src_offset=offset)
+                    dl2 = self.dl2_dir(particle, gamma_src_offset=offset)
+                    paths.append({
+                        'input': dl1,
+                        'path_model': self.models_path(),
+                        'output': dl2
+                    })
+            else:
+                dl1 = self.merge_output_file(particle=particle, step='test')
+                dl2 = self.dl2_dir(particle)
+                paths.append({
+                    'input': dl1,
+                    'path_model': self.models_path(),
+                    'output': dl2
+                })
+        return paths
+
+    def dl2_output_file(self, particle, gamma_src_offset='off0.4deg'):
+        dl2_filename = os.path.basename(self.merge_output_file(particle=particle,
+                                                               step='test',
+                                                               gamma_src_offset=gamma_src_offset)).replace('dl1', 'dl2')
+        return os.path.join(
+            self.dl2_dir(particle=particle, gamma_src_offset=gamma_src_offset),
+            dl2_filename
+        )
+
+    def sensitivity_file(self, offset):
+        return os.path.join(
+            self.irf_dir(gamma_src_offset=offset),
+            f'sensitivity_{self.prod_id}_{offset}.fits.gz'
+        )
+
+    @property
+    def dl2_to_sensitivity(self):
+        paths = []
+
+        def path_dict(gamma_part, offset):
+            d = {
+                'input': {
+                    'gamma_file': self.dl2_output_file(gamma_part),
+                    'proton_file': self.dl2_output_file('proton'),
+                    'electron_file': self.dl2_output_file('electron')
+                },
+                'output': self.sensitivity_file(offset)
+            }
+            return d
+
+        # sensitivity can be computed only on point source gammas at the moment
+        for offset in self.point_src_offsets:
+            paths.append(path_dict('gamma', offset))
+
+        return paths
+
+    @property
+    def dl2_to_irfs(self):
+        paths = []
+
+        def path_dict(gamma_part, offset):
+            d = {
+                'input': {
+                    'gamma_file': self.dl2_output_file(gamma_part),
+                    'proton_file': self.dl2_output_file('proton'),
+                    'electron_file': self.dl2_output_file('electron')
+                },
+                'output': os.path.join(self.irf_dir(gamma_src_offset=offset), f'irf_{self.prod_id}_{offset}.fits.gz'),
+                'options': '--point-like' if gamma_part == 'gamma' else ''
+            }
+            return d
+
+        for gamma_part in ['gamma-diffuse', 'gamma']:
+            if gamma_part == 'gamma-diffuse':
+                paths.append(path_dict(gamma_part, 'diffuse'))
+            else:
+                for offset in self.point_src_offsets:
+                    paths.append(path_dict(gamma_part, offset))
+
+        return paths
+
+    @property
+    def plot_irfs(self):
+        paths = []
+
+        def path_dict(offset):
+            d = {
+                'input': self.sensitivity_file(offset),
+                'output': self.sensitivity_file(offset).replace('fits.gz', 'png')
+            }
+            return d
+
+        for gamma_part in ['gamma-diffuse', 'gamma']:
+            if gamma_part == 'gamma-diffuse':
+                paths.append(path_dict('diffuse'))
+            else:
+                for offset in self.point_src_offsets:
+                    paths.append(path_dict(offset))
+        return paths
+
+
+class PathConfigProd5Trans80DL1ab(PathConfigProd5Trans80):
+
+    def __init__(self, starting_prod_id, new_prod_id, zenith='zenith_20deg'):
+        super(PathConfigProd5Trans80DL1ab, self).__init__(prod_id=new_prod_id, zenith=zenith)
+        self.starting_prod_id = starting_prod_id
+        self.stages.remove('r0_to_dl1')
+        self.stages.remove('train_test_split')
+        self.stages.remove('merge_dl1')
+        self.stages.insert(0, 'dl1ab')
+
+    def starting_dl1(self, particle, step, gamma_src_offset='off0.4deg'):
+        former_merged_dl1 = self.merge_output_file(particle=particle, step=step, gamma_src_offset=gamma_src_offset)
+        return former_merged_dl1.replace(self.prod_id, self.starting_prod_id)
+
+    @property
+    def dl1ab(self):
+        paths = []
+        for step in ['train', 'test']:
+            for particle in self.particles:
+                if particle == 'gamma':
+                    for offset in self.point_src_offsets:
+                        dl1_input = self.starting_dl1(particle=particle, step=step, gamma_src_offset=offset)
+                        dl1_output = self.dl1_dir(particle, gamma_src_offset=offset)
+                        paths.append({
+                            'input': dl1_input,
+                            'output': dl1_output
+                        })
+                else:
+                    dl1_input = self.starting_dl1(particle=particle, step=step, gamma_src_offset='')
+                    dl1_output = self.dl1_dir(particle, gamma_src_offset='')
+                    paths.append({
+                        'input': dl1_input,
+                        'output': dl1_output
+                    })
+        return paths
+
+
+class PathConfigAllSky(PathConfig):
+    """
+    Standard paths configuration for a prod5_trans_80 MC production
+    """
+
+    def __init__(self, prod_id):
+        super().__init__(prod_id)
+        self.prod_id = prod_id
+        self.base_dir = "/fefs/aswg/data/mc/{data_level}/AllSky/{prod_id}/{particle}/{pointing}"
+        self.training_dir = \
+            "/home/georgios.voutsinas/ws/AllSky/TrainingDataset/{particle}/sim_telarray/{pointing}/output_v1.4"
+        self.testing_dir = "/home/georgios.voutsinas/ws/AllSky/TestDataset/Crab/sim_telarray/{pointing}/output_v1.4"
+        self.training_particles = ['GammaDiffuse', 'Protons']
+        self.testing_particles = ['Crab']
+
+        self.paths = {}
+        self.stages = ['r0_to_dl1', 'merge_dl1', 'train_pipe', 'dl1_to_dl2', 'dl2_to_irfs']
+
+    def _search_pointings(self, particle):
+        pointing_dirs = os.listdir(self.r0_dir(particle=particle, pointing='$$$').split('$$$')[0])
+        return pointing_dirs
+
+    def training_pointings(self, particle):
+        if not hasattr(self, '_training_pointings'):
+            try:
+                self.load_pointings()
+            except FileNotFoundError as e:
+                raise FileNotFoundError(
+                    "The class must be run on the cluster to load available pointing nodes"
+                ) from e
+        return self._training_pointings[particle]
+
+    def testing_pointings(self, particle):
+        if not hasattr(self, '_testing_pointings'):
+            try:
+                self.load_pointings()
+            except FileNotFoundError as e:
+                raise FileNotFoundError(
+                    "The class must be run on the cluster to load available pointing nodes"
+                ) from e
+        return self._testing_pointings[particle]
+
+    def load_pointings(self):
+        self._training_pointings = self._get_training_pointings()
+        self._testing_pointings = self._get_testing_pointings()
+
+    def _extract_pointing(self, text):
+        """
+        return a tuple ($0, $1) of pointings based on a text pattern `*_theta_{$0}_az_{$1}_`
+        """
+        return re.search('.*theta\_(.+?)_az\_(.+?)\_', text)
+
+    def _get_training_pointings(self):
+        """
+        Find pointings that exist for all training particles
+        This is overly complicated because pointings directory names are not consistent particle-wise
+        see node_theta_16.087_az_108.090_ vs node_corsika_theta_16.087_az_108.090_
+        see testing pointings for a simpler implementation if this get solved
+        """
+        all_pointings = {}
+        for particle in self.training_particles:
+            all_pointings[particle] = self._search_pointings(particle)
+        intersected_pointings = deepcopy(all_pointings)
+
+        for particle, pointings_text in all_pointings.items():
+            for pointing_text in pointings_text:
+                pointing_tuple = self._extract_pointing(pointing_text)
+                for other_particles, other_pointings_text in all_pointings.items():
+                    other_pointings_tuples = [self._extract_pointing(pt) for pt in other_pointings_text]
+                    if pointing_tuple not in other_pointings_tuples:
+                        if pointing_text in intersected_pointings:
+                            intersected_pointings[particle].remove(pointing_text)
+
+        return intersected_pointings
+
+    def _get_testing_pointings(self):
+        particle = self.testing_particles[0]
+        pointings = set(self._search_pointings(particle))
+        for particle in self.testing_particles[1:]:
+            pointings.intersection_update(self._search_pointings(particle))
+        return {particle: pointings for particle in self.testing_particles}
+
+    def _data_level_dir(self, prod_id, data_level, particle, pointing):
+        """
+
+        Parameters
+        ----------
+        data_level: str
+            `DL0` or `DL1` or `DL2`
+        particle: str
+            `proton`, `gamma-diffuse`, `gamma` or `electron`
+        prod_id:
+        pointing:
+
+        Returns
+        -------
+        str: path to directory
+        """
+        if data_level not in ['models', 'DL1', 'DL2', 'IRF']:
+            raise ValueError("data_level should be DL1, DL2 or IRF")
+        return self.base_dir.format(data_level=data_level, particle=particle, pointing=pointing, prod_id=prod_id)
+
+    def r0_dir(self, particle, pointing):
+        # for R0 dir there is no `prod_id` in the path
+        if particle in self.training_particles:
+            return self.training_dir.format(particle=particle, pointing=pointing)
+        elif particle in self.testing_particles:
+            return self.testing_dir.format(pointing=pointing)
+        else:
+            raise ValueError("unknown particle")
+
+    def dl1_dir(self, particle, pointing):
+        return self._data_level_dir(data_level='DL1', particle=particle, pointing=pointing, prod_id=self.prod_id)
+
+    def dl2_dir(self, particle, pointing):
+        return self._data_level_dir(data_level='DL2', particle=particle, pointing=pointing, prod_id=self.prod_id)
+
+    def irf_dir(self, pointing):
+        return os.path.realpath(self._data_level_dir(data_level='IRF',
+                                                     particle='',
+                                                     pointing=pointing,
+                                                     prod_id=self.prod_id)
+                                )
+
+    @property
+    def r0_to_dl1(self):
+        paths = []
+        for particle in self.training_particles:
+            for pointing in self.training_pointings(particle):
+                r0 = self.r0_dir(particle, pointing)
+                dl1 = self.dl1_dir(particle, pointing)
+                paths.append({'input': r0, 'output': dl1})
+        for particle in self.testing_particles:
+            for pointing in self.testing_pointings(particle):
+                r0 = self.r0_dir(particle, pointing)
+                dl1 = self.dl1_dir(particle, pointing)
+                paths.append({'input': r0, 'output': dl1})
+        return paths
+
+    def training_merged_dl1(self, particle):
+        return os.path.join(
+            os.path.realpath(self.dl1_dir(particle, '')),
+            f'dl1_{particle}_merged.h5'
+        )
+
+    def testing_merged_dl1(self, particle, pointing):
+        return os.path.join(
+            os.path.realpath(self.dl1_dir(particle, '')),
+            f'dl1_{particle}_{pointing}_merged.h5'
+        )
+
+    @property
+    def merge_dl1(self):
+        # for the training particles, all the nodes get merged
+        paths = []
+        for particle in self.training_particles:
+            dl1 = self.dl1_dir(particle, '')
+            merged_dl1 = self.training_merged_dl1(particle)
+            paths.append({
+                'input': dl1,
+                'output': merged_dl1,
+                'options': '--pattern */*.h5 --no-image',
+            })
+
+        # for the testing, we merge per node
+        for particle in self.testing_particles:
+            for pointing in self.testing_pointings(particle):
+                dl1 = self.dl1_dir(particle, pointing)
+                merged_dl1 = self.testing_merged_dl1(particle, pointing)
+                paths.append({
+                    'input': dl1,
+                    'output': merged_dl1,
+                    'options': '--no-image'
+                })
+
+        return paths
+
+    def models_path(self):
+        p = self.base_dir.format(data_level='models', particle='', pointing='', prod_id=self.prod_id).replace(
+            '/mc/', '/')
+        return os.path.realpath(p)
+
+    @property
+    def train_pipe(self):
+        paths = [{
+            'input': {
+                'gamma': self.training_merged_dl1('GammaDiffuse'),
+                'proton': self.training_merged_dl1('Protons'),
+            },
+            'output': self.models_path()
+        }]
+        return paths
+
+    @property
+    def dl1_to_dl2(self):
+        paths = []
+        for particle in self.testing_particles:
+            for pointing in self.testing_pointings(particle):
+                paths.append({
+                    'input': self.testing_merged_dl1(particle, pointing),
+                    'path_model': self.models_path(),
+                    'output': self.dl2_dir(particle, pointing)
+                })
+        return paths
+
+    def dl2_output_file(self, particle, pointing):
+        filename = os.path.basename(self.testing_merged_dl1(particle, pointing).replace('dl1_', 'dl2_'))
+        return os.path.join(self.dl2_dir(particle, pointing), filename)
+
+    @property
+    def dl2_to_irfs(self):
+        paths = []
+
+        for particle in self.testing_particles:
+            for pointing in self.testing_pointings(particle):
+                paths.append({
+                    'input': {'gamma_file': self.dl2_output_file(particle, pointing),
+                              'proton_file': None,
+                              'electron_file': None,
+                              },
+                    'output': os.path.join(self.irf_dir(pointing), f'irf_{self.prod_id}_{pointing}.fits.gz'),
+                    'options': '--point-like'
+                }
+                )
+
+        return paths
diff --git a/lstmcpipe/config/pipeline_config.py b/lstmcpipe/config/pipeline_config.py
index fe722d0b..2c84a88c 100644
--- a/lstmcpipe/config/pipeline_config.py
+++ b/lstmcpipe/config/pipeline_config.py
@@ -1,24 +1,12 @@
-import os
-import yaml
+#!/usr/bin/env python
+
+from ruamel.yaml import YAML
 import calendar
 import logging
-from pathlib import Path
-
 
 log = logging.getLogger(__name__)
 
-
-def create_path(
-    parent_path, stage, obs_date, pointing_zenith, prod_id=None, particles=True
-):
-    p = Path(parent_path) / stage / obs_date
-    if particles:
-        p /= "{}"
-    p /= pointing_zenith
-    if prod_id:
-        p /= prod_id
-    return p.as_posix()  # lets play it safe for now
-
+    
 
 def load_config(config_path):
     """
@@ -38,59 +26,29 @@ def load_config(config_path):
 
     # This could easily be adapted to support different formats
     with open(config_path) as f:
-        loaded_config = yaml.safe_load(f)
+        loaded_config = YAML().load(f)
 
     config_valid(loaded_config)
 
-    config = parse_config_and_handle_global_vars(loaded_config)
+    config = complete_lstmcpipe_config(loaded_config)
 
     log.info(
-        f'************ - Configuration for processing of {config["prod_type"]} using the {config["workflow_kind"]} pipeline:- ************'
-    )
-    if config.get("DL0_input_dir"):
-        log.info(
-            "Simtel DL0 files are going to be searched at: "
-            f'{config["DL0_input_dir"].format("{" + ",".join(config["all_particles"]) + "}")}'
-        )
-    elif config.get("DL1_input_dir"):
-        log.info(
-            "DL1 base files are going to be searched at: "
-            f'{config["DL1_input_dir"].format("{" + ",".join(config["all_particles"]) + "}")}'
-        )
-
-    particle_dirs = [
-        "running_analysis_dir",
-        "DL1_output_dir",
-        "DL2_output_dir",
-        "analysis_log_dir",
-    ]
-    log.info(
-        "The following directories and all the information within them will be either created or overwritten:\n - "
-        + "\n - ".join(
-            [
-                config[pd].format("{" + ",".join(config["all_particles"]) + "}")
-                for pd in particle_dirs
-            ]
-        )
-        + f'\n - {config["IRF_output_dir"]}'
-        + f'\n - {config["model_output_dir"]}'
+        f'************ - lstMCpipe will be launch using the {config["workflow_kind"]} pipeline:- ************'
     )
+    log.info(f'\nPROD_ID to be used: {config["prod_id"]}')
+    log.info("\nStages to be run:\n - " + "\n - ".join(config["stages_to_run"]))
 
-    log.warning(
-        "! Subdirectories with the same PROD_ID and analysed the same day will be overwritten !"
-    )
-    log.info(f'PROD_ID to be used: {config["prod_id"]}')
 
-    log.info("Stages to be run:\n - " + "\n - ".join(config["stages_to_run"]))
-    if "dl1ab" in config["stages_to_run"]:
-        log.info(f'Applying dl1ab processing to MC prod: {config["dl1_reference_id"]}')
-    log.info("Merging options:" f"\n - No-image argument: {config['merging_no_image']}")
     log.info(
         "Slurm configuration:"
         + f"\n - Source environment: {config['batch_config']['source_environment']}"
         + f"\n - Slurm account: {config['batch_config']['slurm_account']}"
     )
 
+    log.warning(
+        "\n! Subdirectories with the same PROD_ID and analysed the same day will be overwritten !"
+    )
+
     return config
 
 
@@ -111,33 +69,33 @@ def config_valid(loaded_config):
     True if config is valid
     """
     # Allowed options
+    compulsory_entries = [
+        "workflow_kind",
+        "source_environment",
+        "stages_to_run",
+        "stages",
+        # TODO dl1_reference_id ?
+    ]
     allowed_workflows = ["hiperta", "lstchain", "ctapipe"]
-    allowed_prods = ["prod3", "prod5"]
-    allowed_obs_date = ["20190415", "20200629_prod5", "20200629_prod5_trans_80"]
+
 
     # Check allowed cases
+    for item in compulsory_entries:
+        if item not in loaded_config:
+            raise Exception(
+                f"The lstMCpipe configuration was not generated correctly. \n"
+                f"Missing: {item} key"
+            )
+
     workflow_kind = loaded_config["workflow_kind"]
+
     if workflow_kind not in allowed_workflows:
         raise Exception(
             f"Please select an allowed `workflow_kind`: {allowed_workflows}"
         )
 
-    prod_type = loaded_config["prod_type"]
-    if prod_type not in allowed_prods:
-        raise Exception(f"Please selected an allowed production type: {allowed_prods}.")
-
-    obs_date = loaded_config["obs_date"]
 
-    if obs_date not in allowed_obs_date:
-        raise Exception(f"Please select an allowed obs_date: {allowed_workflows}")
-
-    # and incompatible possibilities
-    if (prod_type == "prod3" and obs_date != "20190415") or (
-        prod_type == "prod5" and obs_date == "20190415"
-    ):
-        raise Exception("This prod_type and obs_date combination is not possible.")
-
-    stages_to_be_run = loaded_config["stages_to_be_run"]
+    stages_to_be_run = loaded_config["stages_to_run"]
     if "dl1ab" in stages_to_be_run:
         if not "dl1_reference_id" in loaded_config:
             raise KeyError(
@@ -154,72 +112,62 @@ def config_valid(loaded_config):
         )
     elif not dl1_noise_tune_data_run and dl1_noise_tune_mc_run:
         raise KeyError("Please specify an observed dl1 file to " "tune the images.")
+
     log.debug("Configuration deemed valid")
+
     return True
 
 
-def parse_config_and_handle_global_vars(loaded_config):
+def complete_lstmcpipe_config(loaded_config):
     """
-    Generates the config, that is used in the pipeline.
-    Sets file paths for the la palma cluster
-    and handles differences between prod3 and prod5.
+    Completes (and get default values) of some entries of the lstmcpipe config
 
     Parameters
     ----------
     loaded_config: dict
-        Dictionary with the values read from the config file.
-        No checks on the correctness of the data is applied,
-        so this has to be done before.
+        Loaded config generated by lstmcpipe PathConfig
 
     Returns
     -------
     config : dict
-        Dictionary with all the variables needed along the r0_to_dl3 workflow
+        Dictionary with all the key:values completed along the r0_to_dl3 workflow
 
     """
-    config = {}
-
-    prod_id = loaded_config.get("prod_id", "v00")
-    stages_to_be_run = loaded_config["stages_to_be_run"]
-    merging_options = loaded_config["merging_options"]["no_image"]
-    base_path = loaded_config.get("base_path")
-    # to locate the source dl1 files
-    dl1_reference_id = loaded_config.get("dl1_reference_id")
-    # Full path to an observed dl1 file
-    dl1_noise_tune_data_run = loaded_config.get("dl1_noise_tune_data_run")
-    dl1_noise_tune_mc_run = loaded_config.get("dl1_noise_tune_mc_run")
+    config = loaded_config.copy()
 
-    pointing = loaded_config["pointing"]
-    zenith = loaded_config["zenith"]
-    particles = loaded_config["particles"]
-    offset_gammas = loaded_config["offset_gammas"]
-    obs_date = loaded_config["obs_date"]
-    prod_type = loaded_config["prod_type"]
+    suffix_prod_id = loaded_config.get("prod_id", "v00")
     workflow_kind = loaded_config["workflow_kind"]
 
+    # TODO ??
+    # # to locate the source dl1 files
+    # dl1_reference_id = loaded_config.get("dl1_reference_id")
+    # # Full path to an observed dl1 file
+    # dl1_noise_tune_data_run = loaded_config.get("dl1_noise_tune_data_run")
+    # dl1_noise_tune_mc_run = loaded_config.get("dl1_noise_tune_mc_run")
+
     # Prod_id syntax
     t = calendar.datetime.date.today()
     year, month, day = f"{t.year:04d}", f"{t.month:02d}", f"{t.day:02d}"
     if workflow_kind == "lstchain":
-        import lstchain
 
+        import lstchain
         base_prod_id = f"{year}{month}{day}_v{lstchain.__version__}"
+
     elif workflow_kind == "ctapipe":
-        import ctapipe
 
+        import ctapipe
         base_prod_id = f"{year}{month}{day}_vctapipe{ctapipe.__version__}"
+
     elif workflow_kind == "hiperta":  # RTA
+
         # TODO parse version from hiPeRTA module
         import lstchain
-
-        base_prod_id = f"{year}{month}{day}_vRTA320_v{lstchain.__version__}"
+        base_prod_id = f"{year}{month}{day}_vRTA420_v{lstchain.__version__}"
 
     # Create the final config structure to be passed to the pipeline
     # 1 - Prod_id
-    if "trans_80" in obs_date:
-        suffix_id = "_{}_trans_80_{}".format(prod_type, prod_id)
-    else:
-        suffix_id = "_{}_{}".format(prod_type, prod_id)
+    prod_type = f"_{loaded_config['prod_type']}" if 'prod_type' in loaded_config else ''
+    suffix_id = "{}_{}".format(prod_type, suffix_prod_id)
     config["prod_id"] = base_prod_id + suffix_id
 
     # 2 - Parse source environment correctly
@@ -236,67 +184,9 @@ def parse_config_and_handle_global_vars(loaded_config):
         "slurm_account": slurm_account,
     }
 
-    # 3 - particles loop
-    config["all_particles"] = particles
-
-    # 4 - Stages to be run
-    config["stages_to_run"] = stages_to_be_run
-    config["merging_no_image"] = merging_options
-
-    # 5 - production workflow and type
-    config["workflow_kind"] = workflow_kind
-    config["prod_type"] = prod_type
-
-    # 6 - Global paths
-    if prod_type == "prod3":
-        pointing_zenith = pointing
-        config["gamma_offs"] = None
-
-    else:  # Prod5
-        # TODO correct the path for all the prod5_* at the LP_cluster
-        #  as they were run as $ZENITH/$POINTING instead of $POINTING/$ZENITH/
-        pointing_zenith = os.path.join(zenith, pointing)
-        config["gamma_offs"] = offset_gammas
-
-    if "r0_to_dl1" in stages_to_be_run:
-        source_datalevel = "DL0"
-    elif "dl1ab" in stages_to_be_run:
-        source_datalevel = "DL1"
-    else:
-        raise NotImplementedError(
-            "Starting the processing from higher stages is not supported"
-        )
-    if workflow_kind == "lstchain" or workflow_kind == "ctapipe":
-        config["input_dir"] = create_path(
-            base_path, source_datalevel, obs_date, pointing_zenith
-        )
-    else:  # RTA
-        config["input_dir"] = create_path(base_path, "R0", obs_date, pointing_zenith)
-    config["dl1_reference_id"] = dl1_reference_id
-    config["dl1_noise_tune_data_run"] = dl1_noise_tune_data_run
-    config["dl1_noise_tune_mc_run"] = dl1_noise_tune_mc_run
-
-    directories = {
-        "running_analysis_dir": "running_analysis",
-        "analysis_log_dir": "analysis_logs",
-        "DL1_output_dir": "DL1",
-        "DL2_output_dir": "DL2",
-    }
-    for key, value in directories.items():
-        config[key] = create_path(
-            base_path, value, obs_date, pointing_zenith, config["prod_id"]
-        )
+    # TODO ??
+    # config["dl1_reference_id"] = dl1_reference_id
+    # config["dl1_noise_tune_data_run"] = dl1_noise_tune_data_run
+    # config["dl1_noise_tune_mc_run"] = dl1_noise_tune_mc_run
 
-    config["IRF_output_dir"] = create_path(
-        base_path, "IRF", obs_date, pointing_zenith, config["prod_id"], particles=False
-    )
-    # small difference if the user is the lstanalyzer
-    config["model_output_dir"] = create_path(
-        "fefs/aswg/data" if base_path == "/fefs/aswg/data/mc" else base_path,
-        "models",
-        obs_date,
-        pointing_zenith,
-        config["prod_id"],
-        particles=False,
-    )
     return config
diff --git a/lstmcpipe/config_MC_dl1ab_prod.yml b/lstmcpipe/config_MC_dl1ab_prod.yml
index e2b82d65..47672590 100644
--- a/lstmcpipe/config_MC_dl1ab_prod.yml
+++ b/lstmcpipe/config_MC_dl1ab_prod.yml
@@ -46,7 +46,7 @@ slurm_config:
 ## DL1AB related settings
 
 # Set dl1ab as first step
-stages_to_be_run:
+stages_to_run:
   - dl1ab
   - merge_and_copy_dl1
   - train_pipe
diff --git a/lstmcpipe/config_MC_prod.yml b/lstmcpipe/config_MC_prod.yml
deleted file mode 100644
index 0692375c..00000000
--- a/lstmcpipe/config_MC_prod.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-## MC PRODUCTION FULL PIPELINE CONFIGURATION ##
-#
-# The entries are needed to correctly select the MC production type
-#  and to create the common path tree;
-#
-#
-#   $BASE_PATH_DL0/$STAGE/$OBS_DATE/$PARTICLE/$ZENITH/$POINTING/$PROD_ID
-#
-#
-#  This way every MC production will follow the same path logic;
-#    ex;  $BASE_PATH_DL0/{DL0, DL1, DL2, running_analysis}/20200629_prod5/{electron, gamma, gamma-diffuse, proton}/
-#          /zenith_20deg/south_pointing/$PROD_ID
-
-
-workflow_kind: lstchain  # Not expected to be changed by the user
-#prod_id:                 # ex: local_no_n_islands. Default; v00 (if key left empty or None)
-
-base_path: /fefs/aswg/workspace/MC_data_simlink  # Default user path
-  # lstanalyzer user : /fefs/aswg/data/mc
-  #
-prod_type: prod5
-obs_date: 20200629_prod5_trans_80
-zenith: zenith_20deg
-pointing: south_pointing
-
-particles:
-  - electron
-  - gamma
-  - gamma-diffuse
-  - proton
-offset_gammas:
-  - off0.0deg
-  - off0.4deg
-
-source_environment:
-  source_file: /fefs/aswg/software/conda/etc/profile.d/conda.sh
-  conda_env: lstchain-v0.9.3
-  #   source_file: /fefs/home/enrique.garcia/.bashrc
-  #   conda_env: lst-dev
-
-stages_to_be_run:
-  - r0_to_dl1
-  - merge_and_copy_dl1
-  - train_pipe
-  - dl1_to_dl2
-  - dl2_to_irfs
-  - dl2_to_sensitivity
-
-merging_options:
-  no_image: True
-
-slurm_config:
-  user_account: aswg
diff --git a/lstmcpipe/io/data_management.py b/lstmcpipe/io/data_management.py
index 05d287d2..0e1b6df5 100644
--- a/lstmcpipe/io/data_management.py
+++ b/lstmcpipe/io/data_management.py
@@ -1,8 +1,5 @@
 # library of functions used for LST analysis data management
 
-# Thomas Vuillaume, 12/09/2019
-# Enrique Garcia 09/04/2020
-
 import os
 import sys
 import shutil
@@ -70,33 +67,6 @@ def query_continue(question, default="no"):
         return answer
 
 
-# def get_metadata_from_mc_data_path(data_path):
-#     '''
-#     A mc data path is always `/BASE_DIR/data/mc/dlx/<date>/particle_type/pointing/`
-#
-#     Returns
-#     -------
-#     dict
-#     '''
-#
-#     if not data_path.startswith(BASE_DIR):
-#         data_path = os.path.join(BASE_DIR, data_path)
-#     if not os.path.exists(BASE_DIR):
-#         raise ValueError("The input path does not exists")
-#
-#     split = data_path.split('/')
-#     if split[-5] != 'mc':
-#         raise ValueError("The path structure does not correspond to the intended one")
-#
-#     dic = {
-#         'pointing': split[-1],
-#         'particle_type': split[-2],
-#         'date': split[-3],
-#         'data_level': split[-4],
-#     }
-#     return dic
-
-
 def check_data_path(data_path):
     """
     Check if the path to some data exists.
@@ -161,44 +131,6 @@ def check_and_make_dir_without_verification(directory):
     os.makedirs(directory, exist_ok=True)
 
 
-def check_job_logs(job_logs_dir):
-    job_logs = [
-        os.path.join(job_logs_dir, f)
-        for f in os.listdir(job_logs_dir)
-        if f.endswith(".e")
-    ]
-    logs_with_error = []
-    for log_filename in job_logs:
-        with open(log_filename) as log_file:
-            for line in log_file.readlines():
-
-                if (
-                    "Remote data cache could not be accessed due to FileNotFoundError"
-                    in line
-                    or "URLError(" in line
-                    or "<urlopen error Unable to open any source!" in line
-                    or "ftp error: timeout(" in line
-                ):
-
-                    continue  # Known astropy errors, they do not break the workflow
-
-                elif (
-                    line.startswith("Error")
-                    or line.startswith("ERROR")
-                    or "ERROR" in line
-                    or "Error" in line
-                ):
-                    logs_with_error.append(os.path.basename(log_filename))
-                    break
-
-    if not logs_with_error == []:
-        query_continue(
-            "There are errors in the following log files:\n {}\n "
-            "Are you sure you want to continue?".format(logs_with_error),
-            default="no",
-        )
-
-
 def check_files_in_dir_from_file(directory, file):
     """
     Check that a list of files from a file exist in a dir
diff --git a/lstmcpipe/io/lstmcpipe_tree_path.py b/lstmcpipe/io/lstmcpipe_tree_path.py
index 8c6021c6..385d92ca 100644
--- a/lstmcpipe/io/lstmcpipe_tree_path.py
+++ b/lstmcpipe/io/lstmcpipe_tree_path.py
@@ -2,7 +2,7 @@
 
 import shutil
 from pathlib import Path
-from lstmcpipe import lstmcpipe_root_dir, __version__
+from lstmcpipe import prod_logs, __version__
 
 
 def backup_log(file):
@@ -39,7 +39,7 @@ def create_log_dir(prod_id):
     log_dir : Path
         Path to `prod_id` log directory
     """
-    log_dir = lstmcpipe_root_dir.joinpath("prod_logs", f"logs_{prod_id}")
+    log_dir = prod_logs.joinpath(f"logs_{prod_id}")
     log_dir.mkdir(exist_ok=True, parents=True)
     return log_dir
 
diff --git a/lstmcpipe/lstmcpipe_start.py b/lstmcpipe/lstmcpipe_start.py
index 4c324049..6f32a475 100755
--- a/lstmcpipe/lstmcpipe_start.py
+++ b/lstmcpipe/lstmcpipe_start.py
@@ -1,14 +1,16 @@
 # #!/usr/bin/env python
 
-# E. Garcia, Jan '20
+# E. Garcia, T. VUILLAUME, LAPP, CNRS
 #
 # Full workflow from r0 to dl3.
 #  wraps together the individual scripts of
-#   - onsite_mc_r0_to_dl1
-#   - onsite_mc_merge_and_copy_dl1
-#   - onsite_mc_train
-#   - onsite_mc_dl1_to_dl2
-#   - onsite_mc_dl2_to_irfs
+#   - mc_process_dl1
+#   - mc_train_test_split
+#   - mc_merge_dl1
+#   - mc_train
+#   - mc_dl1_to_dl2
+#   - mc_dl2_to_irfs
+#   - mc_dl2_to_sensitivity
 #
 # usage:
 # > lstmcpipe -c config_MC_prod.yml -conf_lst LSTCHAIN_CONFIG_FILE [-conf_rta RTA_CONFIG_FILE]
@@ -25,12 +27,12 @@
     update_scancel_file,
 )
 from lstmcpipe.workflow_management import (
-    create_dl1_filenames_dict,
     batch_mc_production_check,
 )
 from lstmcpipe.stages import (
     batch_process_dl1,
-    batch_merge_and_copy_dl1,
+    batch_train_test_splitting,
+    batch_merge_dl1,
     batch_train_pipe,
     batch_dl1_to_dl2,
     batch_dl2_to_irfs,
@@ -60,7 +62,7 @@
     dest="config_file_lst",
     help="Path to a lstchain-like configuration file. "
     "RF classifier and regressor arguments must be declared here !",
-    default=None,
+    required=True
 )
 
 parser.add_argument(
@@ -128,37 +130,18 @@ def main():
     log = setup_logging(verbose=args.debug, logfile=args.log_file)
     log.info("Starting lstmcpipe processing script")
     # Read MC production configuration file
-    config = load_config(args.config_mc_prod)
+    lstmcpipe_config = load_config(args.config_mc_prod)
     query_continue("Are you sure ?")
 
     # Load variables
-    prod_id = config["prod_id"]
-    workflow_kind = config["workflow_kind"]
-    batch_config = config["batch_config"]
-    stages_to_run = config["stages_to_run"]
-    all_particles = config["all_particles"]
-    input_dir = config["input_dir"]
-    dl1_output_dir = config["DL1_output_dir"]
-    dl2_output_dir = config["DL2_output_dir"]
-    running_analysis_dir = config["running_analysis_dir"]
-    gamma_offs = config.get("gamma_offs")
-    no_image_merging = config.get("merging_no_image")
+    prod_id = lstmcpipe_config["prod_id"]
+    workflow_kind = lstmcpipe_config["workflow_kind"]
+    batch_config = lstmcpipe_config["batch_config"]
+    stages_to_run = lstmcpipe_config["stages_to_run"]
 
     # Create log files and log directory
     logs_files, scancel_file, logs_dir = create_log_files(prod_id)
-
-    # Make sure the lstchain config is defined if needed
-    # It is not exactly required if you process only up to dl1
-    if any(
-        [
-            step not in ("r0_to_dl1", "dl1ab", "merge_and_copy_dl1")
-            for step in stages_to_run
-        ]
-    ):
-        if args.config_file_lst is None:
-            raise Exception(
-                "The lstchain config needs to be defined for all steps following dl1 processing"
-            )
+    all_job_ids = {}
 
     if "r0_to_dl1" in stages_to_run and "reprocess_dl1" in stages_to_run:
         raise Exception("There can only be one stage producing dl1 files")
@@ -168,172 +151,147 @@ def main():
     dl1ab = "dl1ab" in stages_to_run
 
     if r0_to_dl1 or dl1ab:
-        stage_input_dir = Path(input_dir)
+
         if workflow_kind == "lstchain":
             dl1_config = Path(args.config_file_lst).resolve().as_posix()
-            if config.get("dl1_noise_tune_data_run"):
-                # lstchain uses json, but just in case
-                assert ".json" in dl1_config
+            if lstmcpipe_config.get("dl1_noise_tune_data_run"):
                 dl1_config = create_dl1ab_tuned_config(
                     dl1_config,
                     dl1_config.replace(".json", "_tuning.json"),
-                    config.get("dl1_noise_tune_data_run"),
-                    config.get("dl1_noise_tune_mc_run"),
+                    lstmcpipe_config.get("dl1_noise_tune_data_run"),
+                    lstmcpipe_config.get("dl1_noise_tune_mc_run"),
                 )
         elif workflow_kind == "hiperta":
-            dl1_config = Path(args.config_file_rta).resolve()
-        else:  # if this wasnt ctapipe, the config parsing would have failed
-            dl1_config = Path(args.config_file_ctapipe).resolve()
-        if dl1ab:
-            stage_input_dir /= config["dl1_reference_id"]
-
-        particle2jobid_process_dl1_dict, jobs_all_dl1 = batch_process_dl1(
-            input_dir=stage_input_dir.as_posix(),
+            dl1_config = Path(args.config_file_rta).resolve().as_posix()
+        else:  # if this was not ctapipe, the config parsing would have failed
+            dl1_config = Path(args.config_file_ctapipe).resolve().as_posix()
+
+        jobs_from_dl1_processing = batch_process_dl1(
+            lstmcpipe_config['stages'],
             conf_file=dl1_config,
-            prod_id=prod_id,
-            particles_loop=all_particles,
             batch_config=batch_config,
-            gamma_offsets=gamma_offs,
             workflow_kind=workflow_kind,
             new_production=r0_to_dl1,
             logs=logs_files,
         )
 
-        update_scancel_file(scancel_file, jobs_all_dl1)
-
+        update_scancel_file(scancel_file, jobs_from_dl1_processing)
+        if r0_to_dl1:
+            all_job_ids.update({"r0_dl1": jobs_from_dl1_processing})
+        else:
+            all_job_ids.update({"dl1ab": jobs_from_dl1_processing})
     else:
-        jobs_all_dl1 = ""
-        particle2jobid_process_dl1_dict = {}
-        for particle in all_particles:
-            particle2jobid_process_dl1_dict[particle] = ""
-
-    # 2 STAGE --> Merge,copy and move DL1 files
-    if "merge_and_copy_dl1" in stages_to_run:
-        (
-            merged_dl1_paths_dict,
-            jobs_to_train,
-            jobs_all_dl1_finished,
-        ) = batch_merge_and_copy_dl1(
-            running_analysis_dir,
-            particle2jobid_process_dl1_dict,
-            all_particles,
-            smart_merge=False,  # smart_merge=WORKFLOW_KIND
-            no_image_flag=no_image_merging,
-            gamma_offsets=gamma_offs,
-            prod_id=prod_id,
+        jobs_from_dl1_processing = ""
+
+    # 2.1 STAGE --> Train, test splitting
+    if "train_test_split" in stages_to_run:
+        jobs_from_splitting = batch_train_test_splitting(
+            lstmcpipe_config['stages']['train_test_split'],
+            jobids_from_r0dl1=jobs_from_dl1_processing,
             batch_config=batch_config,
-            workflow_kind=workflow_kind,
             logs=logs_files,
         )
 
-        update_scancel_file(scancel_file, jobs_all_dl1_finished)
+        update_scancel_file(scancel_file, jobs_from_splitting)
+        all_job_ids.update({"train_test_split": jobs_from_splitting})
+    else:
+        jobs_from_splitting = ""
 
+    # 2.2 STAGE --> Merge DL1 files
+    if jobs_from_splitting != "":
+        merge_wait_jobs = ','.join([jobs_from_dl1_processing, jobs_from_splitting])
     else:
-        # Create just the needed dictionary inputs (dl1 files must exist !)
-        merged_dl1_paths_dict = create_dl1_filenames_dict(
-            dl1_output_dir, all_particles, gamma_offs
+        merge_wait_jobs = jobs_from_dl1_processing
+
+    if "merge_dl1" in stages_to_run:
+        jobs_from_merge = batch_merge_dl1(
+            lstmcpipe_config['stages']["merge_dl1"],
+            jobid_from_splitting=merge_wait_jobs,
+            batch_config=batch_config,
+            workflow_kind=workflow_kind,
+            logs=logs_files,
         )
-        jobs_to_train = ""
-        jobs_all_dl1_finished = ""
+
+        update_scancel_file(scancel_file, jobs_from_merge)
+        all_job_ids.update({"merge_and_copy_dl1": jobs_from_merge})
+    else:
+        jobs_from_merge = ""
 
     # 3 STAGE --> Train pipe
     if "train_pipe" in stages_to_run:
-        train_config = Path(args.config_file_lst)
 
-        job_from_train_pipe, model_dir = batch_train_pipe(
-            merged_dl1_paths_dict,
-            train_config,
-            jobs_to_train,
+        job_from_train_pipe = batch_train_pipe(
+            lstmcpipe_config['stages']['train_pipe'],
+            jobs_from_merge,
+            config_file=Path(args.config_file_lst).resolve().as_posix(),
             batch_config=batch_config,
             logs=logs_files,
         )
 
         update_scancel_file(scancel_file, job_from_train_pipe)
+        all_job_ids.update({"train_pipe": job_from_train_pipe})
 
         # Plot the RF feature's importance
-        batch_plot_rf_features(
-            model_dir,
-            args.config_file_lst,
+        job_from_plot_rf_feat = batch_plot_rf_features(
+            lstmcpipe_config['stages']['train_pipe'],
+            Path(args.config_file_lst).resolve().as_posix(),
             batch_config,
             job_from_train_pipe,
             logs=logs_files,
         )
+        update_scancel_file(scancel_file, job_from_plot_rf_feat)
+        all_job_ids.update({"plot_rf_feat": job_from_plot_rf_feat})
 
     else:
         job_from_train_pipe = ""
-        model_dir = config["model_output_dir"]
 
     # 4 STAGE --> DL1 to DL2 stage
     if "dl1_to_dl2" in stages_to_run:
-        dl1_to_dl2_config = Path(args.config_file_lst)
 
-        dl2_files_path_dict, jobs_from_dl1_dl2 = batch_dl1_to_dl2(
-            dl1_output_dir,
-            model_dir,
-            dl1_to_dl2_config,
+        jobs_from_dl1_dl2 = batch_dl1_to_dl2(
+            lstmcpipe_config['stages']['dl1_to_dl2'],
+            Path(args.config_file_lst).resolve().as_posix(),
             job_from_train_pipe,  # Single jobid from train
-            jobs_all_dl1_finished,  # jobids from merge
-            merged_dl1_paths_dict,  # final dl1 names
-            all_particles,
             batch_config=batch_config,
-            gamma_offsets=gamma_offs,
             logs=logs_files,
         )
 
         update_scancel_file(scancel_file, jobs_from_dl1_dl2)
-
+        all_job_ids.update({"dl1_to_dl2": jobs_from_dl1_dl2})
     else:
         jobs_from_dl1_dl2 = ""
-        dl2_files_path_dict = {}  # Empty log will be manage inside onsite_dl2_irfs
 
     # 5 STAGE --> DL2 to IRFs stage
     if "dl2_to_irfs" in stages_to_run:
         jobs_from_dl2_irf = batch_dl2_to_irfs(
-            dl2_output_dir,
-            all_particles,
-            gamma_offs,
-            Path(args.config_file_lst),
-            jobs_from_dl1_dl2,  # Final dl2 names
-            log_from_dl1_dl2=dl2_files_path_dict,
+            lstmcpipe_config['stages']['dl2_to_irfs'],
+            Path(args.config_file_lst).resolve().as_posix(),
+            jobs_from_dl1_dl2,
             batch_config=batch_config,
-            prod_id=prod_id,
             logs=logs_files,
         )
 
         update_scancel_file(scancel_file, jobs_from_dl2_irf)
-
-    else:
-        jobs_from_dl2_irf = ""
+        all_job_ids.update({"dl2_to_irfs": jobs_from_dl2_irf})
 
     # 6 STAGE --> DL2 to sensitivity curves
     if "dl2_to_sensitivity" in stages_to_run:
         jobs_from_dl2_sensitivity = batch_dl2_to_sensitivity(
-            dl2_output_dir,
-            gamma_offs,
+            lstmcpipe_config['stages']['dl2_to_sensitivity'],
             jobs_from_dl1_dl2,
-            dl2_files_path_dict,  # Final dl2 names
             batch_config=batch_config,
-            prod_id=prod_id,
             logs=logs_files,
         )
 
         update_scancel_file(scancel_file, jobs_from_dl2_sensitivity)
-
-    else:
-        jobs_from_dl2_sensitivity = ""
+        all_job_ids.update({"dl2_to_sensitivity": jobs_from_dl2_sensitivity})
 
     # Check DL2 jobs and the full workflow if it has finished correctly
     jobid_check = batch_mc_production_check(
-        jobs_all_dl1,
-        jobs_all_dl1_finished,
-        job_from_train_pipe,
-        jobs_from_dl1_dl2,
-        jobs_from_dl2_irf,
-        jobs_from_dl2_sensitivity,
-        prod_id,
+        all_job_ids,
         log_directory=logs_dir,
+        prod_id=prod_id,
         prod_config_file=args.config_mc_prod,
-        last_stage=stages_to_run[-1],
         batch_config=batch_config,
         logs_files=logs_files,
     )
diff --git a/lstmcpipe/scripts/lstmcpipe_generate_config.py b/lstmcpipe/scripts/lstmcpipe_generate_config.py
new file mode 100644
index 00000000..b9076451
--- /dev/null
+++ b/lstmcpipe/scripts/lstmcpipe_generate_config.py
@@ -0,0 +1,86 @@
+# #!/usr/bin/env python
+import argparse
+from pathlib import Path
+from datetime import date
+
+from lstmcpipe.config import paths_config
+
+
+
+class ParseKwargs(argparse.Action):
+    """
+    Parse a string formatted as `option1=foo` into a dict `{option1: foo}`
+    """
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, dict())
+        for value in values:
+            key, value = value.split('=')
+            getattr(namespace, self.dest)[key] = value
+
+
+def list_config_classes():
+    """
+    List the classes in `lstmcpipe.config.paths_config` that inherit from `lstmcpipe.config.paths_config.PathConfig`
+
+    Returns
+    -------
+    [object]: list of classes
+    """
+    all_attrs = []
+    for att in list(paths_config.__dict__):
+        try:
+            cls_base = getattr(paths_config, att).__mro__
+            if paths_config.PathConfig in cls_base:
+                all_attrs.append(att)
+        except:
+            pass
+    return all_attrs
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate a lstmcpipe config.")
+
+    # Required arguments
+    parser.add_argument(dest='config_class',
+                        type=str,
+                        help=f'Config class name to use. '
+                             f'List of implemented classes: {list_config_classes()}'
+                        )
+
+    parser.add_argument('--prod_id',
+                        type=str,
+                        help='production ID'
+                        )
+
+    parser.add_argument('--output', '-o',
+                        type=Path,
+                        help='Path to the output file to dump the generated config. '
+                             'Optional, if not provided, the file is dumped locally',
+                        default=None
+                        )
+
+    parser.add_argument('--kwargs',
+                        nargs='*',
+                        action=ParseKwargs,
+                        help="optional kwargs for the requested config class. Use as: --kwargs option1=foo option2=bar"
+                        )
+
+
+    args = parser.parse_args()
+
+    if not hasattr(paths_config, args.config_class):
+        raise NotImplementedError(f"Config class {args.config_class} not implemented in lstmcpipe.config.paths_config")
+
+    output = f'lstmcpipe_config_{date.today()}_{args.config_class}.yaml' if args.output is None else args.output
+    prod_id = 'prod_00' if args.prod_id is None else args.prod_id
+
+    # we get the class from its name and instantiate it with the required args
+    cfg = getattr(paths_config, args.config_class)(prod_id, **args.kwargs)
+    cfg.generate()
+    cfg.save_yml(output)
+
+    print(f"Config saved in {output}")
+
+
+if __name__ == '__main__':
+    main()
diff --git a/lstmcpipe/scripts/script_compare_irfs.py b/lstmcpipe/scripts/script_compare_irfs.py
index 02a44753..b34ad107 100644
--- a/lstmcpipe/scripts/script_compare_irfs.py
+++ b/lstmcpipe/scripts/script_compare_irfs.py
@@ -9,7 +9,6 @@
 from lstmcpipe.plots.plot_irfs import plot_summary_from_file
 
 
-
 def plot_comparison(filelist, outfile=None, cta_north=False):
     """
     Create a 2x2 plot comparing different sensitivity curves
diff --git a/lstmcpipe/scripts/script_dl2_to_sensitivity.py b/lstmcpipe/scripts/script_dl2_to_sensitivity.py
index 33a3656c..c5731e22 100755
--- a/lstmcpipe/scripts/script_dl2_to_sensitivity.py
+++ b/lstmcpipe/scripts/script_dl2_to_sensitivity.py
@@ -49,7 +49,6 @@
 import argparse
 
 
-
 T_OBS = 50 * u.hour
 
 # scaling between on and off region.
diff --git a/lstmcpipe/scripts/script_merge_utils_copy_config.py b/lstmcpipe/scripts/script_merge_utils_copy_config.py
deleted file mode 100644
index fa3362cb..00000000
--- a/lstmcpipe/scripts/script_merge_utils_copy_config.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env python
-
-import os
-import shutil
-import argparse
-
-parser = argparse.ArgumentParser(
-    description="Script to copy all the config files of the source directory into the "
-    "destination directory."
-)
-
-parser.add_argument(
-    "--source",
-    "-s",
-    type=str,
-    dest="source",
-    help="source argument of move_dir_content",
-)
-
-parser.add_argument(
-    "--destination",
-    "-d",
-    type=str,
-    dest="dest",
-    help="destination argument of move_dir_content",
-)
-
-
-def main():
-    args = parser.parse_args()
-
-    # HiPeRTA config is *.txt (<v4.0.0), otherwise is *.yml
-    config_files = [
-        os.path.join(args.source, f)
-        for f in os.listdir(args.source)
-        if f.endswith((".json", ".txt")) or f.startswith("hiperta_conf*.yml")
-    ]
-
-    for file in config_files:
-        shutil.copyfile(file, os.path.join(args.dest, os.path.basename(file)))
-
-
-if __name__ == "__main__":
-    main()
diff --git a/lstmcpipe/scripts/script_merge_utils_move_dir.py b/lstmcpipe/scripts/script_merge_utils_move_dir.py
deleted file mode 100644
index f72b3ae7..00000000
--- a/lstmcpipe/scripts/script_merge_utils_move_dir.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-
-import argparse
-from lstmcpipe.io.data_management import (
-    move_dir_content,
-    check_and_make_dir_without_verification,
-)
-
-parser = argparse.ArgumentParser(
-    description="Script to move a directory and its content after creating the destination"
-    " directory."
-)
-
-parser.add_argument(
-    "--source",
-    "-s",
-    type=str,
-    dest="source",
-    help="source argument of move_dir_content",
-)
-
-parser.add_argument(
-    "--destination",
-    "-d",
-    type=str,
-    dest="dest",
-    help="destination argument of move_dir_content",
-)
-
-
-def main():
-    args = parser.parse_args()
-
-    # check_and_make_dir(args.dest) CANNOT be used because if demands user interaction.
-    check_and_make_dir_without_verification(args.dest)
-    move_dir_content(args.source, args.dest)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/lstmcpipe/stages/__init__.py b/lstmcpipe/stages/__init__.py
index 0fab69e1..1641ecfa 100644
--- a/lstmcpipe/stages/__init__.py
+++ b/lstmcpipe/stages/__init__.py
@@ -1,5 +1,6 @@
 from .mc_process_dl1 import r0_to_dl1, reprocess_dl1, batch_process_dl1
-from .mc_merge_and_copy_dl1 import merge_dl1, batch_merge_and_copy_dl1
+from .mc_train_test_splitting import batch_train_test_splitting, train_test_split
+from .mc_merge_dl1 import merge_dl1, batch_merge_dl1
 from .mc_train import train_pipe, batch_train_pipe, batch_plot_rf_features
 from .mc_dl1_to_dl2 import dl1_to_dl2, batch_dl1_to_dl2
 from .mc_dl2_to_irfs import dl2_to_irfs, batch_dl2_to_irfs
@@ -9,8 +10,10 @@
     "r0_to_dl1",
     "reprocess_dl1",
     "batch_process_dl1",
+    "batch_train_test_splitting",
+    "train_test_split",
     "merge_dl1",
-    "batch_merge_and_copy_dl1",
+    "batch_merge_dl1",
     "train_pipe",
     "batch_train_pipe",
     "batch_plot_rf_features",
diff --git a/lstmcpipe/stages/mc_dl1_to_dl2.py b/lstmcpipe/stages/mc_dl1_to_dl2.py
index e332c71d..0a64bc0e 100755
--- a/lstmcpipe/stages/mc_dl1_to_dl2.py
+++ b/lstmcpipe/stages/mc_dl1_to_dl2.py
@@ -1,66 +1,44 @@
 #!/usr/bin/env python
 
-# T. Vuillaume,
-# Modifications by E. Garcia
-# lstMCpipe DL1 to DL2 onsite stage (at La Palma cluster)
-
 import os
-import time
 import shutil
 import logging
-from lstmcpipe.io.data_management import check_and_make_dir_without_verification
+from pathlib import Path
 from lstmcpipe.workflow_management import save_log_to_file
+from lstmcpipe.io.data_management import check_and_make_dir_without_verification
 
 
 log = logging.getLogger(__name__)
 
 
 def batch_dl1_to_dl2(
-    dl1_directory,
-    path_to_models,
+    dict_paths,
     config_file,
     jobid_from_training,
-    jobids_from_merge,
-    dict_with_dl1_paths,
-    particles_loop,
     batch_config,
     logs,
-    gamma_offsets=None,
 ):
     """
     Function to batch the dl1_to_dl2 stage once the lstchain train_pipe batched jobs have finished.
 
     Parameters
     ----------
-    dl1_directory : str
-        Path to the dl1 directory
-    path_to_models : str
-        Path to the model directory - should be taken from train_pipe
+    dict_paths : dict
+        Core dictionary with {stage: PATHS} information
     config_file : str
         Path to a configuration file. If none is given, a standard configuration is applied
     jobid_from_training : str
         string containing the jobid from the jobs batched in the train_pipe stage, to be passed to the
         dl1_to_dl2 function (as a slurm dependency)
-    jobids_from_merge : str
-        string containing the jobid from the jobs batched in the merge_and_copy_dl1 stage,
-        to be passed to the dl1_to_dl2 function (as a slurm dependency)
-    dict_with_dl1_paths : dict
-        Indeed the log of the merge_and_copy stage, where the final names of the dl1 files were stored
-    particles_loop : list
-        list with the particles to be processed. Takes the global variable ALL_PARTICLES
     batch_config : dict
         Dictionary containing the (full) source_environment and the slurm_account strings to be passed to
         dl1_dl2 function
-    gamma_offsets : list
-        list off gamma offsets
     logs: dict
         Dictionary with logs files
 
     Returns
     -------
-    log_batch_dl1_to_dl2 : dict
-        Dictionary containing the log of the batched dl1_to_dl2 jobs
-    jobid_4_dl2_to_dl3 : str
+    jobid_for_dl2_to_dl3 : str
         string containing the jobids to be passed to the next stage of the workflow (as a slurm dependency)
 
     """
@@ -70,71 +48,42 @@ def batch_dl1_to_dl2(
     debug_log = {}
 
     log.info("==== START {} ==== \n".format("batch dl1_to_dl2_workflow"))
-    time.sleep(1)
-
-    for particle in particles_loop:
-        if particle == "gamma" and gamma_offsets is not None:
-            for off in gamma_offsets:
-                gamma_dl1_directory = os.path.join(dl1_directory, off)
-                _particle = particle + "_" + off
-
-                job_logs, jobid = dl1_to_dl2(
-                    gamma_dl1_directory.format(particle),
-                    path_models=path_to_models,
-                    config_file=config_file,
-                    particle=_particle,
-                    wait_jobid_train_pipe=jobid_from_training,
-                    wait_jobids_merge=jobids_from_merge,
-                    dictionary_with_dl1_paths=dict_with_dl1_paths,
-                    batch_configuration=batch_config,
-                )
-
-                log_dl1_to_dl2.update(job_logs)
-                jobid_for_dl2_to_dl3.append(jobid)
-
-                debug_log[jobid] = (
-                    f"{_particle} job from dl1_to_dl2 that depends both on : {jobid_from_training} "
-                    f"training jobs AND from {jobids_from_merge} merge_and_copy_dl1 jobs"
-                )
-        else:
-            _particle = particle
-            job_logs, jobid = dl1_to_dl2(
-                dl1_directory.format(particle),
-                path_models=path_to_models,
-                config_file=config_file,
-                particle=_particle,
-                wait_jobid_train_pipe=jobid_from_training,
-                wait_jobids_merge=jobids_from_merge,
-                dictionary_with_dl1_paths=dict_with_dl1_paths,
-                batch_configuration=batch_config,
-            )
-
-            log_dl1_to_dl2.update(job_logs)
-            jobid_for_dl2_to_dl3.append(jobid)
-
-            debug_log[jobid] = (
-                f"{_particle} job from dl1_to_dl2 that depends both on : {jobid_from_training} "
-                f"training jobs AND from {jobids_from_merge} merge_and_copy_dl1 jobs"
-            )
-
-    jobid_4_dl2_to_dl3 = ",".join(jobid_for_dl2_to_dl3)
+
+    for paths in dict_paths:
+
+        job_logs, jobid = dl1_to_dl2(
+            paths["input"],
+            paths["output"],
+            path_models=paths["path_model"],
+            config_file=config_file,
+            wait_jobid_train_pipe=jobid_from_training,
+            batch_configuration=batch_config,
+        )
+
+        log_dl1_to_dl2.update(job_logs)
+
+        # Single particle dl1_dl2 jobid to be appended
+        jobid_for_dl2_to_dl3.append(jobid)
+        debug_log[jobid] = (
+            f"dl1_to_dl2 jobid that depends on : {jobid_from_training} training job"
+        )
+
+    jobid_for_dl2_to_dl3 = ",".join(jobid_for_dl2_to_dl3)
 
     save_log_to_file(log_dl1_to_dl2, logs["log_file"], workflow_step="dl1_to_dl2")
     save_log_to_file(debug_log, logs["debug_file"], workflow_step="dl1_to_dl2")
 
     log.info("==== END {} ====".format("batch dl1_to_dl2_workflow"))
 
-    return log_dl1_to_dl2, jobid_4_dl2_to_dl3
+    return jobid_for_dl2_to_dl3
 
 
 def dl1_to_dl2(
-    input_dir,
+    input_file,
+    output_dir,
     path_models,
     config_file,
-    particle,
     wait_jobid_train_pipe=None,
-    wait_jobids_merge=None,
-    dictionary_with_dl1_paths=None,
     batch_configuration='',
 ):
     """
@@ -142,118 +91,66 @@ def dl1_to_dl2(
 
     Parameters
     ----------
-    input_dir : str
-        DL1 base path
+    input_file : str
+        FILE DL1 path
+    output_dir : str
+        DIR Dl2 path
     path_models : str
-        Path to the trained models
+        DIR trained models path
     config_file : str
         Path to a configuration file. If none is given, a standard configuration is applied
-    particle : str
-        Particle to which apply the dl1_to_dl2 stage
     wait_jobid_train_pipe : str
         Comma-separated string with the batched jobid from the train stage to indicate the
         dependencies of the current job to be batched
-    wait_jobids_merge : str
-        string with merge_and_copy jobids
-    dictionary_with_dl1_paths : dict
-        Dictionary with 'particles' as keys containing final output filenames of dl1 files.
     batch_configuration : dict
-        Dictionary containing the (full) source_environment and the slurm_account strings to be passed to the
-        sbatch commands
+        Dictionary containing the (full) source_environment and the slurm_account strings
+        to be passed to the sbatch commands
         ! NOTE : train_pipe AND dl1_to_dl2 MUST BE RUN WITH THE SAME ENVIRONMENT
 
     Returns
     -------
     log_dl1_to_dl2 : dict
-        dictionary of dictionaries containing the jobid of the batched job as key and the run command (the
-        lstchain_mc_dl1_to_dl2 command with all its corresponding arguments) as value.
+        log dictionary containing {jobid: batch_cmd} information
 
     jobid_dl1_to_dl2 : str
-        jobid of the batched job to be send (for dependencies purposes) to the next stage of the
-        workflow (dl2_to_irfs)
+        batched job_id to be passed to later stages
 
     """
     source_environment = batch_configuration["source_environment"]
     slurm_account = batch_configuration["slurm_account"]
 
-    output_dir = input_dir.replace("DL1", "DL2")
-    log.info("Working on DL1 files in {}".format(input_dir))
+    log.info("Working on DL1 files in {}".format(Path(input_file).parent.as_posix()))
 
     check_and_make_dir_without_verification(output_dir)
-    log.info("Output dir {}: {}".format(particle, output_dir))
-
-    log_dl1_to_dl2 = {particle: {}}
-
-    # path to dl1 files by particle type
-    file_list = [
-        dictionary_with_dl1_paths[particle]["testing"]["test_path_and_outname_dl1"]
-    ]
-
-    return_jobids = []
-
-    if wait_jobid_train_pipe == "":
-        wait_jobs = wait_jobids_merge
-    elif wait_jobids_merge == "":
-        wait_jobs = wait_jobid_train_pipe
-    elif wait_jobids_merge == "" and wait_jobid_train_pipe == "":
-        wait_jobs = ""
-    else:
-        wait_jobs = ",".join([wait_jobid_train_pipe, wait_jobids_merge])
-
-    job_name = {
-        "electron": "dl1-2_e",
-        "gamma": "dl1-2_g",
-        "gamma-diffuse": "dl1-2_gd",
-        "proton": "dl1-2_p",
-        "gamma_off0.0deg": "dl1-2_g.0",
-        "gamma_off0.4deg": "dl1-2_g.4",
-    }
-
-    for file in file_list:
-
-        cmd = f"{source_environment} lstchain_dl1_to_dl2 -f {file} -p {path_models}" \
-              f" -o {output_dir}"
-
-        if config_file is not None:
-            cmd += f" -c {config_file}"
-
-        # TODO dry-run option ?
-        # if dry_run:
-        #     print(cmd)
-
-        # Only applying dl1_to_dl2 stage to test files
-        if "testing" in file:
-            ftype = "test"
-        else:
-            ftype = "-"
-
-        jobe = os.path.join(output_dir, f"dl1_dl2_{particle}_{ftype}job.e")
-        jobo = os.path.join(output_dir, f"dl1_dl2_{particle}_{ftype}job.o")
-
-        # sbatch --parsable --dependency=afterok:{wait_ids_proton_and_gammas} --wrap="{cmd}"
-        batch_cmd = "sbatch --parsable -p short --mem=32G"
-        if slurm_account != "":
-            batch_cmd += f" -A {slurm_account}"
-        batch_cmd += (
-            f" --dependency=afterok:{wait_jobs} -J {job_name[particle]} -e {jobe}"
-            f' -o {jobo} --wrap="{cmd}"'
-        )
+    log.info("Output dir: {}".format(output_dir))
 
-        # Batch the job at La Palma
-        jobid_dl1_to_dl2 = os.popen(batch_cmd).read().strip("\n")
+    log_dl1_to_dl2 = {}
 
-        log_dl1_to_dl2[particle][jobid_dl1_to_dl2] = batch_cmd
-        if "testing" in file:
-            log_dl1_to_dl2[particle]["dl2_test_path"] = file.replace(
-                "/DL1/", "/DL2/"
-            ).replace("dl1_", "dl2_", 1)
-        return_jobids.append(jobid_dl1_to_dl2)
+    cmd = f"{source_environment} lstchain_dl1_to_dl2 -f {input_file} -p {path_models}" \
+          f" -o {output_dir}"
+
+    if config_file is not None:
+        cmd += f" -c {Path(config_file).resolve().as_posix()}"
+
+    jobe = Path(output_dir).joinpath("dl1_dl2-%j.e").resolve().as_posix()
+    jobo = Path(output_dir).joinpath("dl1_dl2-%j.o").resolve().as_posix()
+
+    # sbatch --parsable --dependency=afterok:{wait_ids_proton_and_gammas} --wrap="{cmd}"
+    batch_cmd = "sbatch --parsable -p short --mem=32G"
+    if slurm_account != "":
+        batch_cmd += f" -A {slurm_account}"
+    if wait_jobid_train_pipe is not None:
+        batch_cmd += f" --dependency=afterok:{wait_jobid_train_pipe}"
+    batch_cmd += f' -J dl1_2 -e {jobe} -o {jobo} --wrap="{cmd}"'
+
+    # Batch the job at La Palma
+    jobid_dl1_to_dl2 = os.popen(batch_cmd).read().strip("\n")
+    log_dl1_to_dl2.update({jobid_dl1_to_dl2: batch_cmd})
 
     if config_file is not None:
         shutil.copyfile(
-            config_file, os.path.join(output_dir, os.path.basename(config_file))
+            config_file,
+            Path(output_dir).joinpath(Path(config_file).name)
         )
 
-    return_jobids = ",".join(return_jobids)
-
-    return log_dl1_to_dl2, return_jobids
+    return log_dl1_to_dl2, jobid_dl1_to_dl2
diff --git a/lstmcpipe/stages/mc_dl2_to_irfs.py b/lstmcpipe/stages/mc_dl2_to_irfs.py
index 46843349..90ead3b5 100755
--- a/lstmcpipe/stages/mc_dl2_to_irfs.py
+++ b/lstmcpipe/stages/mc_dl2_to_irfs.py
@@ -1,34 +1,24 @@
 #!/usr/bin/env python
 
-# E. Garcia, Avr '21
-#
 # DL2 to IRFs stage for the R0 to IRFs full MC production workflow.
-#
-#
 # As all the previous stages, this script make use of the lstchain entry points to batch each workflow stage.
-#
 
 import os
-import time
-import glob
 import shutil
 import logging
-from lstmcpipe.io.data_management import check_and_make_dir_without_verification
+from pathlib import Path
 from lstmcpipe.workflow_management import save_log_to_file
+from lstmcpipe.io.data_management import check_and_make_dir_without_verification
 
 
 log = logging.getLogger(__name__)
 
 
 def batch_dl2_to_irfs(
-    dl2_directory,
-    loop_particles,
-    offset_gammas,
+    dict_paths,
     config_file,
     job_ids_from_dl1_dl2,
-    log_from_dl1_dl2,
     batch_config,
-    prod_id,
     logs,
 ):
     """
@@ -36,74 +26,47 @@ def batch_dl2_to_irfs(
 
     Parameters
     ----------
-    dl2_directory: str
-        Base path to DL2 directory to be formatted with particle type
+    dict_paths : dict
+        Core dictionary with {stage: PATHS} information
     config_file: str
         Path to lstchain-like config file
-    loop_particles: list
-        list with particles to be processed.
-    offset_gammas: list
-        list off gamma offsets
     job_ids_from_dl1_dl2: str
         Comma-separated string with the job ids from the dl1_to_dl2 stage to be used as a slurm dependency
         to schedule the current stage
     batch_config : dict
         Dictionary containing the (full) source_environment and the slurm_account strings to be passed to
         dl2_to_irfs function
-    log_from_dl1_dl2: dict
-        Dictionary from dl1_to_dl2 stage with particle path information
-    prod_id: str
-        String with prod_id prefix to complete 'file-naming'
     logs: dict
         Dictionary with logs files
 
     Returns
     -------
     jobs_from_dl2_irf: str
+        Comma-separated jobids batched in the current stage
     """
     log.info("==== START {} ====".format("batch mc_dl2_to_irfs"))
-    time.sleep(1)
 
-    debug_log = {}
-    jobid_for_check = []
     log_dl2_to_irfs = {}
+    jobid_for_check = []
+    debug_log = {}
 
-    for off in offset_gammas:
-
-        job_logs, jobid = dl2_to_irfs(
-            dl2_directory,
-            config_file=config_file,
-            log_from_dl1_dl2=log_from_dl1_dl2,
-            irf_point_like=True,
-            irf_gamma_offset=off,
-            batch_configuration=batch_config,
-            wait_jobs_dl1dl2=job_ids_from_dl1_dl2,
-            prod_id=prod_id,
-        )
-
-        jobid_for_check.append(jobid)
-        log_dl2_to_irfs[f"gamma_{off}"] = job_logs
-        debug_log[jobid] = (
-            f"Gamma_{off} job_id from the dl2_to_irfs stage that depends of the dl1_to_dl2 stage "
-            f"job_ids; {job_ids_from_dl1_dl2}"
-        )
-
-    if "gamma-diffuse" in loop_particles:
+    for paths in dict_paths:
 
         job_logs, jobid = dl2_to_irfs(
-            dl2_directory,
-            irf_point_like=False,
+            paths["input"]["gamma_file"],
+            paths["input"]["electron_file"],
+            paths["input"]["proton_file"],
+            paths["output"],
             config_file=config_file,
+            irf_point_like=paths["options"],
             batch_configuration=batch_config,
-            log_from_dl1_dl2=log_from_dl1_dl2,
             wait_jobs_dl1dl2=job_ids_from_dl1_dl2,
-            prod_id=prod_id,
         )
 
+        log_dl2_to_irfs.update(log_dl2_to_irfs)
         jobid_for_check.append(jobid)
-        log_dl2_to_irfs["gamma-diffuse"] = job_logs
         debug_log[jobid] = (
-            f"Gamma-diffuse job_id from the dl2_to_irfs stage that depends of the dl1_to_dl2 stage "
+            f"jobid from dl2_to_irfs stage that depends of the dl1_to_dl2 stage "
             f"job_ids; {job_ids_from_dl1_dl2}"
         )
 
@@ -117,206 +80,84 @@ def batch_dl2_to_irfs(
     return jobid_for_check
 
 
-def check_dl2_files(dl2_dir, pointlike, gamma_off):
-    """
-    Search DL2 testing files for each of the desired particles depending on the IRF configuration (point-like
-    and gamma-offset).
-
-    Parameters
-    ----------
-    dl2_dir: str
-        General path to DL2 directory, not formatted with the particle.
-    pointlike: bool
-        IRF configuration parameter to indicate point-like or diffuse gammas.
-    gamma_off:
-        IRF configuration parameter with the gamma-offset.
-
-    Returns
-    -------
-    dl2_particle_paths: dict
-        Dictionary containing the path to the DL2 testing files depending on the desired IRF configuration
-
-    """
-    # Make particle loop depending on gamma point like
-    particles_irfs = ["electron", "proton"]
-    if pointlike:
-        particles_irfs.append("gamma")
-    else:
-        particles_irfs.append("gamma-diffuse")
-
-    dl2_particle_paths = {}
-    for particle in particles_irfs:
-        dl2_particle_paths[particle] = {}
-
-        if particle == "gamma":
-            particle_dir_dl2 = os.path.join(dl2_dir.format(particle), f"{gamma_off}")
-        else:
-            particle_dir_dl2 = dl2_dir.format(particle)
-
-        if os.path.isdir(particle_dir_dl2):
-            dl2_particle_paths[particle] = glob.glob(
-                os.path.join(particle_dir_dl2, "*testing.h5")
-            )[0]
-        else:
-            log.info(
-                f"DL2 {particle} directory cannot be found or does not exists:\n {particle_dir_dl2}"
-            )
-            exit(-1)
-
-    return dl2_particle_paths
-
-
 def dl2_to_irfs(
-    dl2_directory,
+    gamma_file,
+    electron_file,
+    proton_file,
+    outfile,
     config_file,
-    log_from_dl1_dl2,
-    irf_point_like=True,
-    irf_gamma_offset="off0.0deg",
-    batch_configuration="",
-    wait_jobs_dl1dl2=None,
-    prod_id=None,
+    irf_point_like,
+    batch_configuration,
+    wait_jobs_dl1dl2,
 ):
     """
     Batches interactively the lstchain `lstchain_create_irf_files` entry point.
 
     Parameters
     ----------
-    dl2_directory: str
-        Base path to DL2 directory, (not formatted with the particle type yet).
+    gamma_file: str
+    electron_file: str
+    proton_file: str
+    output_dir: str
     config_file: str
         Path to a configuration file. If none is given, a standard configuration is applied
-    irf_point_like: bool
+    irf_point_like: str
         MC prod configuration argument to create IRFs: {True: gamma, False: gamma-diffuse}.
-    irf_gamma_offset: str
-        MC prod configuration argument to create IRFs: off0.0deg (for ON/OFF obs) or off0.4deg (for wobble obs).
     batch_configuration : dict
         Dictionary containing the (full) source_environment and the slurm_account strings to be passed to the
         sbatch commands
-    log_from_dl1_dl2: dict
-        Dictionary with dl2 output path. Files are not yet here, but path and full name are needed to batch the job.
     wait_jobs_dl1dl2: str
         Comma separated string with the job ids of previous stages (dl1_to_dl2 stage) to be passed as dependencies to
         the create_irfs_files job to be batched.
-    prod_id: str
-        prod_id defined within config_MC_prod.yml file
 
     Returns
     -------
     log_dl2_to_irfs: dict
         Dictionary-wise log containing {'job_id': 'batched_cmd'} items
-    list_job_id_dl2_irfs: str
-        Job-ids of the batched job to be passed to the last (MC prod check) stage of the workflow.
+    job_id_dl2_irfs: str
+        Job-id of the batched job to be passed to the last (MC prod check) stage of the workflow.
     """
     source_env = batch_configuration["source_environment"]
     slurm_account = batch_configuration["slurm_account"]
-
-    allowed_gamma_off = ["off0.0deg", "off0.4deg"]
-    if irf_gamma_offset not in allowed_gamma_off:
-        log.info(
-            f'Please select a valid gamma_offset to compute the IRFS: {" or ".join(allowed_gamma_off)}'
-        )
-        exit(-1)
-
-    if irf_point_like:
-        output_irfs_dir = os.path.join(
-            dl2_directory.replace("/DL2/", "/IRF/").replace("/{}/", "/"),
-            irf_gamma_offset,
-        )
-    else:
-        output_irfs_dir = os.path.join(
-            dl2_directory.replace("/DL2/", "/IRF/").replace("/{}/", "/"), "diffuse"
-        )
+    
+    output_dir = Path(outfile).parent
 
     log_dl2_to_irfs = {}
-    list_job_id_dl2_irfs = []
 
-    if not log_from_dl1_dl2:  # Empty dict and thus no dl2 path files
-        dl2_particle_paths = check_dl2_files(
-            dl2_directory, irf_point_like, irf_gamma_offset
-        )
-
-        # Comprehension list to find gamma or gamma-diffuse
-        gamma_kind = [g for g in dl2_particle_paths.keys() if g.startswith("gamma")][0]
-
-        gamma_file = dl2_particle_paths[gamma_kind]
-        proton_file = dl2_particle_paths["proton"]
-        electron_file = dl2_particle_paths["electron"]
-
-        irf_kind = gamma_kind
-
-    else:
-        proton_file = log_from_dl1_dl2["proton"]["dl2_test_path"]
-        electron_file = log_from_dl1_dl2["electron"]["dl2_test_path"]
-
-        if irf_point_like and irf_gamma_offset == "off0.0deg":
-            gamma_file = log_from_dl1_dl2["gamma_off0.0deg"]["dl2_test_path"]
-            irf_kind = irf_gamma_offset
-        elif irf_point_like and irf_gamma_offset == "off0.4deg":
-            gamma_file = log_from_dl1_dl2["gamma_off0.4deg"]["dl2_test_path"]
-            irf_kind = irf_gamma_offset
-        else:
-            gamma_file = log_from_dl1_dl2["gamma-diffuse"]["dl2_test_path"]
-            irf_kind = "diffuse"
-
-    if irf_point_like:
-        point_like = "--point-like"
-    else:
-        point_like = ""
-
-    # Final outfile name with IRF kind
-    if prod_id is None:
-        output_filename_irf = os.path.join(output_irfs_dir, "irf.fits.gz")
-    else:
-        if irf_point_like:
-            output_filename_irf = os.path.join(
-                output_irfs_dir,
-                "irf_"
-                + prod_id.replace(".", "")
-                + f'_gamma_point-like_{irf_gamma_offset.replace(".", "")}.fits.gz',
-            )
-        else:
-            output_filename_irf = os.path.join(
-                output_irfs_dir,
-                "irf_" + prod_id.replace(".", "") + "_gamma_diffuse.fits.gz",
-            )
+    check_and_make_dir_without_verification(output_dir)
 
     cmd = (
-        f"lstchain_create_irf_files {point_like} -g {gamma_file} -p {proton_file} -e {electron_file}"
-        f" -o {output_filename_irf}"
+        f"lstchain_create_irf_files {irf_point_like} -g {gamma_file} -o {outfile} "
     )
+    if proton_file is not None:
+        cmd += f" -p {proton_file}"
+    if electron_file is not None:
+        cmd += f" -e {electron_file}"
+
     if config_file:
         cmd += f" --config={config_file}"
 
-    # TODO dry-run option ?
-    # if dry_run:
-    #     print(cmd)
-
-    log.info(f"Output dir IRF {irf_kind}: {output_irfs_dir}")
+    log.info(f"Output dir IRF of {gamma_file}: {output_dir}")
 
-    check_and_make_dir_without_verification(output_irfs_dir)
-
-    jobe = os.path.join(output_irfs_dir, f"job_dl2_to_irfs_gamma_{irf_kind}.e")
-    jobo = os.path.join(output_irfs_dir, f"job_dl2_to_irfs_gamma_{irf_kind}.o")
+    jobe = Path(output_dir).joinpath("job_dl2_to_irfs-%j.e").resolve().as_posix()
+    jobo = Path(output_dir).joinpath("job_dl2_to_irfs-%j.o").resolve().as_posix()
 
     batch_cmd = "sbatch --parsable -p short"
     if slurm_account != "":
         batch_cmd += f" -A {slurm_account}"
     batch_cmd += (
-        f" --dependency=afterok:{wait_jobs_dl1dl2} -J IRF_{irf_kind}"
+        f" --dependency=afterok:{wait_jobs_dl1dl2} -J dl2_IRF"
         f' -e {jobe} -o {jobo} --wrap="{source_env} {cmd}"'
     )
 
     job_id_dl2_irfs = os.popen(batch_cmd).read().strip("\n")
-
-    log_dl2_to_irfs[job_id_dl2_irfs] = batch_cmd
-    list_job_id_dl2_irfs.append(job_id_dl2_irfs)
+    log_dl2_to_irfs.update({job_id_dl2_irfs: batch_cmd})
 
     # Copy config into working dir
     if config_file:
         shutil.copyfile(
-            config_file, os.path.join(output_irfs_dir, os.path.basename(config_file))
+            config_file,
+            Path(output_dir).joinpath(Path(config_file).name)
         )
 
-    list_job_id_dl2_irfs = ",".join(list_job_id_dl2_irfs)
-
-    return log_dl2_to_irfs, list_job_id_dl2_irfs
+    return log_dl2_to_irfs, job_id_dl2_irfs
diff --git a/lstmcpipe/stages/mc_dl2_to_sensitivity.py b/lstmcpipe/stages/mc_dl2_to_sensitivity.py
index b519feb7..a629b020 100644
--- a/lstmcpipe/stages/mc_dl2_to_sensitivity.py
+++ b/lstmcpipe/stages/mc_dl2_to_sensitivity.py
@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 
 import os
-import time
 import logging
+from pathlib import Path
 from lstmcpipe.workflow_management import save_log_to_file
 
 
@@ -10,12 +10,9 @@
 
 
 def batch_dl2_to_sensitivity(
-    dl2_directory,
-    offset_gammas,
+    dict_paths,
     job_ids_from_dl1_dl2,
-    log_from_dl1_dl2,
     batch_config,
-    prod_id,
     logs,
 ):
     """
@@ -24,20 +21,14 @@ def batch_dl2_to_sensitivity(
 
     Parameters
     ----------
-    dl2_directory: str
-        Base path to DL2 directory to be formatted with particle type
-    offset_gammas: list
-        list off gamma offsets
+    dict_paths : dict
+        Core dictionary with {stage: PATHS} information
     job_ids_from_dl1_dl2: str
         Comma-separated string with the job ids from the dl1_to_dl2 stage to be used as a slurm dependency
         to schedule the current stage
-    log_from_dl1_dl2: dict
-        Dictionary from dl1_to_dl2 stage with particle path information
     batch_config : dict
         Dictionary containing the (full) source_environment and the slurm_account strings to be passed to
         dl2_to_sensitivity function
-    prod_id: str
-        String with prod_id prefix to complete 'file-naming'
     logs: dict
         Dictionary with logs files
 
@@ -47,30 +38,22 @@ def batch_dl2_to_sensitivity(
         Comma-separated jobids batched in the current stage
     """
     log.info("==== START {} ====".format("batch mc_dl2_to_sensitivity"))
-    time.sleep(1)
 
-    debug_log = {}
-    jobid_for_check = []
     log_dl2_to_sensitivity = {}
+    jobid_for_check = []
+    debug_log = {}
 
-    for off in offset_gammas:
+    for paths in dict_paths:
         job_logs, jobid = dl2_to_sensitivity(
-            dl2_directory,
-            log_from_dl1_dl2,
-            gamma_offset=off,
-            prod_id=prod_id,
+            paths["input"],
+            paths["output"],
             batch_configuration=batch_config,
             wait_jobs_dl1_dl2=job_ids_from_dl1_dl2,
         )
         jobid_for_check.append(jobid)
-        log_dl2_to_sensitivity[f"gamma_{off}"] = job_logs
+        log_dl2_to_sensitivity.update(job_logs)
         debug_log[jobid] = (
-            f"Gamma_{off} job_ids from the dl2_to_sensitivity stage and the plot_irfs script that "
-            f"depends on the dl1_to_dl2 stage job_ids; {job_ids_from_dl1_dl2}"
-        )
-        log.info("Jobs for gamma offset {} have been submitted".format(off))
-        log.debug(
-            f"Gamma_{off} job_ids from the dl2_to_sensitivity stage and the plot_irfs script that "
+            f"Job_ids from the dl2_to_sensitivity stage and the plot_irfs script that "
             f"depends on the dl1_to_dl2 stage job_ids; {job_ids_from_dl1_dl2}"
         )
 
@@ -84,110 +67,11 @@ def batch_dl2_to_sensitivity(
     return jobid_for_check
 
 
-def compose_sensitivity_outdir(dl2_dir, gamma_offset):
-    """
-    Compute the sensitivity output directory depending on the type of gamma file
-
-    Parameters
-    ----------
-    dl2_dir: str
-        Base path to DL2 directory
-    gamma_offset: str
-        String to indicate the gamma offset. Either 'off0.0deg' or 'off0.4deg'
-
-    Returns
-    -------
-    output_sensitivity_dir: str
-        Absolute path in where to store the sensitivity.fits.fz files
-
-    """
-
-    allowed_gamma_off = ["off0.0deg", "off0.4deg"]
-    if gamma_offset not in allowed_gamma_off:
-        log.info(
-            f'Please select a valid gamma_offset to compute the IRFS: {" or ".join(allowed_gamma_off)}'
-        )
-        exit(-1)
-
-    output_sensitivity_dir = os.path.join(
-        dl2_dir.replace("/DL2/", "/IRF/").replace("/{}/", "/"), gamma_offset
-    )
-
-    os.makedirs(output_sensitivity_dir, exist_ok=True)
-
-    return output_sensitivity_dir
-
-
-def sensitivity_io(
-    dl2_directory, log_from_dl1_dl2, gamma_offset="off0.0deg", prod_id=None
-):
-    """
-    Manages the i/o arguments and parameters to be passed to the batch_dl2_to_sensitivity function.
-
-    Parameters
-    ----------
-    dl2_directory: str
-        Base path to DL2 directory
-    log_from_dl1_dl2: dict
-        Dictionary with particle abs path created in previous stages #TODO to be changed by a glob.glob ?
-    gamma_offset: str
-        String to indicate the gamma offset if gamma_point_like == True. Either 'off0.0deg' or 'off0.4deg'
-    prod_id: str
-        String with prod_id prefix to complete 'file-naming'
-
-    Returns
-    -------
-    gamma_file: str
-        Absolute path to DL2 gamma test file
-    proton_file: str
-        Absolute path to DL2 proton test file
-    electron_file: str
-        Absolute path to DL2 electron test file
-    output_directory: str
-        Absolute path with output directory
-    output_sensitivity_filename: str
-        Output filename
-
-    """
-    output_directory = compose_sensitivity_outdir(dl2_directory, gamma_offset)
-
-    # Find paths to DL2 files
-    proton_file = log_from_dl1_dl2["proton"]["dl2_test_path"]
-    electron_file = log_from_dl1_dl2["electron"]["dl2_test_path"]
-
-    if gamma_offset == "off0.0deg":
-        gamma_file = log_from_dl1_dl2["gamma_off0.0deg"]["dl2_test_path"]
-    else:
-        # gamma_offset == 'off0.4deg'. No other case possible, it has been checked in 'compose_sensitivity_outdir'
-        gamma_file = log_from_dl1_dl2["gamma_off0.4deg"]["dl2_test_path"]
-
-    # Create output filenames
-    if prod_id is None:
-        output_sensitivity_filename = os.path.join(
-            output_directory, "sensitivity.fits.gz"
-        )
-    else:
-        output_sensitivity_filename = os.path.join(
-            output_directory,
-            f'{prod_id.replace(".", "")}_gamma_{gamma_offset.replace(".", "")}_sensitivity.fits.gz',
-        )
-
-    return (
-        gamma_file,
-        proton_file,
-        electron_file,
-        output_directory,
-        output_sensitivity_filename,
-    )
-
-
 def dl2_to_sensitivity(
-    dl2_dir,
-    log_from_dl1_dl2,
-    gamma_offset="off0.0deg",
-    prod_id=None,
-    batch_configuration="",
-    wait_jobs_dl1_dl2="",
+    input_paths,
+    output,
+    batch_configuration,
+    wait_jobs_dl1_dl2,
 ):
     """
     Function to run the `script_dl2_to_sensitivity` for the gamma (and the different gamma offsets) and gamma-diffuse
@@ -196,14 +80,8 @@ def dl2_to_sensitivity(
 
     Parameters
     ----------
-    dl2_dir: str
-        Base path to DL2 directory
-    log_from_dl1_dl2: dict
-        Dictionary with particle abs path created in previous stages #TODO to be changed by a glob.glob ?
-    gamma_offset: str
-        String to indicate the gamma offset if gamma_point_like == True. Either 'off0.0deg' or 'off0.4deg'
-    prod_id: str
-        String with prod_id prefix to complete 'filenaming'
+    input_paths: dict
+    output: str
     batch_configuration : dict
         Dictionary containing the (full) source_environment and the slurm_account strings to be passed to the
         sbatch commands
@@ -220,50 +98,49 @@ def dl2_to_sensitivity(
     """
     log_dl2_to_sensitivity = {}
     jobids_dl2_to_sensitivity = []
-    job_name = gamma_offset.replace(".", "").replace("off", "").replace("deg", "")
 
-    g_file, p_file, e_file, out_dir, out_file = sensitivity_io(
-        dl2_dir, log_from_dl1_dl2, gamma_offset, prod_id
-    )
+    g_file = input_paths["gamma_file"]
+    p_file = input_paths["proton_file"]
+    e_file = input_paths["electron_file"]
 
     source_env = batch_configuration["source_environment"]
     slurm_account = batch_configuration["slurm_account"]
 
     # TODO Move the base commands into scripts so that we can use subprocess properly(makes splitting the string easier)
-    # create sensitivity files
-    base_cmd_sens = f"lstmcpipe_dl2_to_sensitivity -g {g_file} -p {p_file} -e {e_file} -o {out_file}"
-    jobo_sens = os.path.join(out_dir, f"job_dl2_to_sensitivity_gamma_{gamma_offset}.o")
-    jobe_sens = os.path.join(out_dir, f"job_dl2_to_sensitivity_gamma_{gamma_offset}.e")
+    base_cmd_sens = f"lstmcpipe_dl2_to_sensitivity -g {g_file} -p {p_file} -e {e_file} -o {output}"
+
+    jobo_sens = Path(output).parent.joinpath("job_dl2_to_sensitivity.o")
+    jobe_sens = Path(output).parent.joinpath("job_dl2_to_sensitivity.e")
 
     cmd_sens = "sbatch --parsable -p short --mem 32G"
     if slurm_account != "":
         cmd_sens += f" -A {slurm_account}"
     cmd_sens += (
         f" --dependency=afterok:{wait_jobs_dl1_dl2} -e {jobe_sens} -o {jobo_sens}"
-        f' -J {job_name}_sensitivity --wrap="{source_env} {base_cmd_sens}"'
+        f' -J dl2_sens --wrap="{source_env} {base_cmd_sens}"'
     )
 
     job_id_dl2_sens = os.popen(cmd_sens).read().strip("\n")
-    log_dl2_to_sensitivity[job_id_dl2_sens] = cmd_sens
+    log_dl2_to_sensitivity.update({job_id_dl2_sens: cmd_sens})
     jobids_dl2_to_sensitivity.append(job_id_dl2_sens)
 
     # Create plot from sensitivity files
     base_cmd_plot = (
-        f'lstmcpipe_plot_irfs -f {out_file} -o {out_file.replace(".fits.gz", ".png")}'
+        f'lstmcpipe_plot_irfs -f {output} -o {output.replace(".fits.gz", ".png")}'
     )
-    jobe_plot = os.path.join(out_dir, f"job_plot_sensitivity_gamma_{gamma_offset}.e")
-    jobo_plot = os.path.join(out_dir, f"job_plot_sensitivity_gamma_{gamma_offset}.o")
+    jobe_plot = Path(output).parent.joinpath("job_plot_sensitivity-%j.e")
+    jobo_plot = Path(output).parent.joinpath("job_plot_sensitivity-%j.o")
 
     cmd_plot = "sbatch --parsable -p short"
     if slurm_account != "":
         cmd_plot += f" -A {slurm_account}"
     cmd_plot += (
         f" --dependency=afterok:{job_id_dl2_sens} -e {jobe_plot} -o {jobo_plot}"
-        f' -J {job_name}_sens_plot --wrap="export MPLBACKEND=Agg; {source_env} {base_cmd_plot}"'
+        f' -J dl2_sens_plot --wrap="export MPLBACKEND=Agg; {source_env} {base_cmd_plot}"'
     )
 
     job_id_plot_sens = os.popen(cmd_plot).read().strip("\n")
-    log_dl2_to_sensitivity[job_id_plot_sens] = cmd_plot
+    log_dl2_to_sensitivity.update({job_id_plot_sens: cmd_plot})
     jobids_dl2_to_sensitivity.append(job_id_plot_sens)
 
     return log_dl2_to_sensitivity, ",".join(jobids_dl2_to_sensitivity)
diff --git a/lstmcpipe/stages/mc_merge_and_copy_dl1.py b/lstmcpipe/stages/mc_merge_and_copy_dl1.py
deleted file mode 100755
index 132c5a38..00000000
--- a/lstmcpipe/stages/mc_merge_and_copy_dl1.py
+++ /dev/null
@@ -1,458 +0,0 @@
-#!/usr//bin/env python3
-
-# T. Vuillaume, 12/09/2019
-# merge and copy DL1 data after production
-# Modifications by E. Garcia, 21/01/2020
-
-
-# 1. check job_logs
-# 2. check that all files have been created in DL1 based on training and testing lists
-# 3. move DL1 files in final place
-# 4. merge DL1 files
-# 5. move running_dir
-
-import os
-import time
-import logging
-from lstmcpipe.io.data_management import check_job_logs
-from lstmcpipe.workflow_management import save_log_to_file
-
-
-log = logging.getLogger(__name__)
-
-
-def batch_merge_and_copy_dl1(
-    running_analysis_dir,
-    log_jobs_from_r0_to_dl1,
-    particles_loop,
-    batch_config,
-    logs,
-    smart_merge=False,
-    no_image_flag=True,
-    prod_id=None,
-    gamma_offsets=None,
-    workflow_kind="lstchain",
-):
-    """
-    Function to batch the onsite_mc_merge_and_copy function once the all the r0_to_dl1 jobs (batched by particle type)
-    have finished.
-
-    Batch 8 merge_and_copy_dl1 jobs ([train, test] x particle) + the move_dl1 and move_dir jobs (2 per particle).
-
-    Parameters
-    ----------
-    running_analysis_dir : str
-        Directory to dl1 files
-    log_jobs_from_r0_to_dl1 : dict
-        dictionary of dictionaries containing the log (jobids organized by particle) from the previous stage
-        (onsite_mc_r0_to_dl1)
-    smart_merge : bool
-        flag to indicate whether the merge of the files should be done with `--smart True` or `--smart False`,
-        controlling the argument of the `lstchain_merge_hdf5_files.py` script (batched in this function).
-    particles_loop : list
-        list with the particles to be processed. Takes the global variable ALL_PARTICLES
-    no_image_flag : bool
-        flag to indicate whether the --no-image argument of the `lstchain_merge_hdf5_files.py` script (batched in
-        this function) should be either True or False.
-    gamma_offsets : list
-        list containing the offset of the gammas
-    prod_id : str
-        prod_id defined in config_MC_prod.yml
-    batch_config : dict
-        Dictionary containing the (full) source_environment and the slurm_account strings to be passed
-        to `merge_dl1` and `compose_batch_command_of_script` functions.
-    workflow_kind : str
-        Defines workflow kind (lstchain, ctapipe, hiperta)
-    logs: dict
-        Dictionary with logs files
-
-    Returns
-    -------
-    log_merge_and_copy : dict
-        Dictionary containing the log of the batched merge_and_copy_dl1 jobs
-    jobid_4_train : str
-         string containing the jobids to be passed to the next stage of the workflow (as a slurm dependency)
-    all_merge : str
-        string containing all the jobs to indicate that all the dl1 are correctly finished (for dl1_to_dl2)
-
-    """
-    log_merge_and_copy = {}
-    jobid_4_train = []
-    all_jobs_from_merge_stage = []
-    debug_log = {}
-
-    log.info("==== START {} ====".format("batch merge_and_copy_dl1_workflow"))
-    time.sleep(1)
-    if isinstance(smart_merge, str):
-        merge_flag = "lst" in smart_merge
-    else:
-        merge_flag = smart_merge
-    log.debug("Merge flag set: {}".format(merge_flag))
-
-    for particle in particles_loop:
-        if particle == "gamma" and gamma_offsets is not None:
-            for off in gamma_offsets:
-
-                gamma_running_analysis_dir = os.path.join(running_analysis_dir, off)
-                _particle = particle + "_" + off
-
-                job_logs, jobid_mv_all_dl1, jobid_debug = merge_dl1(
-                    gamma_running_analysis_dir.format(particle),
-                    particle2jobs_dict=log_jobs_from_r0_to_dl1,
-                    particle=_particle,
-                    flag_merge=merge_flag,
-                    flag_no_image=no_image_flag,
-                    prod_id=prod_id,
-                    gamma_offset=off,
-                    batch_configuration=batch_config,
-                    workflow_kind=workflow_kind,
-                )
-
-                log_merge_and_copy.update(job_logs)
-                all_jobs_from_merge_stage.append(jobid_debug)
-                if _particle == "gamma-diffuse" or _particle == "proton":
-                    jobid_4_train.append(jobid_mv_all_dl1)
-
-                debug_log[jobid_mv_all_dl1] = (
-                    f"{_particle} merge_and_copy-job - INDEED IT IS JUST PASSED the "
-                    f"move_dl1 jobid - that will be send to the train pipe stage. They "
-                    f"depend on the following {log_jobs_from_r0_to_dl1[_particle]} "
-                    f"r0_t0_dl1 jobs."
-                )
-                debug_log[
-                    jobid_debug
-                ] = f"Are all the {_particle} jobs that have been launched in merge_and_copy_dl1."
-
-        else:
-            _particle = particle
-
-            job_logs, jobid_mv_all_dl1, jobid_debug = merge_dl1(
-                running_analysis_dir.format(particle),
-                particle2jobs_dict=log_jobs_from_r0_to_dl1,
-                particle=_particle,
-                flag_merge=merge_flag,
-                flag_no_image=no_image_flag,
-                prod_id=prod_id,
-                batch_configuration=batch_config,
-                workflow_kind=workflow_kind,
-            )
-
-            log_merge_and_copy.update(job_logs)
-            all_jobs_from_merge_stage.append(jobid_debug)
-            if _particle == "gamma-diffuse" or _particle == "proton":
-                jobid_4_train.append(jobid_mv_all_dl1)
-
-            debug_log[jobid_mv_all_dl1] = (
-                f"{_particle} merge_and_copy-job - INDEED IT IS JUST PASSED the "
-                f"move_dl1 jobid - that will be send to the train pipe stage. They depend "
-                f"on the following {log_jobs_from_r0_to_dl1[_particle]} r0_t0_dl1 jobs."
-            )
-            debug_log[
-                jobid_debug
-            ] = f"Are all the {_particle} jobs that have been launched in merge_and_copy_dl1."
-
-    jobid_4_train = ",".join(jobid_4_train)
-    all_jobs_from_merge_stage = ",".join(all_jobs_from_merge_stage)
-
-    save_log_to_file(log_merge_and_copy, logs["log_file"], "merge_and_copy_dl1")
-    save_log_to_file(debug_log, logs["debug_file"], workflow_step="merge_and_copy_dl1")
-
-    log.info("==== END {} ====".format("batch merge_and_copy_dl1_workflow"))
-
-    return log_merge_and_copy, jobid_4_train, all_jobs_from_merge_stage
-
-
-def compose_batch_command_of_script(
-    source, destination, script, particle, wait_jobs, suffix, slurm_account
-):
-    """
-    Creates the slurm command of the 'cmd' script
-
-    Parameters
-    ----------
-    source: str
-        Source directory
-    destination: str
-        Destination directory
-    script: str
-        Script to be used in the slurm command. Either 'lstmcpipe_utils_move_dir' or 'lstmcpipe_utils_cp_config'
-    particle: str
-        Particle type for slurm job-naming
-    suffix: str
-        Suffix to indicate the kind of job
-    wait_jobs: str
-        Job-id to be used as dependency in the batched slurm command
-    slurm_account: str
-        Slurm account argument (-A; --account)
-
-    Returns
-    -------
-    batch_cmd: str
-        Full slurm batch command ready to batched the script argument
-
-    """
-    cmd = f"{script} -s {source} -d {destination}"
-
-    jobe = f"slurm-{particle}_{suffix}.e"
-    jobo = f"slurm-{particle}_{suffix}.o"
-
-    batch_cmd = "sbatch --parsable -p short"
-    if slurm_account != "":
-        batch_cmd += f" -A {slurm_account}"
-    batch_cmd += (
-        f" -J {particle}_{suffix} -e {jobe} -o {jobo}"
-        f' --dependency=afterok:{wait_jobs} --wrap="{cmd}"'
-    )
-
-    return batch_cmd
-
-
-def merge_dl1(
-    input_dir,
-    particle2jobs_dict,
-    particle=None,
-    flag_merge=False,
-    flag_no_image=True,
-    prod_id=None,
-    gamma_offset=None,
-    batch_configuration=None,
-    workflow_kind="lstchain",
-):
-    """
-    Merge and copy DL1 data after production.
-
-        1. check job_logs
-        2. check that all files have been created in DL1 based on training and testing lists
-        3. move DL1 files in final place
-        4. merge DL1 files
-        5. move running_dir
-
-
-    Parameters
-    ----------
-    input_dir : str
-        path to the DL1 files directory to merge, copy and move.  Compulsory argument.
-    particle2jobs_dict : dict
-        Dictionary used to retrieve the r0 to dl1 jobids that were sent in the previous step of the r0-dl3 workflow.
-        This script will NOT start until all the jobs sent before have finished.
-    particle : str
-        Type of particle used to create the log and dictionary
-    flag_merge : bool
-        Flag to indicate whether the `--smart` argument of the `lstchain_merge_hdf5_files.py` script must be set to
-        True (smart merge) or False (auto merge).
-        Default set to True.
-    flag_no_image : bool
-        Flaf to indicate whether the `--no-image` argument of the `lstchain_merge_hdf5_files.py` script must be set to
-        True (--no-image True) or False (--no-image False).
-        Default set to True.
-    prod_id : str
-        prod_id for output filename.
-    gamma_offset : str
-        if gamma files have various off0.Xdeg observations, include the offset within the filename for completeness.
-    batch_configuration : dict
-        Dictionary containing the (full) source_environment and the slurm_account strings.
-        ! NOTE : train_pipe AND dl1_to_dl2 **MUST** be run with the same environment.
-    workflow_kind: str
-        One of the supported pipelines. Defines the command to be run on r0 files
-
-
-    Returns
-    -------
-
-    log_merge : dict
-        dictionary of dictionaries containing the log information of this script and the jobid of the batched job,
-        separated by particle
-
-         - log_merge[particle][set_type].keys() = ['logs_script_test or logs_script_train',
-                                        'train_path_and_outname_dl1 or test_path_and_outname_dl1', 'jobid']
-    return_jobids4train : str
-        jobid of the batched job to be send (for dependencies purposes) to the next stage of the workflow
-        (train_pipe), by particle
-    return_jobids_debug ; str
-        jobids to store in log_reduced.txt - Mainly debug purposes.
-
-    """
-
-    source_environment = batch_configuration["source_environment"]
-    slurm_account = batch_configuration["slurm_account"]
-
-    log_merge = {particle: {"training": {}, "testing": {}}}
-
-    wait_r0_dl1_jobs = particle2jobs_dict[particle]
-
-    return_jobids4train = []
-    return_jobids_debug = []
-
-    job_name = {
-        "electron": "e_merge",
-        "gamma": "g_merge",
-        "gamma-diffuse": "gd_merge",
-        "proton": "p_merge",
-        "gamma_off0.0deg": "g0.0_merge",
-        "gamma_off0.4deg": "g0.4_merge",
-    }
-
-    JOB_LOGS = os.path.join(input_dir, "job_logs")
-    training_filelist = os.path.join(input_dir, "training.list")
-    running_DL1_dir = os.path.join(input_dir, "DL1")
-    final_DL1_dir = input_dir.replace("running_analysis", "DL1")
-    logs_destination_dir = input_dir.replace("running_analysis", "analysis_logs")
-
-    # 1. check job logs
-    check_job_logs(JOB_LOGS)
-
-    # 2. check that all files have been created in DL1 based on training and testing lists
-    # just check number of files first:
-    # TODO Create check before merging files ?
-
-    log.info("merging starts - {}".format(particle))
-
-    # 3. merge DL1 files
-    wait_both_merges = []
-
-    for set_type in ["testing", "training"]:
-        tdir = os.path.join(running_DL1_dir, set_type)
-
-        # just need to take the base name of the file, so we read a processed bunch and take first file
-        with open(training_filelist, "r") as f:
-            output_filename = f.readline()
-
-        output_filename = os.path.basename(output_filename.split("_run")[0])
-        if not output_filename.startswith("dl1_"):
-            output_filename = "dl1_" + output_filename
-        if particle == "gamma-diffuse":
-            output_filename = output_filename.replace("gamma", "gamma-diffuse")
-        if "_off" in particle:
-            output_filename += f"_{gamma_offset}"
-        output_filename += f"_{prod_id}_{set_type}"
-        output_filename += ".h5"
-
-        output_filename = os.path.join(running_DL1_dir, output_filename)
-        log.info("merge output: {}".format(output_filename))
-
-        # After the workflow the files will be moved, will not stay at output_filename
-        if set_type == "training":
-            log_merge[particle][set_type]["train_path_and_outname_dl1"] = os.path.join(
-                final_DL1_dir, os.path.basename(output_filename)
-            )
-        else:
-            log_merge[particle][set_type]["test_path_and_outname_dl1"] = os.path.join(
-                final_DL1_dir, os.path.basename(output_filename)
-            )
-
-        cmd = "sbatch --parsable -p short"
-        if slurm_account != "":
-            cmd += f" -A {slurm_account}"
-        if wait_r0_dl1_jobs != "":
-            cmd += " --dependency=afterok:" + wait_r0_dl1_jobs
-
-        cmd += (
-            f" -J {job_name[particle]} -e slurm-{job_name[particle]}-{set_type}.o "
-            f'-o slurm-{job_name[particle]}-{set_type}.e --wrap="{source_environment} '
-        )
-
-        # Close " of wrap
-        if workflow_kind == "lstchain":
-            cmd += f'lstchain_merge_hdf5_files -d {tdir} -o {output_filename} --no-image"'
-        elif workflow_kind == "hiperta":
-            cmd += (
-                f"lstchain_merge_hdf5_files -d {tdir} -o {output_filename} --no-image {flag_no_image} "
-                f'--smart {flag_merge}"'
-            )  # HiPeRTA workflow still uses --smart flag
-        else:  # ctapipe case
-            if flag_no_image:
-                cmd += f'ctapipe-merge --input-dir {tdir} --output {output_filename} --skip-images --skip-simu-images"'
-            else:
-                cmd += f'ctapipe-merge --input-dir {tdir} --output {output_filename}"'
-
-        jobid_merge = os.popen(cmd).read().strip("\n")
-        log_merge[particle][set_type][jobid_merge] = cmd
-
-        log.info(f"Submitted batch job {jobid_merge} -- {particle}, {set_type}")
-
-        wait_both_merges.append(jobid_merge)
-        return_jobids_debug.append(jobid_merge)
-
-    # Out of testing/training loop !
-
-    # 4., 5. & 6. in the case of the full workflow are done in a separate sbatch to wait merge, the three steps:
-    # 4 --> move DL1 files in final place
-    # 5 --> copy lstchain config file in final_dir too
-    # 6 --> move running_dir as logs
-
-    log.info("DL1 files will be moved to {}".format(final_DL1_dir))
-
-    # 4 --> move DL1 files in final place
-    wait_both_merges = ",".join(wait_both_merges)
-    cmd_mv_dl1 = compose_batch_command_of_script(
-        running_DL1_dir,
-        final_DL1_dir,
-        script="lstmcpipe_utils_move_dir",
-        particle=job_name[particle].split("_")[0],
-        suffix="mv_dl1_files",
-        wait_jobs=wait_both_merges,
-        slurm_account=slurm_account
-    )
-
-    jobid_move_dl1 = os.popen(cmd_mv_dl1).read().strip("\n")
-    log_merge[particle][set_type][jobid_move_dl1] = cmd_mv_dl1
-
-    log.info(
-        f"Submitted batch job {jobid_move_dl1}. It will move dl1 files when {wait_both_merges} finish."
-    )
-
-    # 5 --> copy lstchain config file in final_dir too
-    cmd_cp_conf = compose_batch_command_of_script(
-        input_dir,
-        final_DL1_dir,
-        script="lstmcpipe_utils_cp_config",
-        particle=job_name[particle].split("_")[0],
-        suffix="cp_config",
-        wait_jobs=jobid_move_dl1,
-        slurm_account=slurm_account
-    )
-
-    jobid_copy_conf = os.popen(cmd_cp_conf).read().strip("\n")
-    log_merge[particle][set_type][jobid_copy_conf] = cmd_cp_conf
-
-    log.info(
-        f"Submitted batch job {jobid_copy_conf}. It will copy the used config when {jobid_move_dl1} finish."
-    )
-
-    # 6 --> move running_dir to final analysis_logs
-    cmd_mv_dir = compose_batch_command_of_script(
-        input_dir,
-        logs_destination_dir,
-        script="lstmcpipe_utils_move_dir",
-        particle=job_name[particle].split("_")[0],
-        suffix="mv_dl1_dir",
-        wait_jobs=jobid_copy_conf,
-        slurm_account=slurm_account
-    )
-
-    jobid_move_log = os.popen(cmd_mv_dir).read().strip("\n")
-    log_merge[particle][set_type][jobid_move_log] = cmd_mv_dir
-
-    log.info(
-        f"Submitted batch job {jobid_move_log}. It will move running_dir when {jobid_copy_conf} finish."
-    )
-
-    return_jobids4train.append(jobid_move_dl1)
-
-    return_jobids_debug.append(jobid_move_dl1)
-    return_jobids_debug.append(jobid_move_log)
-    return_jobids_debug.append(jobid_copy_conf)
-
-    log.info("LOGS will be moved to {}".format(logs_destination_dir))
-
-    # Little clarification (it will not be clear in log). These keys are stored here for 2 purposes:
-    # 1 - train_pipe recover final dl1 names and path.
-    # 2 - dl1_to_dl2 recover the jobids of the merged dl1 files; (all dl1 files MUST be merged and moved
-    # to dl1_dir), so instead of storing the jobid that merges all the *particle*_dl1 (jobid_merge), it will
-    # be store the jobid that move the dl1 final file to dl1_dir. Until this step is not finished, the workflow
-    # cannot continue.
-
-    return_jobids4train = ",".join(return_jobids4train)
-    return_jobids_debug = ",".join(return_jobids_debug)
-
-    return log_merge, return_jobids4train, return_jobids_debug
diff --git a/lstmcpipe/stages/mc_merge_dl1.py b/lstmcpipe/stages/mc_merge_dl1.py
new file mode 100755
index 00000000..ed5efd8f
--- /dev/null
+++ b/lstmcpipe/stages/mc_merge_dl1.py
@@ -0,0 +1,146 @@
+#!/usr//bin/env python3
+
+import os
+import logging
+from pathlib import Path
+from lstmcpipe.workflow_management import save_log_to_file
+
+
+log = logging.getLogger(__name__)
+
+
+def batch_merge_dl1(
+    dict_paths,
+    batch_config,
+    logs,
+    jobid_from_splitting,
+    workflow_kind="lstchain",
+):
+    """
+    Function to batch the onsite_mc_merge_and_copy function once the all the r0_to_dl1 jobs (batched by particle type)
+    have finished.
+
+    Batch 8 merge_and_copy_dl1 jobs ([train, test] x particle) + the move_dl1 and move_dir jobs (2 per particle).
+
+    Parameters
+    ----------
+    dict_paths : dict
+        Core dictionary with {stage: PATHS} information
+    batch_config : dict
+        Dictionary containing the (full) source_environment and the slurm_account strings to be passed
+        to `merge_dl1` and `compose_batch_command_of_script` functions.
+    workflow_kind : str
+        Defines workflow kind (lstchain, ctapipe, hiperta)
+    logs: dict
+        Dictionary with logs files
+    jobid_from_splitting:
+
+    Returns
+    -------
+    jobids_for_train : str
+         Comma-separated str with all the job-ids to be passed to the next
+         stage of the workflow (as a slurm dependency)
+
+    """
+    log_merge = {}
+    all_jobs_merge_stage = []
+    debug_log = {}
+
+    log.info("==== START {} ====".format("batch merge_and_copy_dl1_workflow"))
+    # TODO Lukas: merging option will come inside the
+    #  dict_paths["merge_dl1"]["merging_options"]
+#    if isinstance(smart_merge, str):
+#        merge_flag = "lst" in smart_merge
+#    else:
+#        merge_flag = smart_merge
+#    log.debug("Merge flag set: {}".format(merge_flag))
+
+    for particle in dict_paths:
+        job_logs, jobid_debug = merge_dl1(
+            particle["input"],
+            particle["output"],
+            merging_options=particle.get('options', None),
+            batch_configuration=batch_config,
+            wait_jobs_split=jobid_from_splitting,
+            workflow_kind=workflow_kind,
+        )
+
+        log_merge.update(job_logs)
+        all_jobs_merge_stage.append(jobid_debug)
+
+    jobids_for_train = ','.join(all_jobs_merge_stage)
+
+    save_log_to_file(log_merge, logs["log_file"], "merge_dl1")
+    save_log_to_file(debug_log, logs["debug_file"], workflow_step="merge_dl1")
+
+    log.info("==== END {} ====".format("batch merge_and_copy_dl1_workflow"))
+
+    return jobids_for_train
+
+
+def merge_dl1(
+        input_dir,
+        output_file,
+        batch_configuration,
+        wait_jobs_split="",
+        merging_options=None,
+        workflow_kind="lstchain",
+):
+    """
+
+    Parameters
+    ----------
+    input_dir: str
+    output_file: str
+    batch_configuration: dict
+    wait_jobs_split: str
+    merging_options: dict
+    workflow_kind: str
+
+    Returns
+    -------
+    log_merge: dict
+    jobid_merge: dict
+
+    """
+    source_environment = batch_configuration["source_environment"]
+    slurm_account = batch_configuration["slurm_account"]
+
+    merging_options = "" if merging_options is None else merging_options
+
+    log_merge = {}
+
+    cmd = "sbatch --parsable -p short"
+    if slurm_account != "":
+        cmd += f" -A {slurm_account}"
+    if wait_jobs_split != "":
+        cmd += " --dependency=afterok:" + wait_jobs_split
+
+    jobo = Path(output_file).parent.joinpath("merging-output.o")
+    jobe = Path(output_file).parent.joinpath("merging-error.e")
+
+    cmd += (
+        f' -J merge -e {jobe} -o {jobo} --wrap="{source_environment} '
+    )
+
+    # command passed changes depending on the workflow_kind
+    if workflow_kind == "lstchain":
+        cmd += f'lstchain_merge_hdf5_files -d {input_dir} -o {output_file}  {merging_options}'
+
+    elif workflow_kind == "hiperta":
+        # HiPeRTA workflow still uses --smart flag (lstchain v0.6.3)
+        cmd += (
+            f'lstchain_merge_hdf5_files -d {input_dir} -o {output_file}  {merging_options}'
+        )
+    else:  # ctapipe case
+        cmd += f'ctapipe-merge --input-dir {input_dir} --output {output_file}  {merging_options}'
+
+    # IN ALL THE CASES we need to close the " of the wrap
+    cmd += '"'
+
+    jobid_merge = os.popen(cmd).read().strip("\n")
+    log_merge.update({jobid_merge: cmd})
+
+    log.info(f"Submitted batch job {jobid_merge}")
+
+    return log_merge, jobid_merge
diff --git a/lstmcpipe/stages/mc_process_dl1.py b/lstmcpipe/stages/mc_process_dl1.py
index 215ba265..4991a0b8 100755
--- a/lstmcpipe/stages/mc_process_dl1.py
+++ b/lstmcpipe/stages/mc_process_dl1.py
@@ -1,21 +1,14 @@
 #!/usr/bin/env python
 
-# T. Vuillaume,
-# Modifications by E. Garcia
 # Code to reduce R0 data to DL1 onsite (La Palma cluster)
 
-
 import os
-import time
 import shutil
 import logging
 from pathlib import Path
-from numpy.random import default_rng
 from lstmcpipe.workflow_management import save_log_to_file
 from lstmcpipe.io.data_management import (
     check_data_path,
-    get_input_filelist,
-    check_and_make_dir_without_verification,
 )
 
 
@@ -23,13 +16,10 @@
 
 
 def batch_process_dl1(
-    input_dir,
+    dict_paths,
     conf_file,
-    prod_id,
-    particles_loop,
     batch_config,
     logs,
-    gamma_offsets=None,
     workflow_kind="lstchain",
     new_production=True,
 ):
@@ -38,18 +28,13 @@ def batch_process_dl1(
 
     Parameters
     ----------
-    input_dir : str
-        Path to the r0/DL0 files
+    dict_paths : dict
+        Core dictionary with {stage: PATHS} information
     conf_file : str
         Path to a configuration file. If none is given, a standard configuration is applied
-    prod_id : str
-        Production ID. If None, _v00 will be used, indicating an official base production. Default = None.
-    particles_loop : list
-        list with the particles to be processed. Takes the global variable ALL_PARTICLES
     batch_config : dict
         Dict with source environment (to select the desired conda environment to run the r0/1_to_dl1 stage),
         and the slurm user account.
-    gamma_offsets : list
     workflow_kind: str
         One of the supported pipelines. Defines the command to be run on r0 files
     new_production: bool
@@ -59,108 +44,64 @@ def batch_process_dl1(
 
     Returns
     -------
-    particle2jobid_dict : dict
-        Dictionary of dictionaries containing the full log of the batched jobs (jobids as keys) as well as the
-        4 more keys (one by particle) with all the jobs associated with each particle.
-    all_jobids_from_r0_dl1_stage : str
+    jobids_dl1_processing_stage : str
         string, separated by commas, containing all the jobids of this stage
     """
-    particle2jobid_dict = {"full_logs_all_jobs": {}}
+    log_process_dl1 = {}
     debug_log = {}
-    all_jobids_from_dl1_processing_stage = []
+    jobids_dl1_processing_stage = []
 
     log.info("==== START {} dl1 processing ====".format(workflow_kind))
-    time.sleep(1)
-
-    for particle in particles_loop:
-        if particle == "gamma" and gamma_offsets is not None:
-            for off in gamma_offsets:
-                particle_input_dir = Path(input_dir.format(particle), off).as_posix()
-                _particle = particle + "_" + off
-                if new_production:
-                    job_logs, jobids_by_particle = r0_to_dl1(
-                        particle_input_dir,  # Particle needs to be gamma w/o the off
-                        config_file=conf_file,
-                        particle=_particle,
-                        prod_id=prod_id,
-                        batch_config=batch_config,
-                        offset=off,
-                        workflow_kind=workflow_kind,
-                    )
-                else:
-                    job_logs, jobids_by_particle = reprocess_dl1(
-                        particle_input_dir,  # Particle needs to be gamma w/o the off
-                        config_file=conf_file,
-                        particle=_particle,
-                        prod_id=prod_id,
-                        batch_config=batch_config,
-                        offset=off,
-                        workflow_kind=workflow_kind,
-                    )
-                particle2jobid_dict["full_logs_all_jobs"].update(job_logs)
-                particle2jobid_dict[_particle] = ",".join(jobids_by_particle)
-                all_jobids_from_dl1_processing_stage.append(
-                    particle2jobid_dict[_particle]
-                )  # Create a list with particles elements
-
-                for jid in jobids_by_particle:
-                    debug_log[jid] = f"{_particle} job from process_dl1"
-        else:
-            particle_input_dir = input_dir.format(particle)
-            _particle = particle
-            if new_production:
-                job_logs, jobids_by_particle = r0_to_dl1(
-                    particle_input_dir,  # Particle needs to be gamma w/o the off
-                    config_file=conf_file,
-                    particle=_particle,
-                    prod_id=prod_id,
-                    batch_config=batch_config,
-                    workflow_kind=workflow_kind,
-                )
-            else:
-                job_logs, jobids_by_particle = reprocess_dl1(
-                    particle_input_dir,  # Particle needs to be gamma w/o the off
-                    config_file=conf_file,
-                    particle=_particle,
-                    prod_id=prod_id,
-                    batch_config=batch_config,
-                    workflow_kind=workflow_kind,
-                )
-            particle2jobid_dict["full_logs_all_jobs"].update(job_logs)
-            particle2jobid_dict[_particle] = ",".join(jobids_by_particle)
-            all_jobids_from_dl1_processing_stage.append(
-                particle2jobid_dict[_particle]
-            )  # Create a list with particles elements
-
-            for jid in jobids_by_particle:
-                debug_log[jid] = f"{_particle} job from r0_to_dl1"
-
-    all_jobids_from_dl1_processing_stage = ",".join(
-        all_jobids_from_dl1_processing_stage
-    )  # Create a string to be directly passed
-
-    save_log_to_file(particle2jobid_dict, logs["log_file"], "r0_to_dl1")
+
+    if new_production:
+
+        for paths in dict_paths["r0_to_dl1"]:
+            job_logs, jobid = r0_to_dl1(
+                paths["input"],
+                paths["output"],
+                config_file=conf_file,
+                batch_config=batch_config,
+                workflow_kind=workflow_kind,
+            )
+            log_process_dl1.update(job_logs)
+            jobids_dl1_processing_stage.append(jobid)
+            debug_log[jobid] = f'r0_dl1 job from input dir: {paths["input"]}'
+
+    else:
+
+        for paths in dict_paths["dl1ab"]:
+
+            job_logs, jobid = reprocess_dl1(
+                paths["input"],
+                paths["output"],
+                config_file=conf_file,
+                batch_config=batch_config,
+                workflow_kind=workflow_kind,
+            )
+
+            log_process_dl1.update(job_logs)
+            jobids_dl1_processing_stage.append(jobid)
+            debug_log[jobid] = f'dl1ab job from input dir: {paths["input"]}'
+
+    # Create a string to be directly passed
+    jobids_dl1_processing_stage = ",".join(jobids_dl1_processing_stage)
+
+    save_log_to_file(log_process_dl1, logs["log_file"], "r0_to_dl1")
     save_log_to_file(debug_log, logs["debug_file"], workflow_step="r0_to_dl1")
 
     log.info("==== END {} dl1 processing ====".format(workflow_kind))
 
-    return particle2jobid_dict, all_jobids_from_dl1_processing_stage
+    return jobids_dl1_processing_stage
 
 
 def r0_to_dl1(
     input_dir,
+    output_dir,
+    workflow_kind="lstchain",
     config_file=None,
-    train_test_ratio=0.5,
-    rng=None,
-    dl1_files_per_job=None,
-    particle=None,
-    prod_id=None,
     batch_config=None,
-    offset=None,
-    workflow_kind="lstchain",
-    keep_rta_file=False,
-    n_jobs_parallel=50,
     debug_mode=False,
+    keep_rta_file=False,
 ):
     """
     R0 to DL1 MC onsite conversion.
@@ -169,38 +110,17 @@ def r0_to_dl1(
 
     Parameters
     ----------
-    input_dir : str
+    input_dir : str or Path
         path to the files directory to analyse
-    config_file :str
-        Path to a configuration file. If none is given, the standard configuration of the selected pipeline is applied
-    train_test_ratio: float
-        Ratio of training data. Default = 0.5
-    rng : int
-        Random seed for random processes. Default = 42
-    dl1_files_per_job : int
-        Number of r0 files processed by each r0_to_dl1 batched stage. If set to None (Default), see below the `usual
-        production` case.n_r0_files_per_dl1_job
-
-        If the number of r0 files found in `input_dir` is less than 100, it is consider to be a test on a small
-        production. Therefore, the number of r0 files treated per batched stage (r0_to_dl1 or dl1ab) will be set to 10.
-
-        Usual productions have >=1000 r0 files, in this case, the number of files per job (r0_to_dl1 or dl1ab) will be
-        fixed to 50 (in case of gamma and electrons), and 100 for protons. Because of this protons end up in the
-        long queue and other particles are submitted to the short queue.
-    particle : str
-        particle type (gamma/gamma_off/proton/electron). Determines output directory structure, job naming
-        and n_r0_files_per_dl1_job if not set explicitly.
-    offset : str
-        gamma offset
-    prod_id :str
-        Production ID. If None, _v00 will be used, indicating an official base production. Default = None.
+    output_dir : str or Path
+        Output path to store files
+    config_file : str or Path
+        Path to the {lstchain, ctapipe, HiPeRTA} configuration file
     batch_config : dict
         Dictionary containing the full (source + env) source_environment and the slurm_account strings.
         ! NOTE : train_pipe AND dl1_to_dl2 **MUST** be run with the same environment.
     workflow_kind: str
         One of the supported pipelines. Defines the command to be run on r0 files
-    n_jobs_parallel: int
-        Number of jobs to be run at the same time per array.
 
     # HIPERTA ARGUMENTS
     keep_rta_file : bool
@@ -214,25 +134,12 @@ def r0_to_dl1(
     Returns
     -------
     jobid2log : dict
-        A dictionary of dictionaries containing the full log information of the script. The first `layer` contains
-        only the each jobid that the scripts has batched.
-
-            dict[jobid] = information
-
-        The second layer contains, organized by jobid,
-             - the kind of particle that corresponded to the jobid
-             - the command that was run to batch the job into the server
-             - the path to both the output and error files (job_`jobid`.o and job_`jobid`.e) that were generated
-                 when the job was send to the cluster
-
-             dict[jobid].keys() = ['particle', 'sbatch_command', 'jobe_path', 'jobo_path']
+        dictionary log containing {jobid: batch_cmd} information
     jobids_r0_dl1
-        A list of all the jobs sent by particle (including test and train set types).
+        A list of all the jobs sent for input dir
     """
 
-    log.info("Starting R0 to DL1 processing for particle {}".format(particle))
-
-    train_test_ratio = float(train_test_ratio)
+    log.info("Starting R0 to DL1 processing for files in dir : {}".format(input_dir))
 
     source_environment = batch_config["source_environment"]
     slurm_account = batch_config["slurm_account"]
@@ -259,102 +166,54 @@ def r0_to_dl1(
         log.critical("Please, select an allowed workflow kind.")
         exit(-1)
 
-    job_name = {
-        "electron": f"e_{jobtype_id}_r0dl1",
-        "gamma": f"g_{jobtype_id}_r0dl1",
-        "gamma-diffuse": f"gd_{jobtype_id}_r0dl1",
-        "proton": f"p_{jobtype_id}_r0dl1",
-        "gamma_off0.0deg": f"g0.0_{jobtype_id}_r0dl1",
-        "gamma_off0.4deg": f"g0.4_{jobtype_id}_r0dl1",
-    }
-
     log.info("Working on DL0 files in {}".format(input_dir))
 
     check_data_path(input_dir)
-    raw_files_list = get_input_filelist(input_dir)
+    raw_files_list = [file.resolve().as_posix() for file in Path(input_dir).rglob("**/*.simtel.gz")]
 
-    if len(raw_files_list) < 100:
-        dl1_files_per_job = 10
+    if len(raw_files_list) < 50:
+        dl1_files_per_job = 20
     else:
-        if "gamma" in input_dir:
-            dl1_files_per_job = 25
-        elif "gamma-diffuse" in input_dir or "electron" in input_dir:
-            dl1_files_per_job = 50
-        elif "proton" in input_dir:
-            dl1_files_per_job = 50
-        else:
-            dl1_files_per_job = 50
-
-    if rng is None:
-        rng = default_rng()
-    rng.shuffle(raw_files_list)
-
-    number_files = len(raw_files_list)
-    ntrain = int(number_files * train_test_ratio)
-    ntest = number_files - ntrain
-
-    training_list = raw_files_list[:ntrain]
-    testing_list = raw_files_list[ntrain:]
-
-    log.info("{} raw R0 files".format(number_files))
-    log.info("{} R0 files in training dataset".format(ntrain))
-    log.info("{} R0 files in test dataset".format(ntest))
-
-    with open("training.list", "w+") as newfile:
-        for f in training_list:
-            newfile.write(f)
-            newfile.write("\n")
+        dl1_files_per_job = 50
 
-    with open("testing.list", "w+") as newfile:
-        for f in testing_list:
+    with open("r0_to_dl1.list", "w+") as newfile:
+        for f in raw_files_list:
             newfile.write(f)
             newfile.write("\n")
 
-    if "off" in particle:
-        # Take out /off0.Xdeg
-        running_dir = (
-            Path(str(input_dir).replace("DL0", "running_analysis")).parent
-            / prod_id
-            / offset
-        )
-    else:
-        running_dir = (
-            Path(str(input_dir).replace("DL0", "running_analysis")) / prod_id
-        )
-
-    job_logs_dir = running_dir.joinpath("job_logs")
-    dl1_output_dir = running_dir.joinpath("DL1")
+    log.info("{} raw R0 files".format(len(raw_files_list)))
 
-    log.info("RUNNING_DIR: {}".format(running_dir))
-    log.info("JOB_LOGS DIR: {}".format(job_logs_dir))
-    log.info("DL1 DATA DIR: {}".format(dl1_output_dir))
+    # If file exists, means that prod is being re-run
+    output_dir = Path(output_dir)
+    if output_dir.exists() and any(output_dir.iterdir()):
+        shutil.rmtree(output_dir)
 
-    for directory in [running_dir, dl1_output_dir, job_logs_dir]:
-        check_and_make_dir_without_verification(directory)
+    job_logs_dir = output_dir.joinpath("job_logs_r0dl1")
+    Path(job_logs_dir).mkdir(exist_ok=True, parents=True)
 
-    # dumping the training and testing lists and splitting them in sub-lists for parallel jobs
+    log.info("DL1 DATA DIR: {}".format(output_dir))
 
     jobid2log, jobids_r0_dl1 = submit_dl1_jobs(
+        input_dir,
+        output_dir,
         base_cmd=base_cmd,
-        file_lists={"testing": testing_list, "training": training_list},
-        particle=particle,
-        job_name=job_name[particle],
+        file_list=raw_files_list,
+        job_type_id=jobtype_id,
         dl1_files_per_batched_job=dl1_files_per_job,
-        running_dir=running_dir,
         job_logs_dir=job_logs_dir,
-        n_jobs_parallel=n_jobs_parallel,
-        slurm_account=slurm_account
+        slurm_account=slurm_account,
+        filelist_name="r0_to_dl1",
     )
 
     # copy config into working dir
     if config_file is not None:
         shutil.copyfile(
-            config_file, running_dir.joinpath(Path(config_file).name)
+            config_file,
+            job_logs_dir.joinpath(Path(config_file).name)
         )
 
     # save file lists into logs
-    shutil.move("testing.list", running_dir.joinpath("testing.list"))
-    shutil.move("training.list", running_dir.joinpath("training.list"))
+    shutil.move("r0_to_dl1.list", job_logs_dir)
 
     # return it log dictionary
     return jobid2log, jobids_r0_dl1
@@ -362,14 +221,11 @@ def r0_to_dl1(
 
 def reprocess_dl1(
     input_dir,
+    output_dir,
+    workflow_kind="lstchain",
     config_file=None,
-    dl1_files_per_job=50,
-    particle=None,
-    prod_id=None,
     batch_config=None,
-    offset=None,
-    workflow_kind="lstchain",
-    n_jobs_parallel=50,
+    dl1_files_per_job=50,
 ):
     """
     Reprocessing of existing dl1 files.
@@ -381,47 +237,26 @@ def reprocess_dl1(
     ----------
     input_dir : str
         path to the files directory to analyse
+    output_dir : str or Path
+        Output path to store files
     config_file :str
         Path to a configuration file. If none is given, the standard configuration of the selected pipeline is applied
-    particle : str
-        particle type (gamma/gamma_off/proton/electron). Determines output directory structure, job naming
-        and n_r0_files_per_dl1_job if not set explicitly.
-    offset : str
-        gamma offset
-    prod_id :str
-        Production ID. If None, _v00 will be used, indicating an official base production. Default = None.
     batch_config : dict
         Dictionary containing the (full) source_environment and the slurm_account strings.
         ! NOTE : train_pipe AND dl1_to_dl2 **MUST** be run with the same environment.
     workflow_kind: str
         One of the supported pipelines. Defines the command to be run on r0 files
-    n_jobs_parallel: int
-        Number of jobs to be run at the same time per array.
     dl1_files_per_job: int
         Number of dl1 files to be processed per job array that was batched.
 
     Returns
     -------
     jobid2log : dict
-        A dictionary of dictionaries containing the full log information of the script. The first `layer` contains
-        only the each jobid that the scripts has batched.
-
-            dict[jobid] = information
-
-        The second layer contains, organized by jobid,
-             - the kind of particle that corresponded to the jobid
-             - the command that was run to batch the job into the server
-             - the path to both the output and error files (job_`jobid`.o and job_`jobid`.e) that were generated
-                 when the job was send to the cluster
-
-             dict[jobid].keys() = ['particle', 'sbatch_command', 'jobe_path', 'jobo_path']
+        dictionary log containing {jobid: batch_cmd} information
     jobids_dl1_dl1
-        A list of all the jobs sent by particle (including test and train set types).
+        A list of all the jobs sent for input dir
     """
-
-    log.info("Starting DL1 to DL1 processing for particle {}".format(particle))
-
-    input_dir = Path(input_dir)
+    log.info("Applying DL1ab on DL1 files in {}".format(input_dir))
 
     source_environment = batch_config["source_environment"]
     slurm_account = batch_config["slurm_account"]
@@ -433,165 +268,135 @@ def reprocess_dl1(
         base_cmd = f"{source_environment} lstmcpipe_cta_core_r0_dl1 -c {config_file} "
         jobtype_id = "CTA"
     else:
+        base_cmd = ""
+        jobtype_id = ""
         log.critical("Please, selected an allowed workflow kind.")
         exit(-1)
 
-    job_name = {
-        "electron": f"e_{jobtype_id}_dl1dl1",
-        "gamma": f"g_{jobtype_id}_dl1dl1",
-        "gamma-diffuse": f"gd_{jobtype_id}_dl1dl1",
-        "proton": f"p_{jobtype_id}_dl1dl1",
-        "gamma_off0.0deg": f"g0.0_{jobtype_id}_dl1dl1",
-        "gamma_off0.4deg": f"g0.4_{jobtype_id}_dl1dl1",
-    }
+    log.info("Working on DL1 files in {}".format(input_dir))
 
-    log.info("Working on DL1 files in {}".format(input_dir.as_posix()))
-    training_input_path = input_dir / "training"
-    check_data_path(training_input_path)
-    training_list = get_input_filelist(training_input_path)
-    ntrain = len(training_list)
-    testing_input_path = input_dir / "testing"
-    check_data_path(testing_input_path)
-    testing_list = get_input_filelist(testing_input_path)
-    ntest = len(testing_list)
-
-    log.info("{} DL1 files in training dataset".format(ntrain))
-    log.info("{} DL1 files in test dataset".format(ntest))
-
-    with open("training.list", "w+") as newfile:
-        for f in training_list:
-            newfile.write(f)
-            newfile.write("\n")
+    check_data_path(input_dir)
+    dl1ab_filelist = [file.resolve().as_posix() for file in Path(input_dir).rglob("*.h5")]
+
+    log.info("{} DL1 files".format(len(dl1ab_filelist)))
 
-    with open("testing.list", "w+") as newfile:
-        for f in testing_list:
+    with open("dl1ab.list", "w+") as newfile:
+        for f in dl1ab_filelist:
             newfile.write(f)
             newfile.write("\n")
 
-    if "off" in particle:
-        # Take out /off0.Xdeg
-        running_dir = (
-            Path(input_dir.as_posix().replace("DL1", "running_analysis")).parent.parent
-            / prod_id
-            / offset
-        )
-    else:
-        running_dir = (
-            Path(input_dir.as_posix().replace("DL1", "running_analysis")).parent
-            / prod_id
-        )
-
-    job_logs_dir = running_dir / "job_logs"
-    dl1_output_dir = running_dir / "DL1"
+    job_logs_dir = output_dir.joinpath("job_logs_dl1ab")
+    Path(job_logs_dir).mkdir(exist_ok=True)
 
-    log.info("RUNNING_DIR: {}".format(running_dir))
-    log.info("JOB_LOGS DIR: {}".format(job_logs_dir))
-    log.info("DL1 DATA DIR: {}".format(dl1_output_dir))
+    log.info("DL1ab DATA DIR: {}".format(output_dir))
 
-    for directory in [running_dir, dl1_output_dir, job_logs_dir]:
-        check_and_make_dir_without_verification(directory)
+    jobid2log, jobids_dl1_dl1 = submit_dl1_jobs(
+        input_dir,
+        output_dir,
+        base_cmd=base_cmd,
+        file_list=dl1ab_filelist,
+        job_type_id=jobtype_id,
+        dl1_files_per_batched_job=dl1_files_per_job,
+        job_logs_dir=job_logs_dir,
+        slurm_account=slurm_account,
+        filelist_name="dl1ab",
+    )
 
-    # dumping the training and testing lists and splitting them in sub-lists for parallel jobs
     # copy config into working dir
     if config_file is not None:
         shutil.copyfile(
-            config_file, running_dir.joinpath(Path(config_file).name)
+            config_file,
+            job_logs_dir.joinpath(Path(config_file).name)
         )
 
     # save file lists into logs
-    shutil.move("testing.list", running_dir.joinpath("testing.list"))
-    shutil.move("training.list", running_dir.joinpath("training.list"))
-
-    jobid2log, jobids_dl1_dl1 = submit_dl1_jobs(
-        base_cmd=base_cmd,
-        file_lists={"testing": testing_list, "training": training_list},
-        particle=particle,
-        job_name=job_name[particle],
-        dl1_files_per_batched_job=dl1_files_per_job,
-        running_dir=running_dir,
-        job_logs_dir=job_logs_dir,
-        n_jobs_parallel=n_jobs_parallel,
-        slurm_account=slurm_account
-    )
+    shutil.move("dl1ab.list", job_logs_dir)
 
     # return it log dictionary
     return jobid2log, jobids_dl1_dl1
 
 
 def submit_dl1_jobs(
+    input_dir,
+    output_dir,
     base_cmd,
-    file_lists,
-    particle,
-    job_name,
+    file_list,
+    job_type_id,
     dl1_files_per_batched_job,
-    running_dir,
     job_logs_dir,
-    n_jobs_parallel,
-    slurm_account
+    slurm_account,
+    filelist_name,
+    n_jobs_parallel=100,
+    dl1_processing="r0_dl1",
 ):
+    """
+    Compose sbatch command and batches it
+
+    Parameters
+    ----------
+    input_dir: str
+    output_dir: str
+    base_cmd: str
+    file_list: list
+    job_type_id: str
+    dl1_files_per_batched_job: int
+    job_logs_dir: Path
+    slurm_account: str
+    filelist_name: str
+    n_jobs_parallel: int
+    dl1_processing: str
+
+    Returns
+    -------
+    jobid2log: dict
+    jobid: str
+
+    """
+
     jobid2log = {}
-    jobids_dl1 = []
 
-    # types should be training and testing
-    for set_type, list_type in file_lists.items():
-        log.debug("Generating list for {} step".format(set_type))
-        dir_lists = running_dir.joinpath("file_lists_" + set_type)
-        output_dir = running_dir.joinpath("DL1", set_type)
+    log.info("output dir: {}".format(output_dir))
 
-        check_and_make_dir_without_verification(dir_lists)
-        check_and_make_dir_without_verification(output_dir)
+    number_of_sublists = len(file_list) // dl1_files_per_batched_job + int(
+        len(file_list) % dl1_files_per_batched_job > 0
+    )
 
-        log.info("output dir: {}".format(output_dir))
+    for i in range(number_of_sublists):
+        output_file = job_logs_dir.joinpath("{}_{}.sublist".format(filelist_name, i)).resolve().as_posix()
+        with open(output_file, "w+") as out:
+            for line in file_list[
+                i * dl1_files_per_batched_job: dl1_files_per_batched_job * (i + 1)
+            ]:
+                out.write(line)
+                out.write("\n")
 
-        number_of_sublists = len(list_type) // dl1_files_per_batched_job + int(
-            len(list_type) % dl1_files_per_batched_job > 0
-        )
-        for i in range(number_of_sublists):
-            output_file = dir_lists.joinpath("{}_{}.list".format(set_type, i)).as_posix()
-            with open(output_file, "w+") as out:
-                for line in list_type[
-                    i * dl1_files_per_batched_job : dl1_files_per_batched_job * (i + 1)
-                ]:
-                    out.write(line)
-                    out.write("\n")
-        log.info(f"{number_of_sublists} files generated for {set_type} list")
-
-        files = [f.as_posix() for f in Path(dir_lists).glob("*")]
-
-        slurm_options = {
-            "output": job_logs_dir.joinpath(f"job_%A_%a_{'train' if set_type=='training' else 'test'}.o").as_posix(),
-            "error": job_logs_dir.joinpath(f"job_%A_%a_{'train' if set_type=='training' else 'test'}.o").as_posix(),
-            "array": f"0-{len(files)-1}%{n_jobs_parallel}",
-            "job-name": f"{job_name}"
-        }
-        if particle == "proton":
-            slurm_options.update({"partition": "long"})
-        else:
-            # Computing time sometimes depends on the machine.
-            # e and g_0.X should be working on short, but sometimes runs on timeout.
-            slurm_options.update({"partition": "long"})
-        # `sbatch -A` is the --account slurm argument
-        if slurm_account != '':
-            slurm_options.update({"account": slurm_account})
-
-        # start 1 jobarray with all files included. The job selects its file based on its task id
-        cmd = f'{base_cmd} -f {" ".join(files)} --output_dir {output_dir}'
-        slurm_cmd = "sbatch --parsable "
-        for key, value in slurm_options.items():
-            slurm_cmd += f"--{key} {value} "
-        slurm_cmd += f'--wrap="{cmd}"'
-        log.debug(f"Slurm command to start the jobs: {slurm_cmd}")
-
-        jobid = os.popen(slurm_cmd).read().strip("\n")
-        log.debug(f"Submitted batch job {jobid}")
-        jobids_dl1.append(jobid)
-
-        jobid2log[jobid] = {
-            "particle": particle,
-            "set_type": set_type,
-            "jobe_path": slurm_options["error"],
-            "jobo_path": slurm_options["output"],
-            "sbatch_command": slurm_cmd
-        }
-
-    return jobid2log, jobids_dl1
+    log.info(f"{number_of_sublists} files generated for list of files at {input_dir}")
+
+    sublist_names = [f.as_posix() for f in Path(job_logs_dir).glob("*.sublist")]  # Number of sublists ??
+
+    slurm_options = {
+        "output": job_logs_dir.joinpath("job_%A_%a.o").as_posix(),
+        "error": job_logs_dir.joinpath("job_%A_%a.e").as_posix(),
+        "array": f"0-{len(sublist_names)-1}%{n_jobs_parallel}",
+        "job-name": f"{job_type_id}-{dl1_processing}"
+    }
+
+    # All files to long queue
+    slurm_options.update({"partition": "long"})
+    # `sbatch -A` is the --account slurm argument
+    if slurm_account != '':
+        slurm_options.update({"account": slurm_account})
+
+    # start 1 jobarray with all files included. The job selects its file based on its task id
+    cmd = f'{base_cmd} -f {" ".join(sublist_names)} --output_dir {output_dir}'
+    slurm_cmd = "sbatch --parsable "
+    for key, value in slurm_options.items():
+        slurm_cmd += f"--{key} {value} "
+    slurm_cmd += f'--wrap="{cmd}"'
+
+    jobid = os.popen(slurm_cmd).read().strip("\n")
+    log.debug(f"Submitted batch job {jobid}")
+
+    jobid2log.update({jobid: slurm_cmd})
+
+    return jobid2log, jobid
diff --git a/lstmcpipe/stages/mc_train.py b/lstmcpipe/stages/mc_train.py
index 993036e1..50bfed21 100755
--- a/lstmcpipe/stages/mc_train.py
+++ b/lstmcpipe/stages/mc_train.py
@@ -1,30 +1,31 @@
-#!/usr//bin/env python
-
-# T. Vuillaume,
-# Modifications by E. Garcia
-# Code train models from DL1 files onsite (La Palma cluster)
+#!/usr/bin/env python
 
 import os
-import time
 import shutil
 import logging
-from lstmcpipe.io.data_management import check_and_make_dir_without_verification
+from pathlib import Path
 from lstmcpipe.workflow_management import save_log_to_file
+from lstmcpipe.io.data_management import check_and_make_dir_without_verification
 
 
 log = logging.getLogger(__name__)
 
 
-def batch_train_pipe(log_from_merge, config_file, jobids_from_merge, batch_config, logs):
+def batch_train_pipe(
+        dict_paths,
+        jobids_from_merge,
+        config_file,
+        batch_config,
+        logs
+):
     """
     Function to batch the lstchain train_pipe once the proton and gamma-diffuse merge_and_copy_dl1 batched jobs have
     finished.
 
     Parameters
     ----------
-    log_from_merge : dict
-        dictionary containing the output name and abs path to the DL1 files, derived in merge_and_copy and saved
-        through the log
+    dict_paths : dict
+        Core dictionary with {stage: PATHS} information
     config_file : str
         Path to a configuration file. If none is given, a standard configuration is applied
     jobids_from_merge : str
@@ -41,43 +42,53 @@ def batch_train_pipe(log_from_merge, config_file, jobids_from_merge, batch_confi
     jobid_4_dl1_to_dl2 : str
         string containing the jobid to be passed to the next stage of the workflow (as a slurm dependency).
         For the next stage, however, it will be needed TRAIN + MERGED jobs
-    model_path : str
-        Path with the model's directory
     """
-    debug_log = {}
+    log_train = {}
+    debug_train = {}
+    jobid_for_dl1_to_dl2 = []
 
     log.info("==== START {} ====".format("batch mc_train_workflow"))
-    time.sleep(1)
-
-    gamma_dl1_train_file = log_from_merge["gamma-diffuse"]["training"][
-        "train_path_and_outname_dl1"
-    ]
-    proton_dl1_train_file = log_from_merge["proton"]["training"][
-        "train_path_and_outname_dl1"
-    ]
-
-    log_train, jobid_4_dl1_to_dl2, model_path = train_pipe(
-        gamma_dl1_train_file,
-        proton_dl1_train_file,
-        config_file=config_file,
-        batch_configuration=batch_config,
-        wait_ids_proton_and_gammas=jobids_from_merge,
-    )
 
-    debug_log[jobid_4_dl1_to_dl2] = (
-        f"The single jobid from train_pipe that depends of {jobids_from_merge} - merge"
-        f"_and_copy jobids"
-    )
+    for paths in dict_paths:
+
+        gamma_dl1_train_file = paths["input"]["gamma"]
+        proton_dl1_train_file = paths["input"]["proton"]
+        models_dir = paths["output"]
+
+        job_logs, jobid = train_pipe(
+            gamma_dl1_train_file,
+            proton_dl1_train_file,
+            models_dir,
+            config_file=config_file,
+            batch_configuration=batch_config,
+            wait_jobs_dl1=jobids_from_merge,
+        )
+
+        log_train.update(job_logs)
+        jobid_for_dl1_to_dl2.append(jobid)
+
+        debug_train[jobid] = (
+            f"The single jobid from train_pipe that depends of {jobids_from_merge} - merge"
+            f"_and_copy jobids"
+        )
+
+    jobid_for_dl1_to_dl2 = ",".join(jobid_for_dl1_to_dl2)
 
     save_log_to_file(log_train, logs["log_file"], workflow_step="train_pipe")
-    save_log_to_file(debug_log, logs["debug_file"], workflow_step="train_pipe")
+    save_log_to_file(debug_train, logs["debug_file"], workflow_step="train_pipe")
 
     log.info("==== END {} ====".format("batch mc_train_workflow"))
 
-    return jobid_4_dl1_to_dl2, model_path
+    return jobid_for_dl1_to_dl2
 
 
-def batch_plot_rf_features(dir_models, config_file, batch_configuration, train_jobid, logs):
+def batch_plot_rf_features(
+        dict_paths,
+        config_file,
+        batch_configuration,
+        train_jobid,
+        logs,
+    ):
     """
     Batches the plot_model_importance.py script that creates a .png with the RF feature's importance models
     after the RF are trained.
@@ -85,8 +96,8 @@ def batch_plot_rf_features(dir_models, config_file, batch_configuration, train_j
 
     Parameters
     ----------
-    dir_models: str
-        Path to model's directory
+     dict_paths : dict
+        Core dictionary with {stage: PATHS} information
     config_file: str
         Path to lstchain config file
     batch_configuration : dict
@@ -95,47 +106,62 @@ def batch_plot_rf_features(dir_models, config_file, batch_configuration, train_j
         Single jobid from training stage.
     logs: dict
         Dictionary with logs files
+
+    Returns
+    -------
+    jobid : str
+        jobid of batched stage
     """
     log_rf_feat = {}
     log_debug = {}
+    all_jobs_plot_rf_feat = []
+
+    log.info("==== START {} ====".format("batch plot RF features importance"))
 
     source_env = batch_configuration["source_environment"]
     slurm_account = batch_configuration["slurm_account"]
 
-    log.info("==== START {} ====".format("batch plot RF features importance"))
-    time.sleep(1)
-    jobe = os.path.join(dir_models, "job_plot_rf_feat_importance.e")
-    jobo = os.path.join(dir_models, "job_plot_rf_feat_importance.o")
+    for path in dict_paths:
+        models_dir = path["output"]
 
-    batch_cmd = f"lstmcpipe_plot_models_importance {dir_models} -cf {config_file}"
+        jobe = Path(models_dir).joinpath("job_plot_rf_feat_importance.e").resolve().as_posix()
+        jobo = Path(models_dir).joinpath(models_dir, "job_plot_rf_feat_importance.o").resolve().as_posix()
 
-    slurm_cmd = "sbatch --parsable --mem=16G "
-    if slurm_account != "":
-        slurm_cmd += f" -A {slurm_account}"
-    slurm_cmd += (
-        f" --dependency=afterok:{train_jobid} -e {jobe} -o {jobo} "
-        f' -J RF_importance --wrap="export MPLBACKEND=Agg; {source_env} {batch_cmd}"'
-    )
-    jobid = os.popen(slurm_cmd).read().strip("\n")
+        batch_cmd = f"lstmcpipe_plot_models_importance {models_dir} -cf {config_file}"
+
+        slurm_cmd = "sbatch --parsable --mem=16G "
+        if slurm_account != "":
+            slurm_cmd += f" -A {slurm_account}"
+        slurm_cmd += (
+            f" --dependency=afterok:{train_jobid} -e {jobe} -o {jobo} "
+            f' -J RF_importance --wrap="export MPLBACKEND=Agg; {source_env} {batch_cmd}"'
+        )
+        jobid = os.popen(slurm_cmd).read().strip("\n")
 
-    log_rf_feat[jobid] = slurm_cmd
-    log_debug[jobid] = "Single job_id to plot RF feature s importance"
+        log_rf_feat[jobid] = slurm_cmd
+        log_debug[jobid] = "Single job_id to plot RF feature s importance"
+        all_jobs_plot_rf_feat.append(jobid)
+
+    all_jobs_plot_rf_feat = ','.join(all_jobs_plot_rf_feat)
 
     save_log_to_file(log_rf_feat, logs["log_file"],
                      workflow_step="plot_RF_features_importance")
     save_log_to_file(log_debug, logs["debug_file"],
                      workflow_step="plot_RF_features_importance")
 
-    log.info(" Random Forest importance's plot will be saved at: {}".format(dir_models))
+    log.info(" Random Forest importance's plot will be saved at: {}".format(models_dir))
     log.info("==== END {} ====".format("batch plot RF features importance"))
 
+    return all_jobs_plot_rf_feat
+
 
 def train_pipe(
     gamma_dl1_train_file,
     proton_dl1_train_file,
+    models_dir,
     config_file=None,
     batch_configuration='',
-    wait_ids_proton_and_gammas=None,
+    wait_jobs_dl1=None,
 ):
     """
     Train RF from MC DL1 data (onsite LaPalma cluster)
@@ -143,18 +169,20 @@ def train_pipe(
     Parameters
     ----------
     gamma_dl1_train_file: str
-        path to the gamma file
+        FILE path to the gamma
     proton_dl1_train_file: str
-        path to the proton file
+        FILE path to the proton
+    models_dir:
+        DIR trained models output path
     config_file: str
         Path to a configuration file. If none is given, a standard configuration is applied
     batch_configuration : dict
         Dictionary containing the (full) source_environment and the slurm_account strings to be passed to the
         sbatch commands
         ! NOTE : train_pipe AND dl1_to_dl2 MUST BE RUN WITH THE SAME ENVIRONMENT
-    wait_ids_proton_and_gammas : str
-        a string (of chained jobids separated by ',' and without spaces between each element), to indicate the
-        dependencies of the job to be batched
+    wait_jobs_dl1 : str
+        A string (of chained job_ids separated by ',' and without spaces between each element), containing
+        all the job_ids of the merging stage
 
     Returns
     -------
@@ -165,53 +193,32 @@ def train_pipe(
     jobid_train : str
         jobid of the batched job to be send (for dependencies purposes) to the next stage of the
         workflow (onsite_mc_dl1_to_dl2)
-
-    models_dir : str
-        Path with the directory where the models are stored
-
     """
     log_train = {}
 
     source_environment = batch_configuration["source_environment"]
     slurm_account = batch_configuration["slurm_account"]
 
-    dl1_proton_dir = os.path.dirname(os.path.abspath(proton_dl1_train_file))
-
-    # check if it path follows the established paths (lstchain-like) or not (rta-like) ##
-    if dl1_proton_dir.find("/mc/DL1/") > 0:
-        models_dir = dl1_proton_dir.replace("/mc/DL1", "/models")
-    else:
-        models_dir = dl1_proton_dir.replace("/DL1", "/models")
-    models_dir = models_dir.replace("/proton/", "/")
-
     log.info("Models will be placed in {}".format(models_dir))
     check_and_make_dir_without_verification(models_dir)
 
-    cmd = ""
-    if source_environment is not None:
-        cmd += source_environment
-
-    cmd += (
-        f" lstchain_mc_trainpipe --fg {os.path.abspath(gamma_dl1_train_file)}"
-        f" --fp {os.path.abspath(proton_dl1_train_file)} -o {models_dir}"
+    cmd = (
+        f" {source_environment} lstchain_mc_trainpipe --fg {gamma_dl1_train_file}"
+        f" --fp {proton_dl1_train_file} -o {models_dir}"
     )
 
     if config_file is not None:
         cmd = cmd + " -c {}".format(config_file)
 
-    jobo = os.path.join(models_dir, "train_job.o")
-    jobe = os.path.join(models_dir, "train_job.e")
-
-    # TODO dry-run option ?
-    # if dry_run:
-    #     print(cmd)
+    jobo = Path(models_dir).joinpath("train_job.o").resolve().as_posix()
+    jobe = Path(models_dir).joinpath("train_job.e").resolve().as_posix()
 
     # 'sbatch --parsable --dependency=afterok:{wait_ids_proton_and_gammas} -e {jobe} -o {jobo} --wrap="{base_cmd}"'
     batch_cmd = "sbatch --parsable -p long --mem=32G"
     if slurm_account != "":
         batch_cmd += f" -A {slurm_account}"
-    if wait_ids_proton_and_gammas != "":
-        batch_cmd += " --dependency=afterok:" + wait_ids_proton_and_gammas
+    if wait_jobs_dl1 != "":
+        batch_cmd += " --dependency=afterok:" + wait_jobs_dl1
     batch_cmd += f' -J train_pipe -e {jobe} -o {jobo} --wrap="{cmd}" '
 
     jobid_train = os.popen(batch_cmd).read().strip("\n")
@@ -220,7 +227,8 @@ def train_pipe(
     # copy config into working dir
     if config_file is not None:
         shutil.copyfile(
-            config_file, os.path.join(models_dir, os.path.basename(config_file))
+            config_file,
+            Path(models_dir).joinpath(Path(config_file).name)
         )
 
-    return log_train, jobid_train, models_dir
+    return log_train, jobid_train
diff --git a/lstmcpipe/workflow_management.py b/lstmcpipe/workflow_management.py
index 4fab8437..bcc59bbe 100644
--- a/lstmcpipe/workflow_management.py
+++ b/lstmcpipe/workflow_management.py
@@ -3,7 +3,7 @@
 # Enrique Garcia Nov 2019
 
 import os
-import yaml
+from ruamel.yaml import YAML
 import shutil
 import logging
 from pathlib import Path
@@ -35,74 +35,14 @@ def save_log_to_file(dictionary, output_file, workflow_step=None):
     dict2log = {workflow_step: dictionary}
 
     with open(output_file, "a+") as fileout:
-        yaml.dump(dict2log, fileout)
-
-
-def create_dl1_filenames_dict(dl1_directory, particles_loop, gamma_offsets=None):
-    """
-    Function that creates a dictionary with the filenames of all the final dl1 files (the same is done
-    in the merge_and_copy_dl1 function) so that it can be passed to the rest of the stages, in case the full workflow
-    it is not run from the start.
-
-    Parameters
-    ----------
-    dl1_directory: str
-        path to the dl1 directory files
-    particles_loop : list
-        list with the particles to be processed. Takes the global variable ALL_PARTICLES
-    gamma_offsets : list
-
-    Returns
-    -------
-    dl1_filename_directory : dict
-        dictionary with the name (and absolute path) of the dl1 files with the name of the particles set as key of the
-        dictionary
-
-             dl1_filename_directory[particle].keys() = ['train_path_and_outname_dl1', 'test_path_and_outname_dl1']
-    """
-    dl1_filename_directory = {}
-
-    for particle in particles_loop:
-        if gamma_offsets is not None and particle == "gamma":
-            for off in gamma_offsets:
-                _particle = particle + "_" + off
-                dl1_filename_directory[_particle] = {"training": {}, "testing": {}}
-
-                dl1_filename_directory[_particle]["training"][
-                    "train_path_and_outname_dl1"
-                ] = next(
-                    Path(dl1_directory.format(particle), off).glob("*training*.h5")
-                ).resolve().as_posix()
-                dl1_filename_directory[_particle]["testing"][
-                    "test_path_and_outname_dl1"
-                ] = next(
-                    Path(dl1_directory.format(particle), off).glob("*testing*.h5")
-                ).resolve().as_posix()
-        else:
-            dl1_filename_directory[particle] = {"training": {}, "testing": {}}
-            dl1_filename_directory[particle]["training"][
-                "train_path_and_outname_dl1"
-            ] = next(Path(dl1_directory.format(particle)).glob("*training*.h5")
-                     ).resolve().as_posix()
-            dl1_filename_directory[particle]["testing"][
-                "test_path_and_outname_dl1"
-            ] = next(Path(dl1_directory.format(particle)).glob("*testing*.h5")
-                     ).resolve().as_posix()
-
-    return dl1_filename_directory
+        YAML().dump(dict2log, fileout)
 
 
 def batch_mc_production_check(
-    jobids_from_r0_to_dl1,
-    jobids_from_merge,
-    jobids_from_train_pipe,
-    jobids_from_dl1_to_dl2,
-    jobids_from_dl2_to_irf,
-    jobids_from_dl2_to_sensitivity,
-    prod_id,
+    dict_jobids_all_stages,
     log_directory,
+    prod_id,
     prod_config_file,
-    last_stage,
     batch_config,
     logs_files,
 ):
@@ -113,18 +53,11 @@ def batch_mc_production_check(
 
     Parameters
     ----------
-    jobids_from_r0_to_dl1 : str
-        jobs from the dl1_to_dl2 stage
-    prod_id : str
-        MC Production ID.
-    jobids_from_merge :  str
-    jobids_from_train_pipe : str
-    jobids_from_dl1_to_dl2: str
-    jobids_from_dl2_to_irf: str
-    jobids_from_dl2_to_sensitivity: str
+    dict_jobids_all_stages : dict
+        dict containing the {stage: all_job_ids related} information
     log_directory: Path
+    prod_id: str
     prod_config_file: str
-    last_stage: str
     batch_config: dict
     logs_files: dict
         Dictionary with logs files
@@ -140,68 +73,39 @@ def batch_mc_production_check(
     source_env = batch_config["source_environment"]
     slurm_account = batch_config["slurm_account"]
 
-    jobids_stages = {
-        "r0_to_dl1": jobids_from_r0_to_dl1,
-        "merge_and_copy_dl1": jobids_from_merge,
-        "train_pipe": jobids_from_train_pipe,
-        "dl1_to_dl2": jobids_from_dl1_to_dl2,
-        "dl2_to_irfs": jobids_from_dl2_to_irf,
-        "dl2_to_sensitivity": jobids_from_dl2_to_sensitivity,
-    }
-
-    if jobids_from_r0_to_dl1 != "":
-        all_pipeline_jobs.append(jobids_from_r0_to_dl1)
-    if jobids_from_merge != "":
-        all_pipeline_jobs.append(jobids_from_merge)
-    if jobids_from_train_pipe != "":
-        all_pipeline_jobs.append(jobids_from_train_pipe)
-    if jobids_from_dl1_to_dl2 != "":
-        all_pipeline_jobs.append(jobids_from_dl1_to_dl2)
-    if jobids_from_dl2_to_irf != "":
-        all_pipeline_jobs.append(jobids_from_dl2_to_irf)
-    if jobids_from_dl2_to_sensitivity != "":
-        all_pipeline_jobs.append(jobids_from_dl2_to_sensitivity)
+    for stage, jobids in dict_jobids_all_stages.items():
+        all_pipeline_jobs.append(jobids)
+        debug_log.update({f"SUMMARY_{stage}": jobids})
 
     all_pipeline_jobs = ",".join(all_pipeline_jobs)
 
-    which_last_stage = jobids_stages[last_stage]
+    # Copy lstmcpipe config used to log directory
+    shutil.copyfile(
+        Path(prod_config_file).resolve(),
+        log_directory.joinpath(f"config_MC_prod_{prod_id}.yml")
+    )
 
     # Save machine info into the check file
-    check_prod_file = log_directory.joinpath(f"check_MC_{prod_id}.txt").name
-    save_lstmcpipe_config = log_directory.joinpath(f"config_MC_prod_{prod_id}.yml")
-
-    # Copy lstmcpipe config used
-    shutil.copyfile(Path(prod_config_file).resolve(), save_lstmcpipe_config)
+    check_prod_file = log_directory.joinpath(f"check_MC_{prod_id}.txt").absolute().as_posix()
 
     cmd_wrap = f"touch {check_prod_file}; "
     cmd_wrap += (
         f"sacct --format=jobid,jobname,nodelist,cputime,state,exitcode,avediskread,maxdiskread,avediskwrite,"
         f"maxdiskwrite,AveVMSize,MaxVMSize,avecpufreq,reqmem -j {all_pipeline_jobs} >> {check_prod_file}; "
-        f"mv slurm-* IRFFITSWriter.provenance.log {log_directory.name};"
+        f"mv slurm-* IRFFITSWriter.provenance.log {log_directory.absolute().as_posix()} "
     )
 
     batch_cmd = "sbatch -p short --parsable"
     if slurm_account != "":
         batch_cmd += f" -A {slurm_account}"
     batch_cmd += (
-        f" --dependency=afterok:{which_last_stage} -J prod_check"
+        f" --dependency=afterok:{all_pipeline_jobs} -J prod_check"
         f' --wrap="{source_env} {cmd_wrap}"'
     )
 
     jobid = os.popen(batch_cmd).read().strip("\n")
     log.info(f"Submitted batch CHECK-job {jobid}")
-
-    # and in case the code brakes, here there is a summary of all the jobs by stages
-    debug_log[
-        jobid
-    ] = "single jobid batched to check the check command worked correctly."
-    debug_log["sbatch_cmd"] = batch_cmd
-    debug_log["SUMMARY_r0_dl1"] = jobids_from_r0_to_dl1
-    debug_log["SUMMARY_merge"] = jobids_from_merge
-    debug_log["SUMMARY_train_pipe"] = jobids_from_train_pipe
-    debug_log["SUMMARY_dl1_dl2"] = jobids_from_dl1_to_dl2
-    debug_log["SUMMARY_dl2_irfs"] = jobids_from_dl2_to_irf
-    debug_log["SUMMARY_dl2_sensitivity"] = jobids_from_dl2_to_sensitivity
+    debug_log.update({f"prod_check_{jobid}": batch_cmd})
 
     save_log_to_file(debug_log, logs_files["debug_file"],
                      workflow_step="check_full_workflow")
diff --git a/setup.py b/setup.py
index 83f6c450..34cbca80 100644
--- a/setup.py
+++ b/setup.py
@@ -11,12 +11,10 @@ def find_scripts(script_dir, prefix):
     script_list = [f for f in Path(script_dir).iterdir() if f.name.startswith(prefix)]
     return script_list
 
-
 def readfile(filename):
     with open(filename, "r+") as f:
         return f.read()
 
-
 # Read package info from codemeta.json
 with open(os.path.join(os.path.dirname(__file__), "codemeta.json")) as file:
     metadata = json.load(file)
@@ -29,7 +27,6 @@ def readfile(filename):
 version = metadata["version"]
 description = metadata["description"]
 
-
 scripts_list = find_scripts("lstmcpipe", "onsite_")
 
 entry_points = {
@@ -39,14 +36,14 @@ def readfile(filename):
         "lstmcpipe_plot_irfs = lstmcpipe.plots.plot_irfs:main",
         "lstmcpipe_hiperta_r0_to_dl1lstchain = lstmcpipe.hiperta.hiperta_r0_to_dl1lstchain:main",
         "lstmcpipe_dl2_to_sensitivity = lstmcpipe.scripts.script_dl2_to_sensitivity:main",
-        "lstmcpipe_utils_move_dir = lstmcpipe.scripts.script_merge_utils_move_dir:main",
-        "lstmcpipe_utils_cp_config = lstmcpipe.scripts.script_merge_utils_copy_config:main",
+        "lstmcpipe_train_test_split = lstmcpipe.scripts.script_train_test_splitting:main",
         "lstmcpipe_lst_core_r0_dl1 = lstmcpipe.scripts.script_batch_filelist_lst:main",
         "lstmcpipe_lst_core_dl1ab = lstmcpipe.scripts.script_batch_filelist_lst_dl1ab:main",
         "lstmcpipe_cta_core_r0_dl1 = lstmcpipe.scripts.script_batch_filelist_cta:main",
         "lstmcpipe_rta_core_r0_dl1 = lstmcpipe.scripts.script_batch_filelist_rta:main",
         "lstmcpipe_compare_irfs = lstmcpipe.scripts.script_compare_irfs:main",
         "lstmcpipe_validate_config = lstmcpipe.scripts.script_lstmcpipe_validate_config:main",
+        "lstmcpipe_generate_config = lstmcpipe.scripts.lstmcpipe_generate_config:main",
     ]
 }
 
@@ -56,14 +53,14 @@ def readfile(filename):
     description=description,
     install_requires=[
         "lstchain",
-        "pyyaml",
         "numpy",
         "astropy",
         "ctaplot>=0.5",
         "pyirf>=0.4",
         "matplotlib",
         "pytest",
-        "setuptools_scm"
+        "ruamel.yaml>=0.17",
+        "setuptools_scm",
     ],
     packages=find_packages(exclude="lstmcpipe._dev_version"),
     tests_require=["pytest"],
@@ -79,6 +76,8 @@ def readfile(filename):
     ],
     scripts=scripts_list,
     entry_points=entry_points,
+    include_package_data=True,
+    data_files=[('lstmcpipe', ['lstmcpipe/base_config_lstmcpipe.yaml'])],
     use_scm_version={
         "write_to": Path(__file__).parent.joinpath("lstmcpipe/_version.py"),
         "write_to_template": "__version__ = '{version}'",


Test patch:
diff --git a/lstmcpipe/config/tests/test_paths_config.py b/lstmcpipe/config/tests/test_paths_config.py
new file mode 100644
index 00000000..d2be5c53
--- /dev/null
+++ b/lstmcpipe/config/tests/test_paths_config.py
@@ -0,0 +1,124 @@
+import pytest
+import tempfile
+from ruamel.yaml import YAML
+
+from lstmcpipe.config import paths_config
+from lstmcpipe.config import pipeline_config
+
+default_stages = [
+    "r0_to_dl1",
+    "train_test_split",
+    "merge_dl1",
+    "train_pipe",
+    "dl1_to_dl2",
+    "dl2_to_sensitivity",
+    "dl2_to_irfs",
+]
+
+default_particles = [
+    "gamma-diffuse",
+    "proton",
+    "gamma",
+    "electron",
+    "proton",
+]
+
+default_point_source_offs = [
+    "off0.0deg",
+    "off0.4deg",
+]
+
+
+@pytest.mark.xfail(raises=NotImplementedError)
+def test_path_config_generate_fail():
+    pcfg = paths_config.PathConfig('v00')
+    pcfg.generate()
+
+
+def test_path_config_generate():
+    class NewPathConfig(paths_config.PathConfig):
+        def __init__(self, prod_id):
+            super().__init__(prod_id)
+            self.stages = ['stage1']
+
+        @property
+        def stage1(self):
+            return {'input': 'input_path', 'output': None}
+
+    pcfg = NewPathConfig('v00')
+    pcfg.generate() == {'stage1': {'input': 'input_path', 'output': None}}
+
+
+def test_path_config_save():
+    pcfg = paths_config.PathConfig('v00')
+    pcfg.paths = {'a': 'rick'}
+    with tempfile.NamedTemporaryFile() as f:
+        pcfg.save_yml(f.name, overwrite=True)
+        read_cfg = YAML().load(open(f.name).read())
+        pipeline_config.load_config(f.name)
+    assert read_cfg['prod_type'] == 'PathConfig'
+    assert read_cfg['prod_id'] == 'v00'
+
+
+def test_PathConfigProd5Trans80():
+    from lstmcpipe.config.paths_config import PathConfigProd5Trans80
+    prod_id = '20210416_v0.7.3_prod5_trans_80_local_taicut_8_4'
+    cfg = PathConfigProd5Trans80(prod_id)
+    cfg.generate()
+    assert cfg.prod_id == prod_id
+
+    assert "/20200629_prod5_trans_80/" in cfg.base_dir
+    assert all(stage in default_stages for stage in cfg.stages)
+    assert all(particle in default_particles for particle in cfg.particles)
+    assert all(offset in default_point_source_offs for offset in cfg.point_src_offsets)
+    assert cfg.training_particles == ["gamma-diffuse", "proton"]
+    assert cfg.testing_particles == ["gamma", "electron", "proton", "gamma-diffuse"]
+    assert cfg.zenith == "zenith_20deg"
+    assert cfg.base_dir == \
+           "/fefs/aswg/data/mc/{data_level}/20200629_prod5_trans_80/{particle}/{zenith}/south_pointing/{prod_id}"
+    assert cfg.dl1_dir("gamma") == \
+           f"/fefs/aswg/data/mc/DL1/20200629_prod5_trans_80/gamma/zenith_20deg/south_pointing/{prod_id}/off0.4deg"
+    assert cfg.dl2_dir("gamma") == \
+           f"/fefs/aswg/data/mc/DL2/20200629_prod5_trans_80/gamma/zenith_20deg/south_pointing/{prod_id}/off0.4deg"
+    assert cfg.dl2_output_file("gamma") == \
+           f"/fefs/aswg/data/mc/DL2/20200629_prod5_trans_80/gamma/zenith_20deg/south_pointing/{prod_id}/off0.4deg/dl2_gamma_{prod_id}_test.h5"
+    assert cfg.irf_dir("gamma") == \
+           f"/fefs/aswg/data/mc/IRF/20200629_prod5_trans_80/zenith_20deg/south_pointing/{prod_id}/gamma"
+    assert cfg.merge_output_file("gamma", "train") == \
+           f"/fefs/aswg/data/mc/DL1/20200629_prod5_trans_80/gamma/zenith_20deg/south_pointing/{prod_id}/off0.4deg/dl1_gamma_{prod_id}_train.h5"
+    assert cfg.merge_output_file("gamma", "test") == \
+           f"/fefs/aswg/data/mc/DL1/20200629_prod5_trans_80/gamma/zenith_20deg/south_pointing/{prod_id}/off0.4deg/dl1_gamma_{prod_id}_test.h5"
+    assert cfg.models_path() == \
+           f"/fefs/aswg/data/models/20200629_prod5_trans_80/zenith_20deg/south_pointing/{prod_id}"
+    assert cfg.r0_dir("gamma") == \
+           "/fefs/aswg/data/mc/DL0/20200629_prod5_trans_80/gamma/zenith_20deg/south_pointing/off0.4deg"
+    assert cfg.sensitivity_file(offset="0.0deg") == \
+           f"/fefs/aswg/data/mc/IRF/20200629_prod5_trans_80/zenith_20deg/south_pointing/{prod_id}/0.0deg/sensitivity_{prod_id}_0.0deg.fits.gz"
+    assert cfg.train_dir("gamma") == \
+           f"/fefs/aswg/data/mc/DL1/20200629_prod5_trans_80/gamma/zenith_20deg/south_pointing/{prod_id}/off0.4deg/train"
+    assert cfg.test_dir("gamma") == \
+           f"/fefs/aswg/data/mc/DL1/20200629_prod5_trans_80/gamma/zenith_20deg/south_pointing/{prod_id}/off0.4deg/test"
+
+    for stg in cfg.stages:
+        prop = getattr(cfg, stg)
+        assert isinstance(prop, list)
+
+        for path in prop:
+            assert isinstance(path, dict)
+
+            if stg == "train_pipe":
+                assert isinstance(path["input"], dict)
+                assert all(item in ["gamma", "proton"] for item in path["input"])
+            elif stg == "train_test_split":
+                assert isinstance(path["output"], dict)
+                assert all(item in ["train", "test"] for item in path["output"])
+            elif stg == 'dl1_to_dl2':
+                assert 'path_model' in path
+            elif stg == "dl2_to_sensitivity":
+                assert isinstance(path["input"], dict)
+                assert all(item in ["gamma_file", "proton_file", "electron_file"] for item in path["input"])
+            elif stg == "dl2_to_irfs":
+                assert "options" in path
+                assert isinstance(path["input"], dict)
+                assert all(item in ["gamma_file", "proton_file", "electron_file"] for item in path["input"])
+
diff --git a/lstmcpipe/config/tests/test_config.py b/lstmcpipe/config/tests/test_pipeline_config.py
similarity index 52%
rename from lstmcpipe/config/tests/test_config.py
rename to lstmcpipe/config/tests/test_pipeline_config.py
index a12b1b7a..2d2898d3 100644
--- a/lstmcpipe/config/tests/test_config.py
+++ b/lstmcpipe/config/tests/test_pipeline_config.py
@@ -1,24 +1,19 @@
 from lstmcpipe.config import export_env
 from lstmcpipe.config.pipeline_config import (
     config_valid,
-    parse_config_and_handle_global_vars,
+    complete_lstmcpipe_config,
 )
 import tempfile
 import os
 import pytest
-from datetime import datetime
+# from datetime import datetime
 
 yaml_keys = [
     "prod_id",
-    "stages_to_be_run",
-    "base_path",
-    "pointing",
-    "zenith",
-    "particles",
-    "offset_gammas",
-    "obs_date",
+    "stages_to_run",
     "prod_type",
     "workflow_kind",
+    "stages"
 ]
 dummy_config = {k: None for k in yaml_keys}
 dummy_config["merging_options"] = {"no_image": False}
@@ -43,11 +38,9 @@ def test_config_valid():
         config_valid(dummy_config)
 
     valid = dummy_config.copy()
-    valid["prod_type"] = "prod5"
+    valid["prod_type"] = "PathConfigProd5Trans80"
     valid["workflow_kind"] = "lstchain"
-    valid["obs_date"] = "20200629_prod5_trans_80"
-    valid["stages_to_be_run"] = []
-    valid["base_path"] = "/path/to/dl0"
+    valid["stages_to_run"] = []
     assert config_valid(valid)
 
     invalid_workflow = valid.copy()
@@ -55,29 +48,13 @@ def test_config_valid():
     with pytest.raises(Exception):
         config_valid(invalid_workflow)
 
-    invalid_date = valid.copy()
-    invalid_date["obs_date"] = "1970-01-01"
-    with pytest.raises(Exception):
-        config_valid(invalid_date)
-
-    invalid_prod = valid.copy()
-    invalid_prod["prod_type"] = "prod42"
-    with pytest.raises(Exception):
-        config_valid(invalid_prod)
-
-    invalid_date_prod = valid.copy()
-    # this should only be allowed with prod3, which is not chosen here
-    invalid_date_prod["obs_date"] = "20190415"
-    with pytest.raises(Exception):
-        config_valid(invalid_date_prod)
-
     missing_reference = valid.copy()
-    missing_reference["stages_to_be_run"] = ["dl1ab"]
+    missing_reference["stages_to_run"] = ["dl1ab"]
     with pytest.raises(KeyError):
         config_valid(missing_reference)
 
 
-def test_parse_config_and_handle_global_vars():
+def test_complete_lstmcpipe_config():
     """
     Test that the file paths are properly set.
     This only tests a subset of the variables, that are set in
@@ -85,27 +62,13 @@ def test_parse_config_and_handle_global_vars():
     """
     config = dummy_config.copy()
     config["workflow_kind"] = "lstchain"
-    config["prod_type"] = "prod5"
-    config["obs_date"] = "20200629_prod5_trans_80"
-    config["pointing"] = "90"
-    config["zenith"] = "45"
-    config["base_path"] = "/dummy/path/to/files"
-    config["stages_to_be_run"] = ["r0_to_dl1"]
+    config["prod_type"] = "PathConfigProd5Trans80"
+    config["stages_to_run"] = ["r0_to_dl1"]
 
-    parsed_config = parse_config_and_handle_global_vars(config)
-    date = datetime.today().strftime("%Y%m%d")
-    assert (
-        parsed_config["DL1_output_dir"]
-        == "/dummy/path/to/files/DL1/20200629_prod5_trans_80/{}/45/90/"
-        + date
-        + "_v0.9.1_prod5_trans_80_None"
-    )
-    assert (
-        parsed_config["model_output_dir"]
-        == "/dummy/path/to/files/models/20200629_prod5_trans_80/45/90/"
-        + date
-        + "_v0.9.1_prod5_trans_80_None"
-    )
+    parsed_config = complete_lstmcpipe_config(config)
+    # date = datetime.today().strftime("%Y%m%d")
+    # TODO test check some paths
+    # TODO test check model path
     assert (
         parsed_config["batch_config"]["source_environment"] == "source src_file; conda activate env; "
     )
@@ -115,5 +78,5 @@ def test_parse_config_and_handle_global_vars():
 
     new_dummy = config.copy()
     new_dummy["slurm_config"] = {"user_account": "aswg"}
-    new_parsed_config = parse_config_and_handle_global_vars(new_dummy)
+    new_parsed_config = complete_lstmcpipe_config(new_dummy)
     assert new_parsed_config["batch_config"]["slurm_account"] == "aswg"
diff --git a/lstmcpipe/io/tests/test_lstmcpipe_tree_path.py b/lstmcpipe/io/tests/test_lstmcpipe_tree_path.py
index 580f2943..1ce2fdf7 100644
--- a/lstmcpipe/io/tests/test_lstmcpipe_tree_path.py
+++ b/lstmcpipe/io/tests/test_lstmcpipe_tree_path.py
@@ -1,5 +1,5 @@
 from pathlib import Path
-from lstmcpipe import lstmcpipe_root_dir
+from lstmcpipe import prod_logs
 
 
 def test_create_log_files():
@@ -59,5 +59,5 @@ def test_create_log_dir():
     from ..lstmcpipe_tree_path import create_log_dir
     log_dirname = create_log_dir("test_prodID")
     assert log_dirname.is_dir()
-    assert log_dirname.name == lstmcpipe_root_dir.joinpath("prod_logs", "logs_test_prodID").name
+    assert log_dirname.name == prod_logs.joinpath("logs_test_prodID").name
     log_dirname.rmdir()
diff --git a/lstmcpipe/scripts/generate_test_lapalma.py b/lstmcpipe/scripts/generate_test_lapalma.py
new file mode 100644
index 00000000..5c8a984b
--- /dev/null
+++ b/lstmcpipe/scripts/generate_test_lapalma.py
@@ -0,0 +1,86 @@
+"""
+Script to generate a mini production tree with symlinks to a few simtel files and the corresponding path config.
+This is used to test lstmcpipe on a small prod
+"""
+
+from pathlib import Path
+from tqdm import tqdm
+from datetime import date
+import os
+import argparse
+
+from lstmcpipe.config import paths_config
+
+
+
+def generate_tree(base_dir, working_dir, nfiles):
+    """
+    Walk the base dir looking for simtels files
+    When a directory contains simtels files, it's tree structure is duplicated into the working dir
+    and nfiles are symlinked there
+    """
+    base_dir = Path(base_dir)
+    working_dir = Path(working_dir)
+    
+    for root, dirs, files in tqdm(os.walk(base_dir)):
+        simtel_files = [os.path.join(root, file) for file in files if file.endswith('.simtel.gz')]
+        if simtel_files:
+            for file in simtel_files[:nfiles]:
+                target = working_dir.joinpath(Path(file).relative_to(base_dir))
+                target.parent.mkdir(parents=True, exist_ok=True)
+                if not target.exists():
+                    target.symlink_to(file)
+
+
+
+def generate_test_prod5trans80(working_dir='/fefs/aswg/workspace/lstmcpipe/data/mc/',
+                               nfiles=5, path_to_config_file='.', overwrite=True):
+    base_dir = '/fefs/aswg/workspace/lstmcpipe/data/test_data/mc/DL0/20200629_prod5_trans_80/'
+    working_dir = os.path.join(working_dir, 'DL0/20200629_prod5_trans_80/')
+    
+    generate_tree(base_dir, working_dir, nfiles)
+
+    pc = paths_config.PathConfigProd5Trans80(f'test_prod_{date.today()}')
+    pc.base_dir = os.path.join(working_dir, '{data_level}/20200629_prod5_trans_80/{particle}/{zenith}/south_pointing/{prod_id}')
+    pc.generate()
+    pc.save_yml(os.path.join(path_to_config_file, f'test_prod5trans80_{date.today()}.yaml'), overwrite=overwrite)
+
+
+def generate_test_allsky(working_dir='/fefs/aswg/workspace/lstmcpipe/data/mc/',
+                         nfiles=5, path_to_config_file='.', overwrite=True):
+    allsky_base_dir = '/home/georgios.voutsinas/ws/AllSky/'
+    
+    generate_tree(allsky_base_dir, os.path.join(working_dir, 'DL0/AllSky'), nfiles)
+
+    pc = paths_config.PathConfigAllSky(f'test_prod_{date.today()}')
+    pc.base_dir = os.path.join(working_dir, '{data_level}/AllSky/{prod_id}/{particle}/{pointing}')
+    # config training dir are replaced with local ones
+    pc.training_dir = os.path.join(working_dir, pc.training_dir.replace(allsky_base_dir, 'DL0/AllSky/'))
+    pc.testing_dir = os.path.join(working_dir, pc.testing_dir.replace(allsky_base_dir, 'DL0/AllSky/'))
+    pc.generate()
+    pc.save_yml(os.path.join(path_to_config_file, f'test_AllSky_{date.today()}.yaml'), overwrite=overwrite)
+
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Generate test tree')
+
+    parser.add_argument('prod_type', type=str, 
+                        help='prod5trans80 or allsky')
+    parser.add_argument('--nfiles', type=int, default=5,
+                        help='Number of files')
+    parser.add_argument('--path_config_file', type=Path, default='.',
+                        help='Path to save the corresponding config file')
+    parser.add_argument('--working_dir', type=Path, default='/fefs/aswg/workspace/lstmcpipe/data/mc/',
+                        help='Your working dir where the DL0 tree will be generated, '
+                             '(such as /fefs/aswg/workspace/firstname.surname/data/mc/)')
+
+    args = parser.parse_args()
+    
+    if args.prod_type == 'prod5trans80':
+        generate_test_prod5trans80(args.working_dir, args.nfiles, args.path_config_file)
+    elif args.prod_type == 'allsky':
+        generate_test_allsky(args.working_dir, args.nfiles, args.path_config_file)
+    else:
+        raise NotImplementedError("Unknown prod type")
\ No newline at end of file
diff --git a/lstmcpipe/scripts/script_train_test_splitting.py b/lstmcpipe/scripts/script_train_test_splitting.py
new file mode 100644
index 00000000..a0e29538
--- /dev/null
+++ b/lstmcpipe/scripts/script_train_test_splitting.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+
+import shutil
+import argparse
+from pathlib import Path
+from sklearn.model_selection import train_test_split
+
+parser = argparse.ArgumentParser(
+    description="Script to move a directory and its content after creating the destination"
+    " directory."
+)
+
+parser.add_argument(
+    "--input_dir",
+    "-i",
+    type=Path,
+    dest="input_dir",
+    help="Splitting input dir",
+)
+
+parser.add_argument(
+    "--outdir_test",
+    "--otest",
+    type=Path,
+    dest="outdir_test",
+    help="Output directory for test dataset",
+)
+
+parser.add_argument(
+    "--outdir_train",
+    "--otrain",
+    type=Path,
+    dest="outdir_train",
+    help="Output directory for train dataset",
+)
+
+parser.add_argument(
+    "--ratio",
+    "-r",
+    type=float,
+    dest="ratio",
+    help="Train-test splitting ratio",
+    default=0.5
+)
+
+parser.add_argument(
+    "--log_dir",
+    "-l",
+    type=Path,
+    dest="log_dir",
+    help="Directory to store training and testing filelists"
+)
+
+
+def write_filelist(filelist, outdir, dataset=""):
+    """
+    Write all files within filelist in `{dataset}`.list file at outdir
+
+    Parameters
+    ----------
+    filelist : list
+        list of files to be written
+    outdir : Path
+        Output directory Path
+    dataset : str
+        'training' or 'testing' dataset list
+    """
+    with open(outdir.joinpath(f"{dataset}.list"), "w+") as newfile:
+        for file in filelist:
+            newfile.write(file)
+            newfile.write("\n")
+
+
+def move_files(filelist, outdir):
+    """
+    Move all files within filelist to outdir
+
+    Parameters
+    ----------
+    filelist : list
+        list of files to be written
+    outdir : Path
+        Output directory Path
+    """
+    for file in filelist:
+        shutil.move(file, outdir)
+
+
+def main():
+    args = parser.parse_args()
+
+    file_list = [file.resolve().as_posix() for file in Path(args.input_dir).iterdir()
+                 if file.is_file() and file.name.endswith(".h5")]
+
+    train, test = train_test_split(file_list,
+                                   random_state=42,
+                                   test_size=float(args.ratio),
+                                   shuffle=True
+                                   )
+
+    write_filelist(train, args.log_dir, dataset="training")
+    write_filelist(test, args.log_dir, dataset="testing")
+    move_files(train, args.outdir_train)
+    move_files(test, args.outdir_test)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/lstmcpipe/scripts/tests/test_lstmcpipe_validate_config.py b/lstmcpipe/scripts/tests/test_lstmcpipe_validate_config.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lstmcpipe/stages/mc_train_test_splitting.py b/lstmcpipe/stages/mc_train_test_splitting.py
new file mode 100644
index 00000000..1d15e1de
--- /dev/null
+++ b/lstmcpipe/stages/mc_train_test_splitting.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+
+import os
+import shutil
+import logging
+from pathlib import Path
+from lstmcpipe.workflow_management import save_log_to_file
+
+log = logging.getLogger(__name__)
+
+
+def batch_train_test_splitting(
+        dict_paths,
+        jobids_from_r0dl1,
+        batch_config,
+        logs,
+):
+    log_splitting = {}
+    debug_log = {}
+    jobids_for_merging = []
+
+    log.info("==== START {} ====".format("batch train_test_splitting"))
+
+    for paths in dict_paths:
+        job_logs, jobid = train_test_split(
+            paths["input"],
+            paths["output"],
+            wait_jobid_r0_dl1=jobids_from_r0dl1,
+            batch_configuration=batch_config,
+        )
+
+        log_splitting.update(job_logs)
+        jobids_for_merging.append(jobid)
+        debug_log[jobid] = (
+            f"Train test splitting jobid that depends on : {jobids_from_r0dl1}"
+        )
+
+    jobids_for_merging = ",".join(jobids_for_merging)
+
+    save_log_to_file(log_splitting, logs["log_file"], workflow_step="dl1_to_dl2")
+    save_log_to_file(debug_log, logs["debug_file"], workflow_step="dl1_to_dl2")
+
+    log.info("==== END {} ====".format("batch train_test_splitting"))
+
+    return jobids_for_merging
+
+
+def train_test_split(
+        input_dir,
+        output_dirs,
+        batch_configuration,
+        wait_jobid_r0_dl1=None,
+):
+    source_env = batch_configuration["source_environment"]
+    slurm_account = batch_configuration["slurm_account"]
+
+    log_splitting = {}
+
+    input_dir = Path(input_dir).resolve()
+
+    log.info("Splitting files within the {} dir".format(input_dir.as_posix()))
+
+    # create train, test output directories
+    test_dir = Path(output_dirs["test"]).resolve()
+    train_dir = Path(output_dirs["train"]).resolve()
+    for direct in [test_dir, train_dir]:
+        if direct.exists() and any(direct.iterdir()):
+            shutil.rmtree(direct)
+    train_dir.mkdir(exist_ok=True, parents=True)
+    test_dir.mkdir(exist_ok=True, parents=True)
+
+    # tt ratio
+    if "ratio" not in output_dirs:
+        ratio = 0.5
+    else:
+        ratio = output_dirs["ratio"]
+
+    cmd = f"{source_env} lstmcpipe_train_test_split -i {input_dir} --otest {test_dir}" \
+          f" --otrain {train_dir} -r {ratio} -l {test_dir.parent}"
+
+    # TODO check these dirs
+    jobe = Path(input_dir).joinpath(input_dir, "split_tt_%j.e")
+    jobo = Path(input_dir).joinpath(input_dir, "split_tt_%j.o")
+
+    batch_cmd = "sbatch --parsable -p short"
+    if slurm_account != "":
+        batch_cmd += f" -A {slurm_account}"
+    if wait_jobid_r0_dl1 is not None:
+        batch_cmd += f" --dependency=afterok:{wait_jobid_r0_dl1}"
+    batch_cmd += f' -J splitting_train_test -e {jobe} -o {jobo} --wrap="{cmd}"'
+
+    jobid_split = os.popen(batch_cmd).read().strip("\n")
+    log_splitting.update({jobid_split: batch_cmd})
+
+    return log_splitting, jobid_split
+
+
+def check_empty_dir(directory):
+    """
+    Check if a directory is empty. If not, erase all its content.
+
+    Parameters
+    ----------
+    directory : Path
+        Path to dir to check
+    """
+    if any(directory.iterdir()):
+        for item in directory.iterdir():
+            item.unlink()
+    else:
+        pass
diff --git a/lstmcpipe/tests/test_workflow_management.py b/lstmcpipe/tests/test_workflow_management.py
index 4efdc587..6be0f95c 100644
--- a/lstmcpipe/tests/test_workflow_management.py
+++ b/lstmcpipe/tests/test_workflow_management.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import yaml
+from ruamel.yaml import YAML
 import pytest
 from pathlib import Path
 
@@ -16,7 +16,7 @@ def test_save_log_to_file():
 
     assert outfile_yml.exists()
     with open(outfile_yml) as f:
-        log = yaml.safe_load(f)
+        log = YAML().load(f)
 
     assert "NoKEY" in log.keys()
     assert isinstance(log["NoKEY"], dict)
@@ -48,34 +48,3 @@ def create_fake_dl1_structure(tmp_path):
                     pass
 
     return fake_dl1_dir.as_posix()
-
-
-def test_create_dl1_filenames_dict(create_fake_dl1_structure):
-    from ..workflow_management import create_dl1_filenames_dict
-
-    dl1_dirs_gammas = create_dl1_filenames_dict(create_fake_dl1_structure, ['gamma'], ["off0.0deg", "off0.4deg"])
-
-    assert isinstance(dl1_dirs_gammas, dict)
-    print(dl1_dirs_gammas)
-    assert all(particle in dl1_dirs_gammas for particle in ["gamma_off0.0deg", "gamma_off0.4deg"])
-
-    for key, value in dl1_dirs_gammas.items():
-        assert all(sub_key in dl1_dirs_gammas[key] for sub_key in ["training", "testing"])
-        for ik in dl1_dirs_gammas[key]:
-            if ik == "training":
-                assert dl1_dirs_gammas[key][ik]["train_path_and_outname_dl1"].endswith("training.h5")
-            else:
-                assert dl1_dirs_gammas[key][ik]["test_path_and_outname_dl1"].endswith("testing.h5")
-
-    # Rest of particles
-    dl1_dirs_rest = create_dl1_filenames_dict(create_fake_dl1_structure,
-                                              ["electron", "gamma-diffuse", "proton"])
-    assert isinstance(dl1_dirs_rest, dict)
-    assert all(particle in dl1_dirs_rest for particle in ["gamma-diffuse", "proton", "electron"])
-    for key, value in dl1_dirs_rest.items():
-        assert all(sub_key in dl1_dirs_rest[key] for sub_key in ["training", "testing"])
-        for ik in dl1_dirs_rest[key]:
-            if ik == "training":
-                assert dl1_dirs_rest[key][ik]["train_path_and_outname_dl1"].endswith("training.h5")
-            else:
-                assert dl1_dirs_rest[key][ik]["test_path_and_outname_dl1"].endswith("testing.h5")