Repository: cta-observatory/lstmcpipe (language: python)

Issue:
Refactor bash files
Substitute core bash files by python scripts. 
Renaming and removing files, and create corresponding entry points.
Cleanup config code
Prerequisites: #123

Simplifies some code and adds unit tests.
There is one small change in behaviour:
If you want to avoid setting a `prod_id` in the config, you can just omit the key now. I think this is nicer than specifying the key and setting no value.

Everything else stays the same, diff looks huge until #123 is merged as those commits show up as well.


Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index 1b6d5342..2078ebc9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,4 +9,6 @@ build
 .idea
 __pycache__/
 lstmcpipe/__pycache__/
-.coverage
\ No newline at end of file
+.coverage
+# vim swap files
+*.swp
diff --git a/.pylintrc b/.pylintrc
new file mode 100644
index 00000000..d09e823f
--- /dev/null
+++ b/.pylintrc
@@ -0,0 +1,594 @@
+[MASTER]
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-whitelist=
+
+# Specify a score threshold to be exceeded before program exits with error.
+fail-under=10.0
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Add files or directories matching the regex patterns to the blacklist. The
+# regex matches against base names, not paths.
+ignore-patterns=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use.
+jobs=1
+
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
+
+# List of plugins (as comma separated values of python module names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
+confidence=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=print-statement,
+        parameter-unpacking,
+        unpacking-in-except,
+        old-raise-syntax,
+        backtick,
+        long-suffix,
+        old-ne-operator,
+        old-octal-literal,
+        import-star-module-level,
+        non-ascii-bytes-literal,
+        raw-checker-failed,
+        bad-inline-option,
+        locally-disabled,
+        file-ignored,
+        suppressed-message,
+        useless-suppression,
+        deprecated-pragma,
+        use-symbolic-message-instead,
+        apply-builtin,
+        basestring-builtin,
+        buffer-builtin,
+        cmp-builtin,
+        coerce-builtin,
+        execfile-builtin,
+        file-builtin,
+        long-builtin,
+        raw_input-builtin,
+        reduce-builtin,
+        standarderror-builtin,
+        unicode-builtin,
+        xrange-builtin,
+        coerce-method,
+        delslice-method,
+        getslice-method,
+        setslice-method,
+        no-absolute-import,
+        old-division,
+        dict-iter-method,
+        dict-view-method,
+        next-method-called,
+        metaclass-assignment,
+        indexing-exception,
+        raising-string,
+        reload-builtin,
+        oct-method,
+        hex-method,
+        nonzero-method,
+        cmp-method,
+        input-builtin,
+        round-builtin,
+        intern-builtin,
+        unichr-builtin,
+        map-builtin-not-iterating,
+        zip-builtin-not-iterating,
+        range-builtin-not-iterating,
+        filter-builtin-not-iterating,
+        using-cmp-argument,
+        eq-without-hash,
+        div-method,
+        idiv-method,
+        rdiv-method,
+        exception-message-attribute,
+        invalid-str-codec,
+        sys-max-int,
+        bad-python3-import,
+        deprecated-string-function,
+        deprecated-str-translate-call,
+        deprecated-itertools-function,
+        deprecated-types-field,
+        next-method-defined,
+        dict-items-not-iterating,
+        dict-keys-not-iterating,
+        dict-values-not-iterating,
+        deprecated-operator-function,
+        deprecated-urllib-function,
+        xreadlines-attribute,
+        deprecated-sys-function,
+        exception-escape,
+        comprehension-escape,
+        too-many-arguments,
+        too-many-locals
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+[REPORTS]
+
+# Python expression which should return a score less than or equal to 10. You
+# have access to the variables 'error', 'warning', 'refactor', and 'convention'
+# which contain the number of messages in each category, as well as 'statement'
+# which is the total number of statements analyzed. This score is used by the
+# global evaluation report (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+#msg-template=
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Tells whether to display a full report or only the messages.
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit
+
+
+[LOGGING]
+
+# The type of string formatting that logging methods do. `old` means using %
+# formatting, `new` is for `{}` formatting.
+logging-format-style=old
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format.
+logging-modules=logging
+
+
+[SPELLING]
+
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
+
+# Spelling dictionary name. Available dictionaries: none. To make it work,
+# install the python-enchant package.
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains the private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to the private dictionary (see the
+# --spelling-private-dict-file option) instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,
+      XXX,
+      TODO
+
+# Regular expression of note tags to take in consideration.
+#notes-rgx=
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis). It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=astropy.units
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+# List of decorators that change the signature of a decorated function.
+signature-mutators=
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+          _cb
+
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore.
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Maximum number of characters on a single line.
+max-line-length=90
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[SIMILARITIES]
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[BASIC]
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style.
+#argument-rgx=
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style.
+#attr-rgx=
+
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+          bar,
+          baz,
+          toto,
+          tutu,
+          tata
+
+# Bad variable names regexes, separated by a comma. If names match any regex,
+# they will always be refused
+bad-names-rgxs=
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
+
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style.
+#class-attribute-rgx=
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Regular expression matching correct class names. Overrides class-naming-
+# style.
+#class-rgx=
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+           j,
+           k,
+           x,
+           y,
+           n,
+           ex,
+           _
+
+# Good variable names regexes, separated by a comma. If names match any regex,
+# they will always be accepted
+good-names-rgxs=
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
+
+# Regular expression matching correct method names. Overrides method-naming-
+# style.
+#method-rgx=
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Regular expression matching correct module names. Overrides module-naming-
+# style.
+#module-rgx=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
+
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style.
+#variable-rgx=
+
+
+[STRING]
+
+# This flag controls whether inconsistent-quotes generates a warning when the
+# character used as a quote delimiter is used inconsistently within a module.
+check-quote-consistency=no
+
+# This flag controls whether the implicit-str-concat should generate a warning
+# on implicit string concatenation in sequences defined over several lines.
+check-str-concat-over-line-jumps=no
+
+
+[IMPORTS]
+
+# List of modules that can be imported at any level, not just the top level
+# one.
+allow-any-import-level=
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=optparse,tkinter.tix
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled).
+ext-import-graph=
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled).
+import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+# Couples of modules and preferred modules, separated by a comma.
+preferred-modules=
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+                      __new__,
+                      setUp,
+                      __post_init__
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,
+                  _fields,
+                  _replace,
+                  _source,
+                  _make,
+                  _v_attrs, 
+                  _v_attrnames
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=cls
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method.
+max-args=5
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of boolean expressions in an if statement (see R0916).
+max-bool-expr=5
+
+# Maximum number of branch for function / method body.
+max-branches=12
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=9
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body.
+max-returns=6
+
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=1
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "BaseException, Exception".
+overgeneral-exceptions=BaseException,
+                       Exception
diff --git a/environment.yml b/environment.yml
index 9c40e821..356332a4 100644
--- a/environment.yml
+++ b/environment.yml
@@ -14,4 +14,4 @@ dependencies:
 #      - ctaplot
       - ctapipe_io_lst~=0.9.2
       - lstchain==0.7.3
-      - pyirf~=0.4.0
\ No newline at end of file
+      - pyirf~=0.4.0
diff --git a/lstmcpipe/config/pipeline_config.py b/lstmcpipe/config/pipeline_config.py
index 6425b3d4..be5d04dc 100644
--- a/lstmcpipe/config/pipeline_config.py
+++ b/lstmcpipe/config/pipeline_config.py
@@ -1,7 +1,6 @@
 import os
 import yaml
 import calendar
-import lstchain
 import logging
 
 
@@ -28,20 +27,14 @@ def load_config(config_path):
     with open(config_path) as f:
         loaded_config = yaml.safe_load(f)
 
-    valid, error = config_valid(loaded_config)
-    if not valid:
-        raise Exception(error)
+    config_valid(loaded_config)
 
     config = parse_config_and_handle_global_vars(loaded_config)
 
-    log.info(f'\n\n\t ************ - {config["workflow_kind"]} {config["prod_type"]} - WORKFLOW KIND - ************')
+    log.info(f'************ - Configuration for processing of {config["prod_type"]} using the {config["workflow_kind"]} pipeline:- ************')
     log.info(
         'Simtel DL0 files are going to be searched at: '
-        f'{config["DL0_data_dir"].format(str("""{""") + ",".join(config["all_particles"]) + str("""}"""))}'
-    )
-    log.info(
-        'The following directories and all the information within them will be either created or overwritten:'
-        '[subdirectories with a same PROD_ID and analysed the same day]'
+        f'{config["DL0_data_dir"].format("{" + ",".join(config["all_particles"]) + "}")}'
     )
     particle_dirs = [
         "running_analysis_dir",
@@ -49,16 +42,29 @@ def load_config(config_path):
         "DL2_data_dir",
         "analysis_log_dir",
     ]
-    for pd in particle_dirs:
-        log.info(f'\t{config[pd].format(str("""{""") + ",".join(config["all_particles"]) + str("""}"""))}')
-    log.info(f'- {config["IRFs_dir"]}')
-    log.info(f'- {config["model_dir"]}')
-    log.info(f'- PROD_ID to be used: {config["prod_id"]}')
+    log.info(
+        'The following directories and all the information within them will be either created or overwritten:\n - '
+        + '\n - '.join(
+            [config[pd].format(
+                "{" + ",".join(config["all_particles"]) + "}"
+                ) for pd in particle_dirs]
+            )
+        + f'\n - {config["IRFs_dir"]}'
+        + f'\n - {config["model_dir"]}'
+    )
+
+    log.warning('! Subdirectories with the same PROD_ID and analysed the same day will be overwritten !')
+    log.info(f'PROD_ID to be used: {config["prod_id"]}')
 
-    log.info("Stages to be run:")
-    for stage in config['stages_to_run']:
-        log.info(f"- {stage}")
-    log.info(f"- Merging options: No-image argument: {config['merging_no_image']}")
+    log.info(
+            "Stages to be run:\n - "
+            + "\n - ".join(config["stages_to_run"]
+            )
+    )
+    log.info(
+        "Merging options:"
+        f"\n - No-image argument: {config['merging_no_image']}"
+        )
 
     return config
 
@@ -75,8 +81,7 @@ def config_valid(loaded_config):
 
     Returns:
     --------
-    True, None if config is valid
-    False, error message if config is not valid
+    True if config is valid
     """
     # Allowed options
     allowed_workflows = ['hiperta', 'lstchain', 'ctapipe']
@@ -86,20 +91,20 @@ def config_valid(loaded_config):
     # Check allowed cases
     workflow_kind = loaded_config['workflow_kind']
     if workflow_kind not in allowed_workflows:
-        return False, (
+        raise Exception(
             f'Please select an allowed `workflow_kind`: {allowed_workflows}'
         )
 
     prod_type = loaded_config['prod_type']
     if prod_type not in allowed_prods:
-        return False, (
+        raise Exception(
             f'Please selected an allowed production type: {allowed_prods}.'
         )
 
     obs_date = loaded_config['obs_date']
 
     if obs_date not in allowed_obs_date:
-        return False, (
+        raise Exception(
             f'Please select an allowed obs_date: {allowed_workflows}'
         )
 
@@ -108,10 +113,11 @@ def config_valid(loaded_config):
             (prod_type == 'prod3' and obs_date != '20190415')
             or (prod_type == 'prod5' and obs_date == '20190415')
     ):
-        return False,  (
+        raise Exception(
             'This prod_type and obs_date combination is not possible.'
         )
-    return True, None
+    log.debug("Configuration deemed valid")
+    return True
 
 
 def parse_config_and_handle_global_vars(loaded_config):
@@ -151,12 +157,14 @@ def parse_config_and_handle_global_vars(loaded_config):
     t = calendar.datetime.date.today()
     year, month, day = f"{t.year:04d}", f"{t.month:02d}", f"{t.day:02d}"
     if workflow_kind == 'lstchain':
+        import lstchain
         base_prod_id = f'{year}{month}{day}_v{lstchain.__version__}'
     elif workflow_kind == 'ctapipe':
         import ctapipe
         base_prod_id = f'{year}{month}{day}_vctapipe{ctapipe.__version__}'
     elif workflow_kind == 'hiperta':  # RTA
         # TODO parse version from hiPeRTA module
+        import lstchain
         base_prod_id = f'{year}{month}{day}_vRTA300_v{lstchain.__version__}'
 
     # Create the final config structure to be passed to the pipeline
@@ -177,6 +185,8 @@ def parse_config_and_handle_global_vars(loaded_config):
     config['source_environment'] = src_env
     config['all_particles'] = particles
 
+    # 3.1 - Gammas' offsets
+
     # 4 - Stages to be run
     config['stages_to_run'] = stages_to_be_run
     config['merging_no_image'] = merging_options
diff --git a/lstmcpipe/logging.py b/lstmcpipe/logging.py
new file mode 100644
index 00000000..f6266b43
--- /dev/null
+++ b/lstmcpipe/logging.py
@@ -0,0 +1,39 @@
+"""Helpers to setup logging. Copied and adapted from 
+https://github.com/fact-project/aict-tools/blob/ee091fc933110ff008e32df02ee12a97e412e8b3/aict_tools/logging.py
+"""
+
+import logging
+
+
+def setup_logging(logfile=None, verbose=False):
+
+    level = logging.INFO
+    if verbose is True:
+        level = logging.DEBUG
+
+    log = logging.getLogger()
+    log.level = level
+
+    stream_formatter = logging.Formatter(
+            fmt="%(levelname)s\n%(message)s\n",
+            datefmt="%H:%M:%S",
+            )
+    stream_handler  = logging.StreamHandler()
+    stream_handler.setFormatter(stream_formatter)
+    log.addHandler(stream_handler)
+
+    if logfile is not None:
+        file_formatter = logging.Formatter(
+            fmt="%(asctime)s|%(levelname)s|%(name)s|%(message)s",
+            datefmt="%Y-%m-%dT%H:%M:%S",
+        )
+        file_handler = logging.FileHandler(logfile)
+        file_handler.setFormatter(file_formatter)
+        log.addHandler(file_handler)
+        log.debug("Added logging handler with log file {}".format(logfile))
+
+    # numba produces a ton of logs when set to debug and the solution of using enviroment variables seems to be inconsistent
+    # We dont expect to look through thousands of lines of numba logging anytime soon, so it gets ignored explicitly
+    logging.getLogger('numba').setLevel(logging.WARNING)
+
+    return log
diff --git a/lstmcpipe/lstmcpipe_start.py b/lstmcpipe/lstmcpipe_start.py
index 34696a9f..d7b08625 100755
--- a/lstmcpipe/lstmcpipe_start.py
+++ b/lstmcpipe/lstmcpipe_start.py
@@ -19,21 +19,24 @@
 from os.path import abspath
 import logging
 from lstmcpipe.io.data_management import query_continue
-from lstmcpipe.workflow_management import (
+from lstmcpipe.stages import(
     batch_r0_to_dl1,
     batch_merge_and_copy_dl1,
     batch_train_pipe,
     batch_dl1_to_dl2,
     batch_dl2_to_irfs,
     batch_dl2_to_sensitivity,
+    batch_plot_rf_features,
+)
+from lstmcpipe.workflow_management import (
     save_log_to_file,
     create_dict_with_dl1_filenames,
     batch_mc_production_check,
     create_log_files,
     update_scancel_file,
-    batch_plot_rf_features
 )
 from lstmcpipe.config import load_config
+from lstmcpipe.logging import setup_logging
 
 
 parser = argparse.ArgumentParser(description="MC R0 to DL3 full pipeline")
@@ -79,6 +82,14 @@
     action='store_true', 
     help='print debug messages to stderr'
 )        
+parser.add_argument(
+        '--log-file',
+        action='store',
+        type=str,
+        dest='log_file',
+        help='Optional log file. This is independent of the slurm job logs and only handles lstmcpipe logging',
+        default=None,
+        )
 
 args = parser.parse_args()
 
@@ -86,13 +97,8 @@
 
 
 def main():
-
-    logging.basicConfig(
-        level=logging.DEBUG if args.debug else logging.INFO,
-        format="%(asctime)s %(name)s %(levelname)-8s %(thread)d %(message)s",
-        datefmt="%Y-%m-%d %H:%M:%S"
-        )
-    log = logging.getLogger("My Module")
+    log = setup_logging(verbose=args.debug, logfile=args.log_file)
+    log.info("Starting lstmcpipe processing script")
     # Read MC production configuration file
     config = load_config(args.config_mc_prod)
     query_continue('Are you sure ?')
@@ -151,7 +157,6 @@ def main():
 
     # 2 STAGE --> Merge,copy and move DL1 files
     if 'merge_and_copy_dl1' in stages_to_run:
-
         log_batch_merge_and_copy, jobs_to_train, jobs_all_dl1_finished, debug_merge = batch_merge_and_copy_dl1(
             running_analysis_dir,
             log_batch_r0_dl1,
@@ -278,6 +283,7 @@ def main():
 
     save_log_to_file(debug_mc_check, debug_file, log_format='yml', workflow_step='check_full_workflow')
     update_scancel_file(scancel_file, jobid_check)
+    log.info("Finished lstmcpipe processing script. All jobs have been submitted")
 
 
 if __name__ == '__main__':
diff --git a/lstmcpipe/scripts/script_batch_filelist_cta.py b/lstmcpipe/scripts/script_batch_filelist_cta.py
index 792a36fb..f7e233f5 100644
--- a/lstmcpipe/scripts/script_batch_filelist_cta.py
+++ b/lstmcpipe/scripts/script_batch_filelist_cta.py
@@ -3,6 +3,7 @@
 import argparse
 import subprocess
 from os.path import join, basename
+from os import environ
 
 
 def main():
@@ -11,7 +12,8 @@ def main():
     parser.add_argument('--file_list', '-f', type=str,
                         dest='file_list',
                         help='Path to file containing list of DL0 simtel files to be processed.',
-                        required=True)
+                        required=True,
+                        nargs="+",)
     parser.add_argument('--output_dir', '-o', type=str,
                         dest='output_dir',
                         help='ctapipe-stage1 output directory argument.',
@@ -22,24 +24,25 @@ def main():
                         required=True)
     args = parser.parse_args()
 
-    with open(args.file_list, 'r') as filelist:
+    task_id = int(environ["SLURM_ARRAY_TASK_ID"])
+    file_for_this_job =  args.file_list[task_id]
+    print("Using file: ", file_for_this_job)
+    
+    with open(file_for_this_job, 'r') as filelist:
         for file in filelist:
-
             file = file.strip('\n')
 
+            # ctapipe takes the output filename
+            # so we need to construct it first
             output = join(
                 args.output_dir,
                 basename(file.replace('.simtel.gz', '.dl1.h5'))
             )
 
-            cmd = [f'ctapipe-stage1',
+            cmd = ['ctapipe-stage1',
                    f'--input={file}',
                    f'--output={output}'
                    ]
             if args.config_file:
                 cmd.append('--config={}'.format(args.config_file))
             subprocess.run(cmd)
-
-
-if __name__ == '__main__':
-    main()
diff --git a/lstmcpipe/scripts/script_batch_filelist_lst.py b/lstmcpipe/scripts/script_batch_filelist_lst.py
index 0e60ce89..5cb27084 100644
--- a/lstmcpipe/scripts/script_batch_filelist_lst.py
+++ b/lstmcpipe/scripts/script_batch_filelist_lst.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 import argparse
+from os import environ
 import subprocess
 
 
@@ -10,7 +11,8 @@ def main():
     parser.add_argument('--file_list', '-f', type=str,
                         dest='file_list',
                         help='Path to file containing list of DL0 lstchain file to be processed.',
-                        required=True)
+                        required=True,
+                        nargs="+",)
     parser.add_argument('--output_dir', '-o', type=str,
                         dest='output_dir',
                         help='lstchain_mc_r0_to_dl1 output directory argument.',
@@ -21,17 +23,22 @@ def main():
                         required=True)
     args = parser.parse_args()
 
-    with open(args.file_list, 'r') as filelist:
-        for file in filelist:
+    task_id = int(environ["SLURM_ARRAY_TASK_ID"])
+    file_for_this_job =  args.file_list[task_id]
+    print("Using file: ", file_for_this_job)
 
+    # lstchain takes the output dir and constructs filenanmes itself
+    with open(file_for_this_job, 'r') as filelist:
+        for file in filelist:
             file = file.strip('\n')
 
-            cmd = [f'lstchain_mc_r0_to_dl1',
+            cmd = ['lstchain_mc_r0_to_dl1',
                    f'--input-file={file}',
                    f'--output-dir={args.output_dir}'
                    ]
             if args.config_file:
                 cmd.append('--config={}'.format(args.config_file))
+
             subprocess.run(cmd)
 
 
diff --git a/lstmcpipe/scripts/script_batch_filelist_rta.py b/lstmcpipe/scripts/script_batch_filelist_rta.py
index bdb399fb..054cb74c 100644
--- a/lstmcpipe/scripts/script_batch_filelist_rta.py
+++ b/lstmcpipe/scripts/script_batch_filelist_rta.py
@@ -3,6 +3,7 @@
 import argparse
 import subprocess
 from distutils.util import strtobool
+import os
 
 
 def main():
@@ -11,7 +12,8 @@ def main():
     parser.add_argument('--file_list', '-f', type=str,
                         dest='file_list',
                         help='Path to file containing list of R0 lstchain file to be processed.',
-                        required=True)
+                        required=True,
+                        nargs="+",)
     parser.add_argument('--output_dir', '-o', type=str,
                         dest='output_dir',
                         help='hiperta_r0_to_dl1 output directory argument.',
@@ -34,19 +36,23 @@ def main():
                         )
     args = parser.parse_args()
 
-    with open(args.file_list, 'r') as filelist:
-        for file in filelist:
+    task_id = int(environ["SLURM_ARRAY_TASK_ID"])
+    file_for_this_job =  args.file_list[task_id]
+    print("Using file: ", file_for_this_job)
 
+    with open(file_for_this_job, 'r') as filelist:
+        for file in filelist:
             file = file.strip('\n')
 
-            cmd = [f'lstmcpipe_hiperta_r0_to_dl1lstchain',
-                   f'-i={file}',
-                   f'-o={args.output_dir}',
-                   f'-k={args.keep_file}',
-                   f'-d={args.debug_mode}'
+            cmd = ['lstmcpipe_hiperta_r0_to_dl1lstchain',
+                   f'-i {file}',
+                   f'-o {args.output_dir}',
+                   f'-k {args.keep_file}',
+                   f'-d {args.debug_mode}'
                    ]
             if args.config_file:
                 cmd.append('--config={}'.format(args.config_file))
+
             subprocess.run(cmd)
 
 
diff --git a/lstmcpipe/scripts/script_dl2_to_sensitivity.py b/lstmcpipe/scripts/script_dl2_to_sensitivity.py
index de124dba..87a306c2 100755
--- a/lstmcpipe/scripts/script_dl2_to_sensitivity.py
+++ b/lstmcpipe/scripts/script_dl2_to_sensitivity.py
@@ -147,7 +147,7 @@ def main():
     logging.getLogger("pyirf").setLevel(logging.DEBUG)
 
     for particle_type, p in particles.items():
-        log.info(f"Simulated {particle_type.title()} Events:")
+        log.info("Simulated Events: {}".format(particle_type.title()))
         p["events"], p["simulation_info"] = read_mc_dl2_to_QTable(p["file"])
         # p['events'] = filter_events(p['events'], filters)
 
@@ -180,7 +180,7 @@ def main():
     )
 
     INITIAL_GH_CUT = np.quantile(gammas['gh_score'], (1 - INITIAL_GH_CUT_EFFICENCY))
-    log.info(f"Using fixed G/H cut of {INITIAL_GH_CUT} to calculate theta cuts")
+    log.info("Using fixed G/H cut of {} to calculate theta cuts".format(INITIAL_GH_CUT))
 
     # event display uses much finer bins for the theta cut than
     # for the sensitivity
diff --git a/lstmcpipe/stages/__init__.py b/lstmcpipe/stages/__init__.py
index 9bc95166..c144870e 100644
--- a/lstmcpipe/stages/__init__.py
+++ b/lstmcpipe/stages/__init__.py
@@ -1,15 +1,22 @@
-from .mc_r0_to_dl1 import r0_to_dl1
-from .mc_merge_and_copy_dl1 import merge_dl1
-from .mc_train import train_pipe
-from .mc_dl1_to_dl2 import dl1_to_dl2
-from .mc_dl2_to_irfs import dl2_to_irfs
-from .mc_dl2_to_sensitivity import dl2_to_sensitivity
+from .mc_r0_to_dl1 import r0_to_dl1, batch_r0_to_dl1
+from .mc_merge_and_copy_dl1 import merge_dl1, batch_merge_and_copy_dl1 
+from .mc_train import train_pipe, batch_train_pipe, batch_plot_rf_features
+from .mc_dl1_to_dl2 import dl1_to_dl2, batch_dl1_to_dl2
+from .mc_dl2_to_irfs import dl2_to_irfs, batch_dl2_to_irfs
+from .mc_dl2_to_sensitivity import dl2_to_sensitivity, batch_dl2_to_sensitivity
 
 __all__ = [
     "r0_to_dl1",
+    "batch_r0_to_dl1",
     "merge_dl1",
+    "batch_merge_and_copy_dl1",
     "train_pipe",
+    "batch_train_pipe",
+    "batch_plot_rf_features",
     "dl1_to_dl2",
+    "batch_dl1_to_dl2",
     "dl2_to_irfs",
-    "dl2_to_sensitivity"
+    "batch_dl2_to_irfs",
+    "dl2_to_sensitivity",
+    "batch_dl2_to_sensitivity",
 ]
diff --git a/lstmcpipe/stages/mc_dl1_to_dl2.py b/lstmcpipe/stages/mc_dl1_to_dl2.py
index f647d928..ad7427a3 100755
--- a/lstmcpipe/stages/mc_dl1_to_dl2.py
+++ b/lstmcpipe/stages/mc_dl1_to_dl2.py
@@ -6,11 +6,111 @@
 
 import os
 import shutil
+import logging
+import time
 from lstmcpipe.io.data_management import (
     check_and_make_dir_without_verification
 )
 
 
+log = logging.getLogger(__name__)
+
+
+def batch_dl1_to_dl2(dl1_directory, path_to_models, config_file, jobid_from_training, jobids_from_merge,
+                     dict_with_dl1_paths, particles_loop, source_env, gamma_offsets=None):
+    """
+    Function to batch the dl1_to_dl2 stage once the lstchain train_pipe batched jobs have finished.
+
+    Parameters
+    ----------
+    dl1_directory : str
+        Path to the dl1 directory
+    path_to_models : str
+        Path to the model directory - should be taken from train_pipe
+    config_file : str
+        Path to a configuration file. If none is given, a standard configuration is applied
+    jobid_from_training : str
+        string containing the jobid from the jobs batched in the train_pipe stage, to be passed to the
+        dl1_to_dl2 function (as a slurm dependency)
+    jobids_from_merge : str
+        string containing the jobid from the jobs batched in the merge_and_copy_dl1 stage,
+        to be passed to the dl1_to_dl2 function (as a slurm dependency)
+    dict_with_dl1_paths : dict
+        Indeed the log of the merge_and_copy stage, where the final names of the dl1 files were stored
+    particles_loop : list
+        list with the particles to be processed. Takes the global variable ALL_PARTICLES
+    source_env : str
+        source environment to select the desired conda environment to run train_pipe and dl1_to_dl2 stages
+    gamma_offsets : list
+        list off gamma offsets
+
+    Returns
+    -------
+    log_batch_dl1_to_dl2 : dict
+        Dictionary containing the log of the batched dl1_to_dl2 jobs
+    jobid_4_dl2_to_dl3 : str
+        string containing the jobids to be passed to the next stage of the workflow (as a slurm dependency)
+    debug_log : dict
+        Debug and summary purposes
+
+    """
+
+    log_dl1_to_dl2 = {}
+    jobid_for_dl2_to_dl3 = []
+    debug_log = {}
+
+    log.info("==== START {} ==== \n".format('batch dl1_to_dl2_workflow'))
+    time.sleep(1)
+
+    for particle in particles_loop:
+        if particle == 'gamma' and gamma_offsets is not None:
+            for off in gamma_offsets:
+                gamma_dl1_directory = os.path.join(dl1_directory, off)
+                _particle = particle + '_' + off
+
+                job_logs, jobid = dl1_to_dl2(
+                    gamma_dl1_directory.format(particle),
+                    path_models=path_to_models,
+                    config_file=config_file,
+                    particle=_particle,
+                    wait_jobid_train_pipe=jobid_from_training,
+                    wait_jobids_merge=jobids_from_merge,
+                    dictionary_with_dl1_paths=dict_with_dl1_paths,
+                    source_environment=source_env
+                )
+
+                log_dl1_to_dl2.update(job_logs)
+                jobid_for_dl2_to_dl3.append(jobid)
+
+                debug_log[jobid] = f'{_particle} job from dl1_to_dl2 that depends both on : {jobid_from_training} ' \
+                                   f'training jobs AND from {jobids_from_merge} merge_and_copy_dl1 jobs'
+        else:
+            _particle = particle
+            job_logs, jobid = dl1_to_dl2(
+                dl1_directory.format(particle),
+                path_models=path_to_models,
+                config_file=config_file,
+                particle=_particle,
+                wait_jobid_train_pipe=jobid_from_training,
+                wait_jobids_merge=jobids_from_merge,
+                dictionary_with_dl1_paths=dict_with_dl1_paths,
+                source_environment=source_env
+            )
+
+            log_dl1_to_dl2.update(job_logs)
+            jobid_for_dl2_to_dl3.append(jobid)
+
+            debug_log[jobid] = f'{_particle} job from dl1_to_dl2 that depends both on : {jobid_from_training} ' \
+                               f'training jobs AND from {jobids_from_merge} merge_and_copy_dl1 jobs'
+
+    jobid_4_dl2_to_dl3 = ','.join(jobid_for_dl2_to_dl3)
+
+    log.info("==== END {} ====".format('batch dl1_to_dl2_workflow'))
+
+    return log_dl1_to_dl2, jobid_4_dl2_to_dl3, debug_log
+
+
+
 def dl1_to_dl2(input_dir, path_models, config_file,  particle, wait_jobid_train_pipe=None,
                wait_jobids_merge=None, dictionary_with_dl1_paths=None, source_environment=None):
     """
@@ -51,9 +151,10 @@ def dl1_to_dl2(input_dir, path_models, config_file,  particle, wait_jobid_train_
     """
 
     output_dir = input_dir.replace('DL1', 'DL2')
+    log.info("Working on DL1 files in {}".format(input_dir))
 
     check_and_make_dir_without_verification(output_dir)
-    print(f"\tOutput dir {particle}: {output_dir}")
+    log.info("Output dir {}: {}".format(particle, output_dir))
 
     log_dl1_to_dl2 = {particle: {}}
 
diff --git a/lstmcpipe/stages/mc_dl2_to_irfs.py b/lstmcpipe/stages/mc_dl2_to_irfs.py
index fca2422a..5acfc5fb 100755
--- a/lstmcpipe/stages/mc_dl2_to_irfs.py
+++ b/lstmcpipe/stages/mc_dl2_to_irfs.py
@@ -11,9 +11,95 @@
 import os
 import glob
 import shutil
+import logging
+import time
 from lstmcpipe.io.data_management import check_and_make_dir_without_verification
 
 
+log = logging.getLogger(__name__)
+
+
+def batch_dl2_to_irfs(dl2_directory, loop_particles, offset_gammas, config_file, job_ids_from_dl1_dl2,
+                      log_from_dl1_dl2, source_env, prod_id):
+    """
+    Batches the dl2_to_irfs stage (lstchain lstchain_create_irf_files script) once the dl1_to_dl2 stage had finished.
+
+    Parameters
+    ----------
+    dl2_directory: str
+        Base path to DL2 directory to be formatted with particle type
+    config_file: str
+        Path to lstchain-like config file
+    loop_particles: list
+        list with particles to be processed.
+    offset_gammas: list
+        list off gamma offsets
+    job_ids_from_dl1_dl2: str
+        Comma-separated string with the job ids from the dl1_to_dl2 stage to be used as a slurm dependency
+        to schedule the current stage
+    source_env: str
+        source environment to select the desired conda environment (source .bnashrc + conda activate $ENV)
+    log_from_dl1_dl2: dict
+        Dictionary from dl1_to_dl2 stage with particle path information
+    prod_id: str
+        String with prod_id prefix to complete 'file-naming'
+
+    Returns
+    -------
+    log_batch_dl2_to_irfs: dict
+    jobs_from_dl2_irf: str
+    debug_dl2_to_irfs: dict
+    """
+    log.info("==== START {} ====".format('batch mc_dl2_to_irfs'))
+    time.sleep(1)
+
+    debug_log = {}
+    jobid_for_check = []
+    log_dl2_to_irfs = {}
+
+    for off in offset_gammas:
+
+        job_logs, jobid = dl2_to_irfs(
+            dl2_directory,
+            config_file=config_file,
+            log_from_dl1_dl2=log_from_dl1_dl2,
+            irf_point_like=True,
+            irf_gamma_offset=off,
+            source_env=source_env,
+            wait_jobs_dl1dl2=job_ids_from_dl1_dl2,
+            prod_id=prod_id
+        )
+
+        jobid_for_check.append(jobid)
+        log_dl2_to_irfs[f'gamma_{off}'] = job_logs
+        debug_log[jobid] = f'Gamma_{off} job_id from the dl2_to_irfs stage that depends of the dl1_to_dl2 stage ' \
+                           f'job_ids; {job_ids_from_dl1_dl2}'
+
+    if 'gamma-diffuse' in loop_particles:
+
+        job_logs, jobid = dl2_to_irfs(
+            dl2_directory,
+            irf_point_like=False,
+            config_file=config_file,
+            source_env=source_env,
+            log_from_dl1_dl2=log_from_dl1_dl2,
+            wait_jobs_dl1dl2=job_ids_from_dl1_dl2,
+            prod_id=prod_id
+        )
+
+        jobid_for_check.append(jobid)
+        log_dl2_to_irfs[f'gamma-diffuse'] = job_logs
+        debug_log[jobid] = f'Gamma-diffuse job_id from the dl2_to_irfs stage that depends of the dl1_to_dl2 stage ' \
+                           f'job_ids; {job_ids_from_dl1_dl2}'
+
+    jobid_for_check = ','.join(jobid_for_check)
+
+    log.info("==== END {} ====".format('batch mc_dl2_to_irfs'))
+
+    return log_dl2_to_irfs, jobid_for_check, debug_log
+
+
+
 def check_dl2_files(dl2_dir, pointlike, gamma_off):
     """
     Search DL2 testing files for each of the desired particles depending on the IRF configuration (point-like
@@ -56,7 +142,7 @@ def check_dl2_files(dl2_dir, pointlike, gamma_off):
                                        '*testing.h5')
                           )[0]
         else:
-            print(f'DL2 {particle} directory cannot be found or does not exists:\n {particle_dir_dl2}')
+            log.info(f'DL2 {particle} directory cannot be found or does not exists:\n {particle_dir_dl2}')
             exit(-1)
 
     return dl2_particle_paths
@@ -97,7 +183,7 @@ def dl2_to_irfs(dl2_directory, config_file, log_from_dl1_dl2, irf_point_like=Tru
     """
     allowed_gamma_off = ['off0.0deg', 'off0.4deg']
     if irf_gamma_offset not in allowed_gamma_off:
-        print(f'Please select a valid gamma_offset to compute the IRFS: {" or ".join(allowed_gamma_off)}')
+        log.info(f'Please select a valid gamma_offset to compute the IRFS: {" or ".join(allowed_gamma_off)}')
         exit(-1)
 
     if irf_point_like:
@@ -168,7 +254,7 @@ def dl2_to_irfs(dl2_directory, config_file, log_from_dl1_dl2, irf_point_like=Tru
     # if dry_run:
     #     print(cmd)
 
-    print(f'\tOutput dir IRF {irf_kind}: {output_irfs_dir}')
+    log.info(f'Output dir IRF {irf_kind}: {output_irfs_dir}')
 
     check_and_make_dir_without_verification(output_irfs_dir)
 
diff --git a/lstmcpipe/stages/mc_dl2_to_sensitivity.py b/lstmcpipe/stages/mc_dl2_to_sensitivity.py
index 30c79958..b515ae27 100644
--- a/lstmcpipe/stages/mc_dl2_to_sensitivity.py
+++ b/lstmcpipe/stages/mc_dl2_to_sensitivity.py
@@ -1,96 +1,74 @@
 #!/usr/bin/env python
 
 import os
+import logging
+import time
+import subprocess
 
 
-def batch_plot_sensitivity(sensitivity_filename, wait_jobid_dl2_to_sens, gamma_offset, source_env):
-    """
-    Batches the the `plot_irfs` entry point after the computation of the `dl2_to_sensitivity` script
+log = logging.getLogger(__name__)
 
-    Parameters
-    ----------
-    sensitivity_filename: str
-        Path to sensitivity.fits.gz file
-    wait_jobid_dl2_to_sens: str
-        Jobid from dl2_to_sensitivity stage to be used as a slurm dependency
-    gamma_offset: str
-        String to indicate the gamma offset. Either 'off0.0deg' or 'off0.4deg'
-    source_env: str
-        Source environment (source .bashrc + conda activate env) to be used in the slurm cmd
 
-    Returns
-    -------
-    log: dict
-        Dictionary with job_id-slurm command key-value pair used for logging
-    job_id: str
-        String with single job_id batched by the lstmcpipe_plot_irfs entry point
-
-    """
-    cmd = f'lstmcpipe_plot_irfs -f {sensitivity_filename} -o {sensitivity_filename.replace(".fits.gz", ".png")}'
-
-    jobe = os.path.join(os.path.abspath(os.path.dirname(sensitivity_filename)),
-                        f'job_plot_sensitivity_gamma_{gamma_offset}.e')
-    jobo = os.path.join(os.path.abspath(os.path.dirname(sensitivity_filename)),
-                        f'job_plot_sensitivity_gamma_{gamma_offset}.o')
-
-    job_name = gamma_offset.replace(".", "").replace("off", "").replace("deg", "")
-    base_cmd = f'sbatch --parsable -p short --dependency=afterok:{wait_jobid_dl2_to_sens} -e {jobe} -o {jobo}' \
-               f' -J {job_name}_sens_plot --wrap="export MPLBACKEND=Agg; {source_env} {cmd}"'
-
-    job_id = os.popen(base_cmd).read().strip('\n')
-    log = {job_id: base_cmd}
-
-    return log, job_id
-
-
-def batch_dl2_to_sensitivity(gamma_file, proton_file, electron_file, gamma_offset, output_directory, output_filename,
-                             source_env, wait_jobs_dl1dl2):
+def batch_dl2_to_sensitivity(dl2_directory, offset_gammas, job_ids_from_dl1_dl2, log_from_dl1_dl2, source_env, prod_id):
     """
-    Batches the dl2_to_sensitivity slurm script with all the needed args
+    Batches the dl2_to_sensitivity stage (`stages.script_dl2_to_sensitivity` based in the pyIRF iib) once the
+    dl1_to_dl2 stage had finished.
 
     Parameters
     ----------
-    gamma_file: str
-        Path to DL2 gamma test file
-    proton_file: str
-        Path to DL2 proton test file
-    electron_file: str
-        Path to DL2 electron test file
-    gamma_offset: str
-        String to indicate the gamma offset. Either 'off0.0deg' or 'off0.4deg'
-    output_directory: str
-        Absolute path with output directory
-    output_filename: str
-        Output filename
+    dl2_directory: str
+        Base path to DL2 directory to be formatted with particle type
+    offset_gammas: list
+        list off gamma offsets
+    job_ids_from_dl1_dl2: str
+        Comma-separated string with the job ids from the dl1_to_dl2 stage to be used as a slurm dependency
+        to schedule the current stage
+    log_from_dl1_dl2: dict
+        Dictionary from dl1_to_dl2 stage with particle path information
     source_env: str
-        Source environment (source .bashrc + conda activate env) to be used in the slurm cmd
-    wait_jobs_dl1dl2: str
-        Comma-separated string with the jobs (dependency) to wait for before launching the cmd
+        source environment to select the desired conda environment (source .bashrc + conda activate $ENV)
+    prod_id: str
+        String with prod_id prefix to complete 'file-naming'
 
     Returns
     -------
-    log: dict
+    log_dl2_to_sensitivity: dict
         Dictionary with job_id-slurm command key-value pair used for logging
-    job_id: str
-        String with single job_id batched by the dl2_to_sensitivity script
-
+    jobid_for_check: str
+        Comma-separated jobids batched in the current stage
+    debug_log: dict
+        Dictionary with the job-id and stage explanation to be stored in the debug file
     """
+    log.info("==== START {} ====".format('batch mc_dl2_to_sensitivity'))
+    time.sleep(1)
 
-    cmd = f'lstmcpipe_dl2_to_sensitivity -g {gamma_file} -p {proton_file} -e {electron_file} ' \
-          f' -o {output_filename}'
-
-    jobo = os.path.join(output_directory, f'job_dl2_to_sensitivity_gamma_{gamma_offset}.o')
-    jobe = os.path.join(output_directory, f'job_dl2_to_sensitivity_gamma_{gamma_offset}.e')
+    debug_log = {}
+    jobid_for_check = []
+    log_dl2_to_sensitivity = {}
 
-    job_name = gamma_offset.replace(".", "").replace("off", "")
-    base_cmd = f'sbatch --parsable -p short --dependency=afterok:{wait_jobs_dl1dl2} -e {jobe} -o {jobo} ' \
-               f' -J {job_name}_sensitivity --wrap="{source_env} {cmd}"'
+    for off in offset_gammas:
+        job_logs, jobid = dl2_to_sensitivity(dl2_directory,
+                                        log_from_dl1_dl2,
+                                        gamma_offset=off,
+                                        prod_id=prod_id,
+                                        source_env=source_env,
+                                        wait_jobs_dl1_dl2=job_ids_from_dl1_dl2
+                                        )
+        jobid_for_check.append(jobid)
+        log_dl2_to_sensitivity[f'gamma_{off}'] = job_logs
+        debug_log[jobid] = f'Gamma_{off} job_ids from the dl2_to_sensitivity stage and the plot_irfs script that ' \
+                           f'depends on the dl1_to_dl2 stage job_ids; {job_ids_from_dl1_dl2}'
+        log.info("Jobs for gamma offset {} have been submitted".format(off))
+        log.debug(
+            f'Gamma_{off} job_ids from the dl2_to_sensitivity stage and the plot_irfs script that ' \
+            f'depends on the dl1_to_dl2 stage job_ids; {job_ids_from_dl1_dl2}'
+        )
 
-    job_id = os.popen(base_cmd).read().strip('\n')
+    jobid_for_check = ','.join(jobid_for_check)
 
-    log = {job_id: base_cmd}
+    log.info("==== END {} ====".format('batch mc_dl2_to_sensitivity'))
 
-    return log, job_id
+    return log_dl2_to_sensitivity, jobid_for_check, debug_log
 
 
 def compose_sensitivity_outdir(dl2_dir, gamma_offset):
@@ -113,7 +91,7 @@ def compose_sensitivity_outdir(dl2_dir, gamma_offset):
 
     allowed_gamma_off = ['off0.0deg', 'off0.4deg']
     if gamma_offset not in allowed_gamma_off:
-        print(f'Please select a valid gamma_offset to compute the IRFS: {" or ".join(allowed_gamma_off)}')
+        log.info(f'Please select a valid gamma_offset to compute the IRFS: {" or ".join(allowed_gamma_off)}')
         exit(-1)
 
     output_sensitivity_dir = os.path.join(dl2_dir.replace('/DL2/', '/IRF/').replace('/{}/', '/'),
@@ -208,6 +186,7 @@ def dl2_to_sensitivity(dl2_dir, log_from_dl1_dl2, gamma_offset='off0.0deg', prod
     """
     log_dl2_to_sensitivity = {}
     jobids_dl2_to_sensitivity = []
+    job_name = gamma_offset.replace(".", "").replace("off", "").replace("deg", "")
 
     g_file, p_file, e_file, out_dir, out_file = \
         sensitivity_io(dl2_dir,
@@ -215,29 +194,27 @@ def dl2_to_sensitivity(dl2_dir, log_from_dl1_dl2, gamma_offset='off0.0deg', prod
                        gamma_offset,
                        prod_id)
 
+    # TODO Move the base commands into scripts so that we can use subprocess properly(makes splitting the string easier)
     # create sensitivity files
-    log_dl2_sens, job_id_dl2_sens = \
-        batch_dl2_to_sensitivity(g_file,
-                                 p_file,
-                                 e_file,
-                                 gamma_offset,
-                                 out_dir,
-                                 out_file,
-                                 source_env,
-                                 wait_jobs_dl1_dl2)
-
-    log_dl2_to_sensitivity.update(log_dl2_sens)
+    base_cmd_sens = f'lstmcpipe_dl2_to_sensitivity -g {g_file} -p {p_file} -e {e_file} -o {out_file}'
+    jobo_sens = os.path.join(out_dir, f'job_dl2_to_sensitivity_gamma_{gamma_offset}.o')
+    jobe_sens = os.path.join(out_dir, f'job_dl2_to_sensitivity_gamma_{gamma_offset}.e')
+    cmd_sens = f'sbatch --parsable -p short --dependency=afterok:{wait_jobs_dl1_dl2} -e {jobe_sens} -o {jobo_sens} ' \
+               f' -J {job_name}_sensitivity --wrap="{source_env} {base_cmd_sens}"'
+
+    job_id_dl2_sens = os.popen(cmd_sens).read().strip('\n')
+    log_dl2_to_sensitivity[job_id_dl2_sens] = cmd_sens
     jobids_dl2_to_sensitivity.append(job_id_dl2_sens)
 
     # Create plot from sensitivity files
-    log_plot_sens, job_id_plot_sens = \
-        batch_plot_sensitivity(out_file,
-                               job_id_dl2_sens,
-                               gamma_offset,
-                               source_env)
-
-    log_dl2_to_sensitivity.update(log_plot_sens)
+    base_cmd_plot = f'lstmcpipe_plot_irfs -f {out_file} -o {out_file.replace(".fits.gz", ".png")}'
+    jobe_plot = os.path.join(out_dir, f'job_plot_sensitivity_gamma_{gamma_offset}.e')
+    jobo_plot = os.path.join(out_dir, f'job_plot_sensitivity_gamma_{gamma_offset}.o')
+    cmd_plot = f'sbatch --parsable -p short --dependency=afterok:{job_id_dl2_sens} -e {jobe_plot} -o {jobo_plot}' \
+               f' -J {job_name}_sens_plot --wrap="export MPLBACKEND=Agg; {source_env} {base_cmd_plot}"'
+
+    job_id_plot_sens = os.popen(cmd_plot).read().strip('\n')
+    log_dl2_to_sensitivity[job_id_plot_sens] = cmd_plot
     jobids_dl2_to_sensitivity.append(job_id_plot_sens)
-    jobids_dl2_to_sensitivity = ','.join(jobids_dl2_to_sensitivity)
 
-    return log_dl2_to_sensitivity, jobids_dl2_to_sensitivity
+    return log_dl2_to_sensitivity, ','.join(jobids_dl2_to_sensitivity)
diff --git a/lstmcpipe/stages/mc_merge_and_copy_dl1.py b/lstmcpipe/stages/mc_merge_and_copy_dl1.py
index 3547ca0e..83c13be2 100755
--- a/lstmcpipe/stages/mc_merge_and_copy_dl1.py
+++ b/lstmcpipe/stages/mc_merge_and_copy_dl1.py
@@ -12,6 +12,8 @@
 # 5. move running_dir 
 
 import os
+import logging
+import time
 from lstmcpipe.io.data_management import (
     check_job_logs,
     # read_lines_file,
@@ -19,6 +21,127 @@
 )
 
 
+log = logging.getLogger(__name__)
+
+
+def batch_merge_and_copy_dl1(running_analysis_dir, log_jobs_from_r0_to_dl1, particles_loop, source_env,
+                             smart_merge=False, no_image_flag=True, prod_id=None, gamma_offsets=None, workflow_kind='lstchain'):
+    """
+    Function to batch the onsite_mc_merge_and_copy function once the all the r0_to_dl1 jobs (batched by particle type)
+    have finished.
+
+    Batch 8 merge_and_copy_dl1 jobs ([train, test] x particle) + the move_dl1 and move_dir jobs (2 per particle).
+
+    Parameters
+    ----------
+    running_analysis_dir : str
+        Directory to dl1 files
+    log_jobs_from_r0_to_dl1 : dict
+        dictionary of dictionaries containing the log (jobids organized by particle) from the previous stage
+        (onsite_mc_r0_to_dl1)
+    smart_merge : bool
+        flag to indicate whether the merge of the files should be done with `--smart True` or `--smart False`,
+        controlling the argument of the `lstchain_merge_hdf5_files.py` script (batched in this function).
+    particles_loop : list
+        list with the particles to be processed. Takes the global variable ALL_PARTICLES
+    no_image_flag : bool
+        flag to indicate whether the --no-image argument of the `lstchain_merge_hdf5_files.py` script (batched in
+        this function) should be either True or False.
+    gamma_offsets : list
+        list containing the offset of the gammas
+    prod_id : str
+        prod_id defined in config_MC_prod.yml
+    source_env : str
+        source environment to select the desired conda environment to run the r0/1_to_dl1 stage.
+
+    Returns
+    -------
+    log_merge_and_copy : dict
+        Dictionary containing the log of the batched merge_and_copy_dl1 jobs
+    jobid_4_train : str
+         string containing the jobids to be passed to the next stage of the workflow (as a slurm dependency)
+    all_merge : str
+        string containing all the jobs to indicate that all the dl1 are correctly finished (for dl1_to_dl2)
+    debug_log : dict
+        Debug purposes
+
+    """
+    log_merge_and_copy = {}
+    jobid_4_train = []
+    all_jobs_from_merge_stage = []
+    debug_log = {}
+
+    log.info("==== START {} ====".format('batch merge_and_copy_dl1_workflow'))
+    time.sleep(1)
+    if isinstance(smart_merge, str):
+        merge_flag = 'lst' in smart_merge
+    else:
+        merge_flag = smart_merge
+    log.debug("Merge flag set: {}".format(merge_flag))
+
+    for particle in particles_loop:
+        if particle == 'gamma' and gamma_offsets is not None:
+            for off in gamma_offsets:
+
+                gamma_running_analysis_dir = os.path.join(running_analysis_dir, off)
+                _particle = particle + '_' + off
+
+                job_logs, jobid_mv_all_dl1, jobid_debug = merge_dl1(
+                    gamma_running_analysis_dir.format(particle),
+                    particle2jobs_dict=log_jobs_from_r0_to_dl1,
+                    particle=_particle,
+                    flag_merge=merge_flag,
+                    flag_no_image=no_image_flag,
+                    prod_id=prod_id,
+                    gamma_offset=off,
+                    source_environment=source_env,
+                    workflow_kind=workflow_kind,
+                )
+
+                log_merge_and_copy.update(job_logs)
+                all_jobs_from_merge_stage.append(jobid_debug)
+                if _particle == 'gamma-diffuse' or _particle == 'proton':
+                    jobid_4_train.append(jobid_mv_all_dl1)
+
+                debug_log[jobid_mv_all_dl1] = f'{_particle} merge_and_copy-job - INDEED IT IS JUST PASSED the ' \
+                                              f'move_dl1 jobid - that will be send to the train pipe stage. They ' \
+                                              f'depend on the following {log_jobs_from_r0_to_dl1[_particle]} ' \
+                                              f'r0_t0_dl1 jobs.'
+                debug_log[jobid_debug] = f'Are all the {_particle} jobs that have been launched in merge_and_copy_dl1.'
+
+        else:
+            _particle = particle
+
+            job_logs, jobid_mv_all_dl1, jobid_debug = merge_dl1(
+                running_analysis_dir.format(particle),
+                particle2jobs_dict=log_jobs_from_r0_to_dl1,
+                particle=_particle,
+                flag_merge=merge_flag,
+                flag_no_image=no_image_flag,
+                prod_id=prod_id,
+                source_environment=source_env,
+                workflow_kind=workflow_kind,
+            )
+
+            log_merge_and_copy.update(job_logs)
+            all_jobs_from_merge_stage.append(jobid_debug)
+            if _particle == 'gamma-diffuse' or _particle == 'proton':
+                jobid_4_train.append(jobid_mv_all_dl1)
+
+            debug_log[jobid_mv_all_dl1] = f'{_particle} merge_and_copy-job - INDEED IT IS JUST PASSED the ' \
+                                          f'move_dl1 jobid - that will be send to the train pipe stage. They depend ' \
+                                          f'on the following {log_jobs_from_r0_to_dl1[_particle]} r0_t0_dl1 jobs.'
+            debug_log[jobid_debug] = f'Are all the {_particle} jobs that have been launched in merge_and_copy_dl1.'
+
+    jobid_4_train = ','.join(jobid_4_train)
+    all_jobs_from_merge_stage = ','.join(all_jobs_from_merge_stage)
+
+    log.info("==== END {} ====".format('batch merge_and_copy_dl1_workflow'))
+
+    return log_merge_and_copy, jobid_4_train, all_jobs_from_merge_stage, debug_log
+
+
+
 def compose_batch_command_of_script(source, destination, script, particle, wait_jobs, suffix):
     """
     Creates the slurm command of the 'cmd' script
@@ -131,7 +254,10 @@ def merge_dl1(input_dir, particle2jobs_dict, particle=None, flag_merge=False, fl
 
     JOB_LOGS = os.path.join(input_dir, 'job_logs')
     training_filelist = os.path.join(input_dir, 'training.list')
+    testing_filelist = os.path.join(input_dir, 'testing.list')
     running_DL1_dir = os.path.join(input_dir, 'DL1')
+    DL1_training_dir = os.path.join(running_DL1_dir, 'training')
+    DL1_testing_dir = os.path.join(running_DL1_dir, 'testing')
     final_DL1_dir = input_dir.replace('running_analysis', 'DL1')
     logs_destination_dir = input_dir.replace('running_analysis', 'analysis_logs')
 
@@ -142,7 +268,7 @@ def merge_dl1(input_dir, particle2jobs_dict, particle=None, flag_merge=False, fl
     # just check number of files first:
     # TODO Create check before merging files ?
 
-    print(f"\n\tmerging starts - {particle}")
+    log.info("merging starts - {}".format(particle))
 
     # 3. merge DL1 files
     wait_both_merges = []
@@ -163,7 +289,7 @@ def merge_dl1(input_dir, particle2jobs_dict, particle=None, flag_merge=False, fl
         output_filename += '.h5'
 
         output_filename = os.path.join(running_DL1_dir, output_filename)
-        print(f"\t\tmerge output: {output_filename}")
+        log.info("merge output: {}".format(output_filename))
 
         # After the workflow the files will be moved, will not stay at output_filename
         if set_type == 'training':
@@ -193,7 +319,7 @@ def merge_dl1(input_dir, particle2jobs_dict, particle=None, flag_merge=False, fl
         jobid_merge = os.popen(cmd).read().strip('\n')
         log_merge[particle][set_type][jobid_merge] = cmd
 
-        print(f'\t\tSubmitted batch job {jobid_merge} -- {particle}, {set_type}')
+        log.info(f'Submitted batch job {jobid_merge} -- {particle}, {set_type}')
 
         wait_both_merges.append(jobid_merge)
         return_jobids_debug.append(jobid_merge)
@@ -205,7 +331,7 @@ def merge_dl1(input_dir, particle2jobs_dict, particle=None, flag_merge=False, fl
     # 5 --> copy lstchain config file in final_dir too
     # 6 --> move running_dir as logs
 
-    print(f"\tDL1 files will be moved to {final_DL1_dir}")
+    log.info("DL1 files will be moved to {}".format(final_DL1_dir))
 
     # 4 --> move DL1 files in final place
     wait_both_merges = ','.join(wait_both_merges)
@@ -219,7 +345,7 @@ def merge_dl1(input_dir, particle2jobs_dict, particle=None, flag_merge=False, fl
     jobid_move_dl1 = os.popen(cmd_mv_dl1).read().strip('\n')
     log_merge[particle][set_type][jobid_move_dl1] = cmd_mv_dl1
 
-    print(f'\n\t\tSubmitted batch job {jobid_move_dl1}. It will move dl1 files when {wait_both_merges} finish.')
+    log.info(f'Submitted batch job {jobid_move_dl1}. It will move dl1 files when {wait_both_merges} finish.')
 
     # 5 --> copy lstchain config file in final_dir too
     cmd_cp_conf = compose_batch_command_of_script(input_dir,
@@ -232,7 +358,7 @@ def merge_dl1(input_dir, particle2jobs_dict, particle=None, flag_merge=False, fl
     jobid_copy_conf = os.popen(cmd_cp_conf).read().strip('\n')
     log_merge[particle][set_type][jobid_copy_conf] = cmd_cp_conf
 
-    print(f'\t\tSubmitted batch job {jobid_copy_conf}. It will copy the used config when {jobid_move_dl1} finish.')
+    log.info(f'Submitted batch job {jobid_copy_conf}. It will copy the used config when {jobid_move_dl1} finish.')
 
     # 6 --> move running_dir to final analysis_logs
     cmd_mv_dir = compose_batch_command_of_script(input_dir,
@@ -245,7 +371,7 @@ def merge_dl1(input_dir, particle2jobs_dict, particle=None, flag_merge=False, fl
     jobid_move_log = os.popen(cmd_mv_dir).read().strip('\n')
     log_merge[particle][set_type][jobid_move_log] = cmd_mv_dir
 
-    print(f'\t\tSubmitted batch job {jobid_move_log}. It will move running_dir when {jobid_copy_conf} finish.')
+    log.info(f'Submitted batch job {jobid_move_log}. It will move running_dir when {jobid_copy_conf} finish.')
 
     return_jobids4train.append(jobid_move_dl1)
 
@@ -253,7 +379,7 @@ def merge_dl1(input_dir, particle2jobs_dict, particle=None, flag_merge=False, fl
     return_jobids_debug.append(jobid_move_log)
     return_jobids_debug.append(jobid_copy_conf)
 
-    print(f"\tLOGS will be moved to {logs_destination_dir}")
+    log.info("LOGS will be moved to {}".format(logs_destination_dir))
 
     # Little clarification (it will not be clear in log). These keys are stored here for 2 purposes:
     # 1 - train_pipe recover final dl1 names and path.
diff --git a/lstmcpipe/stages/mc_r0_to_dl1.py b/lstmcpipe/stages/mc_r0_to_dl1.py
index 96083759..5b02d8be 100755
--- a/lstmcpipe/stages/mc_r0_to_dl1.py
+++ b/lstmcpipe/stages/mc_r0_to_dl1.py
@@ -8,7 +8,11 @@
 import os
 import time
 import shutil
-import random
+from numpy.random import default_rng
+from pathlib import Path
+import subprocess
+import logging
+import time
 from lstmcpipe.io.data_management import (
     check_data_path,
     get_input_filelist,
@@ -16,10 +20,97 @@
 )
 
 
-def r0_to_dl1(input_dir, config_file=None, train_test_ratio=0.5, random_seed=42, n_r0_files_per_dl1_job=0,
-              particle=None, prod_id=None, source_environment=None, offset=None, workflow_kind='lstchain', keep_rta_file=False):
+log = logging.getLogger(__name__)
+
+
+def batch_r0_to_dl1(input_dir, conf_file, prod_id, particles_loop, source_env, gamma_offsets=None,
+                    workflow_kind='lstchain'):
+    """
+    Batch the r0_to_dl1 jobs by particle type.
+
+    Parameters
+    ----------
+    input_dir : str
+        Path to the r0/DL0 files
+    conf_file : str
+        Path to a configuration file. If none is given, a standard configuration is applied
+    prod_id : str
+        Production ID. If None, _v00 will be used, indicating an official base production. Default = None.
+    particles_loop : list
+        list with the particles to be processed. Takes the global variable ALL_PARTICLES
+    source_env : str
+        source environment to select the desired conda environment to run the r0/1_to_dl1 stage.
+    gamma_offsets : list
+    workflow_kind: str
+        One of the supported pipelines. Defines the command to be run on r0 files
+
+    Returns
+    -------
+    full_log : dict
+        Dictionary of dictionaries containing the full log of the batched jobs (jobids as keys) as well as the
+        4 more keys (one by particle) with all the jobs associated with each particle.
+    debug_log : dict
+            dictionary containing minimum information - jobids -  for log_reduced.txt
+    all_jobids_from_r0_dl1_stage : str
+        string, separated by commas, containing all the jobids of this stage
+    """
+    full_log = {'log_all_job_ids': {}}
+    debug_log = {}
+    all_jobids_from_r0_dl1_stage = []
+
+    log.info("==== START {} r0 to dl1 processing ====".format(workflow_kind))
+    time.sleep(1)
+
+    for particle in particles_loop:
+        if particle == 'gamma' and gamma_offsets is not None:
+            for off in gamma_offsets:
+                particle_input_dir = os.path.join(input_dir, off).format(particle)
+                _particle = particle + '_' + off
+                job_logs, jobids_by_particle = r0_to_dl1(
+                    particle_input_dir,  # Particle needs to be gamma w/o the off
+                    config_file=conf_file,
+                    particle=_particle,
+                    prod_id=prod_id,
+                    source_environment=source_env,
+                    offset=off,
+                    workflow_kind=workflow_kind,
+                )
+                full_log['log_all_job_ids'].update(job_logs)
+                full_log[_particle] = ','.join(jobids_by_particle)
+                all_jobids_from_r0_dl1_stage.append(full_log[_particle])  # Create a list with particles elements
+
+                for jid in jobids_by_particle:
+                    debug_log[jid] = f'{_particle} job from r0_to_dl1'        
+        else:
+            particle_input_dir = input_dir.format(particle)
+            _particle = particle
+            job_logs, jobids_by_particle = r0_to_dl1(
+                particle_input_dir,  # Particle needs to be gamma w/o the off
+                config_file=conf_file,
+                particle=_particle,
+                prod_id=prod_id,
+                source_environment=source_env,
+                workflow_kind=workflow_kind,
+            )
+            full_log['log_all_job_ids'].update(job_logs)
+            full_log[_particle] = ','.join(jobids_by_particle)
+            all_jobids_from_r0_dl1_stage.append(full_log[_particle])  # Create a list with particles elements
+
+            for jid in jobids_by_particle:
+                debug_log[jid] = f'{_particle} job from r0_to_dl1'
+    all_jobids_from_r0_dl1_stage = ','.join(all_jobids_from_r0_dl1_stage)  # Create a string to be directly passed
+
+    log.info("==== END {} r0 to dl1 processing ====".format(workflow_kind))
+
+    return full_log, debug_log, all_jobids_from_r0_dl1_stage  # ids_by_particle_ok
+
+
+def r0_to_dl1(input_dir, config_file=None, train_test_ratio=0.5, rng=None, n_r0_per_dl1_job=None,
+              particle=None, prod_id=None, source_environment=None, offset=None, workflow_kind='lstchain', keep_rta_file=False,
+              n_jobs_parallel=20):
     """
     R0 to DL1 MC onsite conversion.
+    Organizes files and launches slurm jobs in two slurm arrays.
 
 
     Parameters
@@ -27,46 +118,45 @@ def r0_to_dl1(input_dir, config_file=None, train_test_ratio=0.5, random_seed=42,
     input_dir : str
         path to the files directory to analyse
     config_file :str
-        Path to a configuration file. If none is given, a standard configuration is applied
-    train_test_ratio :int
+        Path to a configuration file. If none is given, the standard configuration of the selected pipeline is applied
+    train_test_ratio: float
         Ratio of training data. Default = 0.5
     random_seed : int
         Random seed for random processes. Default = 42
     n_r0_files_per_dl1_job : int
-        Number of r0 files processed by each r0_to_dl1 batched stage. If set to 0 (Default), see below the `usual
+        Number of r0 files processed by each r0_to_dl1 batched stage. If set to None (Default), see below the `usual
         production` case.n_r0_files_per_dl1_job
 
         If the number of r0 files found in `input_dir` is less than 100, it is consider to be a test on a small
         production. Therefore, the number of r0 files treated per batched stage will be set to 10.
 
-        Usual productions have =>1000 r0 files, in this case, the number of batched jobs will be fixed to 50 (in case
-        of gamma and electrons), 80 for (gamma-diffuse) and 125 to protons. This means that there will be batched a
-        total of 50+50+80+125 = 305 jobs only for the r0_to_dl1 stage. (there are 1k r0 files for gammas (although 2
-        offsets, thus 2k files), 2k r0 files for gd and e- and 5k for protons).
-
-        Default = 0
-
+        Usual productions have =>1000 r0 files, in this case, the number of files per job will be fixed to 50 (in case
+        of gamma and electrons), and 100 for protons. Because of this protons end up in the long queue and other particles
+        are submitted to the short queue.
     particle : str
-        particle type for `flag_full_workflow` = True
+        particle type (gamma/gamma_off/proton/electron). Determines output directory structure, job naming
+        and n_r0_files_per_dl1_job if not set explicitly.
     offset : str
         gamma offset
     prod_id :str
         Production ID. If None, _v00 will be used, indicating an official base production. Default = None.
     source_environment : str
-        path to a .bashrc file to source (can be configurable for custom runs @ mc_r0_to_dl3 script)
-         to activate a certain conda environment.
-         DEFAULT: `source /fefs/aswg/software/virtual_env/.bashrc; conda activate cta`.
+        path to a .bashrc file to source (can be configurable for custom runs @lstmcpipe_start script)
+        and command to activate a certain conda environment.
+        Passed to the core script of the selected pipeline and activated there.
+        Has no effect for hiperta currently
         ! NOTE : train_pipe AND dl1_to_dl2 **MUST** be run with the same environment.
     workflow_kind: str
         One of the supported pipelines. Defines the command to be run on r0 files
     keep_rta_file : bool
         Argument to be passed to the hiperta_r0_to_dl1lstchain script, which runs the hiperta_r0_dl1 and
         re-organiser stage
+    n_jobs_parallel: int
+        Number of jobs to be run at the same time per array.
+
     Returns
     -------
-
-    jobid2log : dict (if flag_full_workflow is True)
-
+    jobid2log : dict
         A dictionary of dictionaries containing the full log information of the script. The first `layer` contains
         only the each jobid that the scripts has batched.
 
@@ -79,16 +169,12 @@ def r0_to_dl1(input_dir, config_file=None, train_test_ratio=0.5, random_seed=42,
                  when the job was send to the cluster
 
              dict[jobid].keys() = ['particle', 'sbatch_command', 'jobe_path', 'jobo_path']
-
-             ****  otherwise : (if flag_full_workflow is False, by default) ****
-            None is returned -- THIS IS APPLIED FOR THE ARGUMENTS SHOWN BELOW TOO
-
     jobids_r0_dl1
-
         A list of all the jobs sent by particle (including test and train set types).
-
     """
 
+    log.info("Starting R0 to DL1 processing for particle {}".format(particle))
+
     PROD_ID = prod_id
 
     if workflow_kind == 'lstchain':
@@ -102,48 +188,49 @@ def r0_to_dl1(input_dir, config_file=None, train_test_ratio=0.5, random_seed=42,
         base_cmd = f'{rta_source_env} lstmcpipe_rta_core_r0_dl1 -k {keep_rta_file} -d False '
         jobtype_id = 'RTA'
     else:
-        print("Please, selected an allowed workflow kind.")
+        log.critical("Please, selected an allowed workflow kind.")
         exit(-1)
 
-    TRAIN_TEST_RATIO = float(train_test_ratio)
-    RANDOM_SEED = random_seed
-    #NFILES_PER_DL1 = n_files_per_dl1
-    #DESIRED_DL1_SIZE_MB = 1000
+    job_name = {'electron': f'e_{jobtype_id}_r0dl1',
+                'gamma': f'g_{jobtype_id}_r0dl1',
+                'gamma-diffuse': f'gd_{jobtype_id}_r0dl1',
+                'proton': f'p_{jobtype_id}_r0dl1',
+                'gamma_off0.0deg': f'g0.0_{jobtype_id}_r0dl1',
+                'gamma_off0.4deg': f'g0.4_{jobtype_id}_r0dl1'
+                }
 
-    #N_R0_PER_DL1_JOB = n_r0_files_per_dl1_job
+
+    TRAIN_TEST_RATIO = float(train_test_ratio)
 
     DL0_DATA_DIR = input_dir
 
     ##############################################################################
 
-    print(f"Working on DL0 files in {DL0_DATA_DIR}")
+    log.info("Working on DL0 files in {}".format(DL0_DATA_DIR))
 
     check_data_path(DL0_DATA_DIR)
 
     raw_files_list = get_input_filelist(DL0_DATA_DIR)
 
     if len(raw_files_list) < 100:
-        N_R0_PER_DL1_JOB = 10
-    elif n_r0_files_per_dl1_job == 0:
-        if 'gamma' in input_dir:
-            N_R0_PER_DL1_JOB = 25
-        elif 'gamma-diffuse' in input_dir or 'electron' in input_dir:
-            N_R0_PER_DL1_JOB = 50
-        elif 'proton' in input_dir:
-            N_R0_PER_DL1_JOB = 125
+        n_r0_per_dl1_job = 10
+    
+    if not n_r0_per_dl1_job:
+        if len(raw_files_list) < 100:
+            n_r0_per_dl1_job = 10
         else:
-            N_R0_PER_DL1_JOB = 50
-    else:
-        N_R0_PER_DL1_JOB = n_r0_files_per_dl1_job
-
-    # if NFILES_PER_DL1 == 0:
-    #     size_dl0 = os.stat(raw_files_list[0]).st_size / 1e6
-    #     reduction_dl0_dl1 = 5
-    #     size_dl1 = size_dl0 / reduction_dl0_dl1
-    #     NFILES_PER_DL1 = max(1, int(DESIRED_DL1_SIZE_MB / size_dl1))
+            if 'gamma' in input_dir:
+                n_r0_per_dl1_job = 25
+            elif 'gamma-diffuse' in input_dir or 'electron' in input_dir:
+                n_r0_per_dl1_job = 50
+            elif 'proton' in input_dir:
+                n_r0_per_dl1_job = 125
+            else:
+                n_r0_per_dl1_job = 50
 
-    random.seed(RANDOM_SEED)
-    random.shuffle(raw_files_list)
+    if rng is None:
+        rng = default_rng()
+    rng.shuffle(raw_files_list)
 
     number_files = len(raw_files_list)
     ntrain = int(number_files * TRAIN_TEST_RATIO)
@@ -152,9 +239,9 @@ def r0_to_dl1(input_dir, config_file=None, train_test_ratio=0.5, random_seed=42,
     training_list = raw_files_list[:ntrain]
     testing_list = raw_files_list[ntrain:]
 
-    print("\t{} raw files".format(number_files))
-    print("\t{} files in training dataset".format(ntrain))
-    print("\t{} files in test dataset".format(ntest))
+    log.info("{} raw R0 files".format(number_files))
+    log.info("{} R0 files in training dataset".format(ntrain))
+    log.info("{} R0 files in test dataset".format(ntest))
 
     with open('training.list', 'w+') as newfile:
         for f in training_list:
@@ -167,7 +254,6 @@ def r0_to_dl1(input_dir, config_file=None, train_test_ratio=0.5, random_seed=42,
             newfile.write('\n')
 
     if 'off' in particle:
-        # join(BASE_PATH, 'DL0', OBS_DATE, '{particle}', ZENITH, POINTING, 'PLACE_4_PROD_ID', GAMMA_OFF)
         DL0_DATA_DIR = DL0_DATA_DIR.split(offset)[0]   # Take out /off0.Xdeg
         RUNNING_DIR = os.path.join(
                 DL0_DATA_DIR.replace(
@@ -189,9 +275,9 @@ def r0_to_dl1(input_dir, config_file=None, train_test_ratio=0.5, random_seed=42,
     # DIR_LISTS_BASE = os.path.join(RUNNING_DIR, 'file_lists')
     # ADD CLEAN QUESTION
 
-    print("\tRUNNING_DIR: \t", RUNNING_DIR)
-    print("\tJOB_LOGS DIR: \t", JOB_LOGS)
-    print("\tDL1 DATA DIR: \t", DL1_DATA_DIR)
+    log.info("RUNNING_DIR: {}".format(RUNNING_DIR))
+    log.info("JOB_LOGS DIR: {}".format(JOB_LOGS))
+    log.info("DL1 DATA DIR: {}".format(DL1_DATA_DIR))
 
     for directory in [RUNNING_DIR, DL1_DATA_DIR, JOB_LOGS]:
         check_and_make_dir_without_verification(directory)
@@ -202,6 +288,7 @@ def r0_to_dl1(input_dir, config_file=None, train_test_ratio=0.5, random_seed=42,
     jobids_r0_dl1 = []
 
     for set_type in 'training', 'testing':
+        log.debug("Generating list for {} step".format(set_type))
         if set_type == 'training':
             list_type = training_list
         else:
@@ -213,69 +300,62 @@ def r0_to_dl1(input_dir, config_file=None, train_test_ratio=0.5, random_seed=42,
         check_and_make_dir_without_verification(dir_lists)
         check_and_make_dir_without_verification(output_dir)
 
-        print("\toutput dir: \t", output_dir)
+        log.info("output dir: {}".format(output_dir))
 
-        number_of_sublists = len(list_type) // N_R0_PER_DL1_JOB + int(len(list_type) % N_R0_PER_DL1_JOB > 0)
+        number_of_sublists = len(list_type) // n_r0_per_dl1_job + int(len(list_type) % n_r0_per_dl1_job > 0)
         for i in range(number_of_sublists):
             output_file = os.path.join(dir_lists, '{}_{}.list'.format(set_type, i))
             with open(output_file, 'w+') as out:
-                for line in list_type[i * N_R0_PER_DL1_JOB:N_R0_PER_DL1_JOB * (i + 1)]:
+                for line in list_type[i * n_r0_per_dl1_job:n_r0_per_dl1_job * (i + 1)]:
                     out.write(line)
                     out.write('\n')
-        print(f'\t{number_of_sublists} files generated for {set_type} list')
+        log.info(f'{number_of_sublists} files generated for {set_type} list')
 
-        # LSTCHAIN #
-        counter = 0
         save_job_ids = []
 
-        for file in os.listdir(dir_lists):
-            if set_type == 'training':
-                jobo = os.path.join(JOB_LOGS, f"job{counter}_train.o")
-                jobe = os.path.join(JOB_LOGS, f"job{counter}_train.e")
-            else:
-                jobo = os.path.join(JOB_LOGS, f"job{counter}_test.o")
-                jobe = os.path.join(JOB_LOGS, f"job{counter}_test.e")
-
-            job_name = {'electron': f'e_{jobtype_id}_r0dl1',
-                        'gamma': f'g_{jobtype_id}_r0dl1',
-                        'gamma-diffuse': f'gd_{jobtype_id}_r0dl1',
-                        'proton': f'p_{jobtype_id}_r0dl1',
-                        'gamma_off0.0deg': f'g0.0_{jobtype_id}_r0dl1',
-                        'gamma_off0.4deg': f'g0.4_{jobtype_id}_r0dl1'
-                        }
-
-            if particle == 'proton':
-                queue = 'long'
-            else:
-                queue = 'short'
+        files = [f.resolve().as_posix() for f in Path(dir_lists).glob("*")]
 
-            cmd = f'sbatch --parsable -p {queue} -J {job_name[particle]} -e {jobe} -o {jobo} ' \
-                  f'--wrap="{base_cmd} -f {os.path.join(dir_lists, file)} -o {output_dir}"'
+        if set_type == 'training':
+            jobo = os.path.join(JOB_LOGS, "job_%A_%a_train.o")
+            jobe = os.path.join(JOB_LOGS, "job_%A_%a_train.e")
+        else:
+            jobo = os.path.join(JOB_LOGS, "job_%A_%a_test.o")
+            jobe = os.path.join(JOB_LOGS, "job_%A_%a_test.e")
+
+        if particle == 'proton':
+            queue = 'long'
+        else:
+            queue = 'short'
 
-            jobid = os.popen(cmd).read().strip('\n')
-            jobids_r0_dl1.append(jobid)
+        slurm_options = f"--array=0-{len(files)-1}%{n_jobs_parallel} "
+        slurm_options += f"-p {queue} "
+        slurm_options += f"-e {jobe} "
+        slurm_options += f"-o {jobo} "
+        slurm_options += f"-J {job_name[particle]} "
 
-            # Fill the dictionaries if IN workflow mode
-            jobid2log[jobid] = {}
-            jobid2log[jobid]['particle'] = particle
-            jobid2log[jobid]['set_type'] = set_type
-            jobid2log[jobid]['jobe_path'] = jobe
-            jobid2log[jobid]['jobo_path'] = jobo
-            jobid2log[jobid]['sbatch_command'] = cmd
+        # start 1 jobarray with all files included. The job selects its file based on its task id
+        cmd = f'sbatch --parsable {slurm_options} --wrap="{base_cmd} -f {" ".join(files)} --output_dir {output_dir}"'
+        log.debug(f'Slurm command to start the jobs: {cmd}')
 
-            # print(f'\t\t{cmd}')
-            # print(f'\t\tSubmitted batch job {jobid}')
-            save_job_ids.append(jobid)
+        jobid = os.popen(cmd).read().strip('\n')
+        log.debug(f'Submitted batch job {jobid}')
+        jobids_r0_dl1.append(jobid)
 
-            counter += 1
+        jobid2log[jobid] = {}
+        jobid2log[jobid]['particle'] = particle
+        jobid2log[jobid]['set_type'] = set_type
+        jobid2log[jobid]['jobe_path'] = jobe
+        jobid2log[jobid]['jobo_path'] = jobo
+        jobid2log[jobid]['sbatch_command'] = cmd
 
-        print(f"\n\t{counter} jobs submitted - {particle} {set_type}. "
-              f"From jobid {save_job_ids[0]} - {save_job_ids[-1]}\n")
-        time.sleep(1)  # Avoid collapsing LP cluster
+        save_job_ids.append(jobid)
 
     # copy config into working dir
     if config_file is not None:
-        shutil.copyfile(config_file, os.path.join(RUNNING_DIR, os.path.basename(config_file)))
+        shutil.copyfile(
+            config_file,
+            os.path.join(RUNNING_DIR, os.path.basename(config_file))
+        )
 
     # save file lists into logs
     shutil.move('testing.list', os.path.join(RUNNING_DIR, 'testing.list'))
diff --git a/lstmcpipe/stages/mc_train.py b/lstmcpipe/stages/mc_train.py
index 5473bdb0..6c095a67 100755
--- a/lstmcpipe/stages/mc_train.py
+++ b/lstmcpipe/stages/mc_train.py
@@ -6,9 +6,110 @@
 
 import os
 import shutil
+import logging
+import time
 from lstmcpipe.io.data_management import check_and_make_dir_without_verification
 
 
+log = logging.getLogger(__name__)
+
+
+def batch_train_pipe(log_from_merge, config_file, jobids_from_merge, source_env):
+    """
+    Function to batch the lstchain train_pipe once the proton and gamma-diffuse merge_and_copy_dl1 batched jobs have
+    finished.
+
+    Parameters
+    ----------
+    log_from_merge : dict
+        dictionary containing the output name and abs path to the DL1 files, derived in merge_and_copy and saved
+        through the log
+    config_file : str
+        Path to a configuration file. If none is given, a standard configuration is applied
+    jobids_from_merge : str
+        string containing the jobids (***ONLY from proton and gamma-diffuse***) from the jobs batched in the
+         merge_and_copy_dl1 stage, to be passed to the train_pipe function (as a slurm dependency)
+    source_env : str
+        source environment to select the desired conda environment to run train_pipe and dl1_to_dl2 stages
+
+    Returns
+    -------
+    log_train : dict
+        Dictionary containing the log of the batched train_pipe jobs
+    jobid_4_dl1_to_dl2 : str
+        string containing the jobid to be passed to the next stage of the workflow (as a slurm dependency).
+        For the next stage, however, it will be needed TRAIN + MERGED jobs
+    model_path : str
+        Path with the model's directory
+    debug_log : dict
+        Debug and summary purposes
+    """
+    debug_log = {}
+
+    log.info("==== START {} ====".format('batch mc_train_workflow'))
+    time.sleep(1)
+
+    gamma_dl1_train_file = log_from_merge['gamma-diffuse']['training']['train_path_and_outname_dl1']
+    proton_dl1_train_file = log_from_merge['proton']['training']['train_path_and_outname_dl1']
+
+    log_train, jobid_4_dl1_to_dl2, model_path = train_pipe(
+        gamma_dl1_train_file,
+        proton_dl1_train_file,
+        config_file=config_file,
+        source_environment=source_env,
+        wait_ids_proton_and_gammas=jobids_from_merge
+    )
+
+    debug_log[jobid_4_dl1_to_dl2] = f'The single jobid from train_pipe that depends of {jobids_from_merge} - merge' \
+                                    f'_and_copy jobids'
+
+    log.info("==== END {} ====".format('batch mc_train_workflow'))
+
+    return log_train, jobid_4_dl1_to_dl2, model_path, debug_log
+
+
+
+def batch_plot_rf_features(dir_models, config_file, source_env, train_jobid):
+    """
+    Batches the plot_model_importance.py script that creates a .png with the RF feature's importance models
+    after the RF are trained.
+    The plot is saved in the same dir in where the modes are stored.
+
+    Parameters
+    ----------
+    dir_models: str
+        Path to model's directory
+    config_file: str
+        Path to lstchain config file
+    source_env: str
+        String containing the .bashrc file to source and the conda env to call
+    train_jobid: str
+        Single jobid from training stage.
+
+    Returns
+    -------
+    log: dict
+        Dictionary with lstmcpipe_plot_models_importance single job id to be passed to debug log.
+    """
+    logs = {}
+    log.info("==== START {} ====".format('batch plot RF features importance'))
+    time.sleep(1)
+    jobe = os.path.join(dir_models, 'job_plot_rf_feat_importance.e')
+    jobo = os.path.join(dir_models, 'job_plot_rf_feat_importance.o')
+
+    base_cmd = f'lstmcpipe_plot_models_importance {dir_models} -cf {config_file}'
+    cmd = f'sbatch --parsable --dependency=afterok:{train_jobid} -e {jobe} -o {jobo} -J RF_importance ' \
+          f' --wrap="export MPLBACKEND=Agg; {source_env} {base_cmd}"'
+    jobid = os.popen(cmd).read().strip('\n')
+
+    logs[jobid] = 'Single job_id to plot RF feature s importance'
+
+    log.info(" Random Forest importance's plot will be saved at: {}".format(dir_models))
+    log.info("==== END {} ====".format('batch plot RF features importance'))
+
+    return logs
+
+
 def train_pipe(gamma_dl1_train_file, proton_dl1_train_file, config_file=None, source_environment=None,
                wait_ids_proton_and_gammas=None):
     """
@@ -55,7 +156,7 @@ def train_pipe(gamma_dl1_train_file, proton_dl1_train_file, config_file=None, so
         models_dir = dl1_proton_dir.replace('/DL1', '/models')
     models_dir = models_dir.replace('/proton/', '/')
 
-    print(f"\tModels will be placed in {models_dir}")
+    log.info("Models will be placed in {}".format(models_dir))
     check_and_make_dir_without_verification(models_dir)
 
     cmd = ''
diff --git a/lstmcpipe/workflow_management.py b/lstmcpipe/workflow_management.py
index 237e1ed8..d4b1deb6 100644
--- a/lstmcpipe/workflow_management.py
+++ b/lstmcpipe/workflow_management.py
@@ -7,538 +7,11 @@
 import glob
 import yaml
 import pprint
+import logging
 import calendar
-import lstchain
-from lstmcpipe.stages import r0_to_dl1, merge_dl1, train_pipe, dl1_to_dl2, dl2_to_irfs, dl2_to_sensitivity
 
 
-def batch_r0_to_dl1(input_dir, conf_file, prod_id, particles_loop, source_env, gamma_offsets=None,
-                    workflow_kind='lstchain'):
-    """
-    Batch the r0_to_dl1 jobs by particle type.
-
-    Parameters
-    ----------
-    input_dir : str
-        Path to the r0/DL0 files
-    conf_file : str
-        Path to a configuration file. If none is given, a standard configuration is applied
-    prod_id : str
-        Production ID. If None, _v00 will be used, indicating an official base production. Default = None.
-    particles_loop : list
-        list with the particles to be processed. Takes the global variable ALL_PARTICLES
-    source_env : str
-        source environment to select the desired conda environment to run the r0/1_to_dl1 stage.
-    gamma_offsets : list
-    workflow_kind: str
-        One of the supported pipelines. Defines the command to be run on r0 files
-
-    Returns
-    -------
-    full_log : dict
-        Dictionary of dictionaries containing the full log of the batched jobs (jobids as keys) as well as the
-        4 more keys (one by particle) with all the jobs associated with each particle.
-    debug_log : dict
-            dictionary containing minimum information - jobids -  for log_reduced.txt
-    all_jobids_from_r0_dl1_stage : str
-        string, separated by commas, containing all the jobids of this stage
-    """
-    full_log = {'log_all_job_ids': {}}
-    debug_log = {}
-    all_jobids_from_r0_dl1_stage = []
-
-    print(f"\n ==== START {workflow_kind} r0 to dl1 processing ==== \n")
-
-    for particle in particles_loop:
-        if particle == 'gamma' and gamma_offsets is not None:
-            for off in gamma_offsets:
-                particle_input_dir = os.path.join(input_dir, off).format(particle)
-                _particle = particle + '_' + off
-                log, jobids_by_particle = r0_to_dl1(
-                    particle_input_dir,  # Particle needs to be gamma w/o the off
-                    config_file=conf_file,
-                    particle=_particle,
-                    prod_id=prod_id,
-                    source_environment=source_env,
-                    offset=off,
-                    workflow_kind=workflow_kind,
-                )
-                full_log['log_all_job_ids'].update(log)
-                full_log[_particle] = ','.join(jobids_by_particle)
-                all_jobids_from_r0_dl1_stage.append(full_log[_particle])  # Create a list with particles elements
-
-                for jid in jobids_by_particle:
-                    debug_log[jid] = f'{_particle} job from r0_to_dl1'        
-        else:
-            particle_input_dir = input_dir.format(particle)
-            _particle = particle
-            log, jobids_by_particle = r0_to_dl1(
-                particle_input_dir,  # Particle needs to be gamma w/o the off
-                config_file=conf_file,
-                particle=_particle,
-                prod_id=prod_id,
-                source_environment=source_env,
-                workflow_kind=workflow_kind,
-            )
-            full_log['log_all_job_ids'].update(log)
-            full_log[_particle] = ','.join(jobids_by_particle)
-            all_jobids_from_r0_dl1_stage.append(full_log[_particle])  # Create a list with particles elements
-
-            for jid in jobids_by_particle:
-                debug_log[jid] = f'{_particle} job from r0_to_dl1'
-    all_jobids_from_r0_dl1_stage = ','.join(all_jobids_from_r0_dl1_stage)  # Create a string to be directly passed
-
-    print(f"\n ==== END {workflow_kind} r0 to dl1 processing ==== \n")
-
-    return full_log, debug_log, all_jobids_from_r0_dl1_stage  # ids_by_particle_ok
-
-
-def batch_merge_and_copy_dl1(running_analysis_dir, log_jobs_from_r0_to_dl1, particles_loop, source_env,
-                             smart_merge=False, no_image_flag=True, prod_id=None, gamma_offsets=None, workflow_kind='lstchain'):
-    """
-    Function to batch the onsite_mc_merge_and_copy function once the all the r0_to_dl1 jobs (batched by particle type)
-    have finished.
-
-    Batch 8 merge_and_copy_dl1 jobs ([train, test] x particle) + the move_dl1 and move_dir jobs (2 per particle).
-
-    Parameters
-    ----------
-    running_analysis_dir : str
-        Directory to dl1 files
-    log_jobs_from_r0_to_dl1 : dict
-        dictionary of dictionaries containing the log (jobids organized by particle) from the previous stage
-        (onsite_mc_r0_to_dl1)
-    smart_merge : bool
-        flag to indicate whether the merge of the files should be done with `--smart True` or `--smart False`,
-        controlling the argument of the `lstchain_merge_hdf5_files.py` script (batched in this function).
-    particles_loop : list
-        list with the particles to be processed. Takes the global variable ALL_PARTICLES
-    no_image_flag : bool
-        flag to indicate whether the --no-image argument of the `lstchain_merge_hdf5_files.py` script (batched in
-        this function) should be either True or False.
-    gamma_offsets : list
-        list containing the offset of the gammas
-    prod_id : str
-        prod_id defined in config_MC_prod.yml
-    source_env : str
-        source environment to select the desired conda environment to run the r0/1_to_dl1 stage.
-
-    Returns
-    -------
-    log_merge_and_copy : dict
-        Dictionary containing the log of the batched merge_and_copy_dl1 jobs
-    jobid_4_train : str
-         string containing the jobids to be passed to the next stage of the workflow (as a slurm dependency)
-    all_merge : str
-        string containing all the jobs to indicate that all the dl1 are correctly finished (for dl1_to_dl2)
-    debug_log : dict
-        Debug purposes
-
-    """
-    log_merge_and_copy = {}
-    jobid_4_train = []
-    all_jobs_from_merge_stage = []
-    debug_log = {}
-
-    if smart_merge == 'lst' or smart_merge == 'lstchain':
-        merge_flag = True
-    elif smart_merge == 'rta' or smart_merge == 'hiperta':
-        merge_flag = False
-    elif smart_merge:
-        merge_flag = True
-    elif not smart_merge:
-        merge_flag = False
-    else:
-        merge_flag = False
-
-    print("\n ==== START {} ==== \n".format('batch merge_and_copy_dl1_workflow'))
-
-    for particle in particles_loop:
-        if particle == 'gamma' and gamma_offsets is not None:
-            for off in gamma_offsets:
-
-                gamma_running_analysis_dir = os.path.join(running_analysis_dir, off)
-                _particle = particle + '_' + off
-
-                log, jobid_mv_all_dl1, jobid_debug = merge_dl1(
-                    gamma_running_analysis_dir.format(particle),
-                    particle2jobs_dict=log_jobs_from_r0_to_dl1,
-                    particle=_particle,
-                    flag_merge=merge_flag,
-                    flag_no_image=no_image_flag,
-                    prod_id=prod_id,
-                    gamma_offset=off,
-                    source_environment=source_env,
-                    workflow_kind=workflow_kind,
-                )
-
-                log_merge_and_copy.update(log)
-                all_jobs_from_merge_stage.append(jobid_debug)
-                if _particle == 'gamma-diffuse' or _particle == 'proton':
-                    jobid_4_train.append(jobid_mv_all_dl1)
-
-                debug_log[jobid_mv_all_dl1] = f'{_particle} merge_and_copy-job - INDEED IT IS JUST PASSED the ' \
-                                              f'move_dl1 jobid - that will be send to the train pipe stage. They ' \
-                                              f'depend on the following {log_jobs_from_r0_to_dl1[_particle]} ' \
-                                              f'r0_t0_dl1 jobs.'
-                debug_log[jobid_debug] = f'Are all the {_particle} jobs that have been launched in merge_and_copy_dl1.'
-
-        else:
-            _particle = particle
-
-            log, jobid_mv_all_dl1, jobid_debug = merge_dl1(
-                running_analysis_dir.format(particle),
-                particle2jobs_dict=log_jobs_from_r0_to_dl1,
-                particle=_particle,
-                flag_merge=merge_flag,
-                flag_no_image=no_image_flag,
-                prod_id=prod_id,
-                source_environment=source_env,
-                workflow_kind=workflow_kind,
-            )
-
-            log_merge_and_copy.update(log)
-            all_jobs_from_merge_stage.append(jobid_debug)
-            if _particle == 'gamma-diffuse' or _particle == 'proton':
-                jobid_4_train.append(jobid_mv_all_dl1)
-
-            debug_log[jobid_mv_all_dl1] = f'{_particle} merge_and_copy-job - INDEED IT IS JUST PASSED the ' \
-                                          f'move_dl1 jobid - that will be send to the train pipe stage. They depend ' \
-                                          f'on the following {log_jobs_from_r0_to_dl1[_particle]} r0_t0_dl1 jobs.'
-            debug_log[jobid_debug] = f'Are all the {_particle} jobs that have been launched in merge_and_copy_dl1.'
-
-    jobid_4_train = ','.join(jobid_4_train)
-    all_jobs_from_merge_stage = ','.join(all_jobs_from_merge_stage)
-
-    print("\n ==== END {} ==== \n".format('batch merge_and_copy_dl1_workflow'))
-
-    return log_merge_and_copy, jobid_4_train, all_jobs_from_merge_stage, debug_log
-
-
-def batch_train_pipe(log_from_merge, config_file, jobids_from_merge, source_env):
-    """
-    Function to batch the lstchain train_pipe once the proton and gamma-diffuse merge_and_copy_dl1 batched jobs have
-    finished.
-
-    Parameters
-    ----------
-    log_from_merge : dict
-        dictionary containing the output name and abs path to the DL1 files, derived in merge_and_copy and saved
-        through the log
-    config_file : str
-        Path to a configuration file. If none is given, a standard configuration is applied
-    jobids_from_merge : str
-        string containing the jobids (***ONLY from proton and gamma-diffuse***) from the jobs batched in the
-         merge_and_copy_dl1 stage, to be passed to the train_pipe function (as a slurm dependency)
-    source_env : str
-        source environment to select the desired conda environment to run train_pipe and dl1_to_dl2 stages
-
-    Returns
-    -------
-    log_train : dict
-        Dictionary containing the log of the batched train_pipe jobs
-    jobid_4_dl1_to_dl2 : str
-        string containing the jobid to be passed to the next stage of the workflow (as a slurm dependency).
-        For the next stage, however, it will be needed TRAIN + MERGED jobs
-    model_path : str
-        Path with the model's directory
-    debug_log : dict
-        Debug and summary purposes
-    """
-    debug_log = {}
-
-    print("\n ==== START {} ==== \n".format('batch mc_train_workflow'))
-
-    gamma_dl1_train_file = log_from_merge['gamma-diffuse']['training']['train_path_and_outname_dl1']
-    proton_dl1_train_file = log_from_merge['proton']['training']['train_path_and_outname_dl1']
-
-    log_train, jobid_4_dl1_to_dl2, model_path = train_pipe(
-        gamma_dl1_train_file,
-        proton_dl1_train_file,
-        config_file=config_file,
-        source_environment=source_env,
-        wait_ids_proton_and_gammas=jobids_from_merge
-    )
-
-    debug_log[jobid_4_dl1_to_dl2] = f'The single jobid from train_pipe that depends of {jobids_from_merge} - merge' \
-                                    f'_and_copy jobids'
-
-    print("\n ==== END {} ==== \n".format('batch mc_train_workflow'))
-
-    return log_train, jobid_4_dl1_to_dl2, model_path, debug_log
-
-
-def batch_plot_rf_features(dir_models, config_file, source_env, train_jobid):
-    """
-    Batches the plot_model_importance.py script that creates a .png with the RF feature's importance models
-    after the RF are trained.
-    The plot is saved in the same dir in where the modes are stored.
-
-    Parameters
-    ----------
-    dir_models: str
-        Path to model's directory
-    config_file: str
-        Path to lstchain config file
-    source_env: str
-        String containing the .bashrc file to source and the conda env to call
-    train_jobid: str
-        Single jobid from training stage.
-
-    Returns
-    -------
-    log: dict
-        Dictionary with lstmcpipe_plot_models_importance single job id to be passed to debug log.
-    """
-    log = {}
-    print("\n ==== START {} ==== \n".format('batch plot RF features importance'))
-    jobe = os.path.join(dir_models, 'job_plot_rf_feat_importance.e')
-    jobo = os.path.join(dir_models, 'job_plot_rf_feat_importance.o')
-
-    base_cmd = f'lstmcpipe_plot_models_importance {dir_models} -cf {config_file}'
-    cmd = f'sbatch --parsable --dependency=afterok:{train_jobid} -e {jobe} -o {jobo} -J RF_importance ' \
-          f' --wrap="export MPLBACKEND=Agg; {source_env} {base_cmd}"'
-    jobid = os.popen(cmd).read().strip('\n')
-
-    log[jobid] = 'Single job_id to plot RF feature s importance'
-
-    print(f" Random Forest importance's plot will be saved at:\n   {dir_models}")
-    print("\n ==== END {} ==== \n".format('batch plot RF features importance'))
-
-    return log
-
-
-def batch_dl1_to_dl2(dl1_directory, path_to_models, config_file, jobid_from_training, jobids_from_merge,
-                     dict_with_dl1_paths, particles_loop, source_env, gamma_offsets=None):
-    """
-    Function to batch the dl1_to_dl2 stage once the lstchain train_pipe batched jobs have finished.
-
-    Parameters
-    ----------
-    dl1_directory : str
-        Path to the dl1 directory
-    path_to_models : str
-        Path to the model directory - should be taken from train_pipe
-    config_file : str
-        Path to a configuration file. If none is given, a standard configuration is applied
-    jobid_from_training : str
-        string containing the jobid from the jobs batched in the train_pipe stage, to be passed to the
-        dl1_to_dl2 function (as a slurm dependency)
-    jobids_from_merge : str
-        string containing the jobid from the jobs batched in the merge_and_copy_dl1 stage,
-        to be passed to the dl1_to_dl2 function (as a slurm dependency)
-    dict_with_dl1_paths : dict
-        Indeed the log of the merge_and_copy stage, where the final names of the dl1 files were stored
-    particles_loop : list
-        list with the particles to be processed. Takes the global variable ALL_PARTICLES
-    source_env : str
-        source environment to select the desired conda environment to run train_pipe and dl1_to_dl2 stages
-    gamma_offsets : list
-        list off gamma offsets
-
-    Returns
-    -------
-    log_batch_dl1_to_dl2 : dict
-        Dictionary containing the log of the batched dl1_to_dl2 jobs
-    jobid_4_dl2_to_dl3 : str
-        string containing the jobids to be passed to the next stage of the workflow (as a slurm dependency)
-    debug_log : dict
-        Debug and summary purposes
-
-    """
-
-    log_dl1_to_dl2 = {}
-    jobid_for_dl2_to_dl3 = []
-    debug_log = {}
-
-    print("\n ==== START {} ==== \n".format('batch dl1_to_dl2_workflow'))
-
-    for particle in particles_loop:
-
-        if particle == 'gamma' and gamma_offsets is not None:
-            for off in gamma_offsets:
-                gamma_dl1_directory = os.path.join(dl1_directory, off)
-                _particle = particle + '_' + off
-
-                log, jobid = dl1_to_dl2(
-                    gamma_dl1_directory.format(particle),
-                    path_models=path_to_models,
-                    config_file=config_file,
-                    particle=_particle,
-                    wait_jobid_train_pipe=jobid_from_training,
-                    wait_jobids_merge=jobids_from_merge,
-                    dictionary_with_dl1_paths=dict_with_dl1_paths,
-                    source_environment=source_env
-                )
-
-                log_dl1_to_dl2.update(log)
-                jobid_for_dl2_to_dl3.append(jobid)
-
-                debug_log[jobid] = f'{_particle} job from dl1_to_dl2 that depends both on : {jobid_from_training} ' \
-                                   f'training jobs AND from {jobids_from_merge} merge_and_copy_dl1 jobs'
-
-        else:
-            _particle = particle
-            log, jobid = dl1_to_dl2(
-                dl1_directory.format(particle),
-                path_models=path_to_models,
-                config_file=config_file,
-                particle=_particle,
-                wait_jobid_train_pipe=jobid_from_training,
-                wait_jobids_merge=jobids_from_merge,
-                dictionary_with_dl1_paths=dict_with_dl1_paths,
-                source_environment=source_env
-            )
-
-            log_dl1_to_dl2.update(log)
-            jobid_for_dl2_to_dl3.append(jobid)
-
-            debug_log[jobid] = f'{_particle} job from dl1_to_dl2 that depends both on : {jobid_from_training} ' \
-                               f'training jobs AND from {jobids_from_merge} merge_and_copy_dl1 jobs'
-
-    jobid_4_dl2_to_dl3 = ','.join(jobid_for_dl2_to_dl3)
-
-    print("\n ==== END {} ==== \n".format('batch dl1_to_dl2_workflow'))
-
-    return log_dl1_to_dl2, jobid_4_dl2_to_dl3, debug_log
-
-
-def batch_dl2_to_irfs(dl2_directory, loop_particles, offset_gammas, config_file, job_ids_from_dl1_dl2,
-                      log_from_dl1_dl2, source_env, prod_id):
-    """
-    Batches the dl2_to_irfs stage (lstchain lstchain_create_irf_files script) once the dl1_to_dl2 stage had finished.
-
-    Parameters
-    ----------
-    dl2_directory: str
-        Base path to DL2 directory to be formatted with particle type
-    config_file: str
-        Path to lstchain-like config file
-    loop_particles: list
-        list with particles to be processed.
-    offset_gammas: list
-        list off gamma offsets
-    job_ids_from_dl1_dl2: str
-        Comma-separated string with the job ids from the dl1_to_dl2 stage to be used as a slurm dependency
-        to schedule the current stage
-    source_env: str
-        source environment to select the desired conda environment (source .bnashrc + conda activate $ENV)
-    log_from_dl1_dl2: dict
-        Dictionary from dl1_to_dl2 stage with particle path information
-    prod_id: str
-        String with prod_id prefix to complete 'file-naming'
-
-    Returns
-    -------
-    log_batch_dl2_to_irfs: dict
-    jobs_from_dl2_irf: str
-    debug_dl2_to_irfs: dict
-    """
-    print("\n ==== START {} ==== \n".format('batch mc_dl2_to_irfs'))
-
-    debug_log = {}
-    jobid_for_check = []
-    log_dl2_to_irfs = {}
-
-    for off in offset_gammas:
-
-        log, jobid = dl2_to_irfs(
-            dl2_directory,
-            config_file=config_file,
-            log_from_dl1_dl2=log_from_dl1_dl2,
-            irf_point_like=True,
-            irf_gamma_offset=off,
-            source_env=source_env,
-            wait_jobs_dl1dl2=job_ids_from_dl1_dl2,
-            prod_id=prod_id
-        )
-
-        jobid_for_check.append(jobid)
-        log_dl2_to_irfs[f'gamma_{off}'] = log
-        debug_log[jobid] = f'Gamma_{off} job_id from the dl2_to_irfs stage that depends of the dl1_to_dl2 stage ' \
-                           f'job_ids; {job_ids_from_dl1_dl2}'
-
-    if 'gamma-diffuse' in loop_particles:
-
-        log, jobid = dl2_to_irfs(
-            dl2_directory,
-            irf_point_like=False,
-            config_file=config_file,
-            source_env=source_env,
-            log_from_dl1_dl2=log_from_dl1_dl2,
-            wait_jobs_dl1dl2=job_ids_from_dl1_dl2,
-            prod_id=prod_id
-        )
-
-        jobid_for_check.append(jobid)
-        log_dl2_to_irfs[f'gamma-diffuse'] = log
-        debug_log[jobid] = f'Gamma-diffuse job_id from the dl2_to_irfs stage that depends of the dl1_to_dl2 stage ' \
-                           f'job_ids; {job_ids_from_dl1_dl2}'
-
-    jobid_for_check = ','.join(jobid_for_check)
-
-    print("\n ==== END {} ==== \n".format('batch mc_dl2_to_irfs'))
-
-    return log_dl2_to_irfs, jobid_for_check, debug_log
-
-
-def batch_dl2_to_sensitivity(dl2_directory, offset_gammas, job_ids_from_dl1_dl2, log_from_dl1_dl2, source_env, prod_id):
-    """
-    Batches the dl2_to_sensitivity stage (`stages.script_dl2_to_sensitivity` based in the pyIRF iib) once the
-    dl1_to_dl2 stage had finished.
-
-    Parameters
-    ----------
-    dl2_directory: str
-        Base path to DL2 directory to be formatted with particle type
-    offset_gammas: list
-        list off gamma offsets
-    job_ids_from_dl1_dl2: str
-        Comma-separated string with the job ids from the dl1_to_dl2 stage to be used as a slurm dependency
-        to schedule the current stage
-    log_from_dl1_dl2: dict
-        Dictionary from dl1_to_dl2 stage with particle path information
-    source_env: str
-        source environment to select the desired conda environment (source .bashrc + conda activate $ENV)
-    prod_id: str
-        String with prod_id prefix to complete 'file-naming'
-
-    Returns
-    -------
-    log_dl2_to_sensitivity: dict
-        Dictionary with job_id-slurm command key-value pair used for logging
-    jobid_for_check: str
-        Comma-separated jobids batched in the current stage
-    debug_log: dict
-        Dictionary with the job-id and stage explanation to be stored in the debug file
-
-    """
-    print("\n ==== START {} ==== \n".format('batch mc_dl2_to_sensitivity'))
-
-    debug_log = {}
-    jobid_for_check = []
-    log_dl2_to_sensitivity = {}
-
-    for off in offset_gammas:
-
-        log, jobid = dl2_to_sensitivity(dl2_directory,
-                                        log_from_dl1_dl2,
-                                        gamma_offset=off,
-                                        prod_id=prod_id,
-                                        source_env=source_env,
-                                        wait_jobs_dl1_dl2=job_ids_from_dl1_dl2
-                                        )
-
-        jobid_for_check.append(jobid)
-        log_dl2_to_sensitivity[f'gamma_{off}'] = log
-        debug_log[jobid] = f'Gamma_{off} job_ids from the dl2_to_sensitivity stage and the plot_irfs script that ' \
-                           f'depends on the dl1_to_dl2 stage job_ids; {job_ids_from_dl1_dl2}'
-
-    jobid_for_check = ','.join(jobid_for_check)
-
-    print("\n ==== END {} ==== \n".format('batch mc_dl2_to_sensitivity'))
-
-    return log_dl2_to_sensitivity, jobid_for_check, debug_log
+log = logging.getLogger(__name__)
 
 
 def save_log_to_file(dictionary, output_file, log_format, workflow_step=None):
@@ -699,7 +172,7 @@ def batch_mc_production_check(jobids_from_r0_to_dl1, jobids_from_merge, jobids_f
                 f'--wrap="{cmd_wrap}"'
 
     jobid = os.popen(batch_cmd).read().strip('\n')
-    print(f'\n\tSubmitted batch CHECK-job {jobid}\n')
+    log.info(f'Submitted batch CHECK-job {jobid}')
 
     # and in case the code brakes, here there is a summary of all the jobs by stages
     debug_log[jobid] = 'single jobid batched to check that all the dl1_to_dl2 stage jobs finish correctly.'


Test patch:
diff --git a/lstmcpipe/config/tests/test_config.py b/lstmcpipe/config/tests/test_config.py
index d359b1e4..92a4dd37 100644
--- a/lstmcpipe/config/tests/test_config.py
+++ b/lstmcpipe/config/tests/test_config.py
@@ -2,6 +2,7 @@
 from lstmcpipe.config.pipeline_config import config_valid, parse_config_and_handle_global_vars
 import tempfile
 import os
+import pytest
 import yaml
 from datetime import datetime
 
@@ -39,30 +40,35 @@ def test_config_valid():
     This does not test if no valid configuration is marked
     as invalid!
     """
-    assert config_valid(dummy_config)[0] is False
+    with pytest.raises(Exception):
+        config_valid(dummy_config)
 
     valid = dummy_config.copy()
     valid["prod_type"] = "prod5"
     valid["workflow_kind"] = "lstchain"
     valid["obs_date"] = "20200629_prod5_trans_80"
-    assert config_valid(valid)[0] is True
+    assert config_valid(valid)
 
     invalid_workflow = valid.copy()
     invalid_workflow["workflow_kind"] = "MARS"
-    assert config_valid(invalid_workflow)[0] is False
+    with pytest.raises(Exception):
+        config_valid(invalid_workflow)
 
     invalid_date = valid.copy()
     invalid_date["obs_date"] = "1970-01-01"
-    assert config_valid(invalid_date)[0] is False
+    with pytest.raises(Exception):
+        config_valid(invalid_date)
 
     invalid_prod = valid.copy()
     invalid_prod["prod_type"] = "prod42"
-    assert config_valid(invalid_prod)[0] is False
+    with pytest.raises(Exception):
+        config_valid(invalid_prod)
 
     invalid_date_prod = valid.copy()
     # this should only be allowed with prod3, which is not chosen here
     invalid_date_prod["obs_date"] = "20190415"
-    assert config_valid(invalid_date_prod)[0] is False
+    with pytest.raises(Exception):
+        config_valid(invalid_date_prod)
 
 
 def test_parse_config_and_handle_global_vars():