Repository: gammapy/gammapy

Issue:
Nested Estimators create child processes
  I noticed than setting `n_jobs=2` on the `FluxProfileEstimator ` will also set `n_jobs=2` to the `FluxPointEstimator` created by the `FluxProfileEstimator` but using the context manager set only `n_jobs=2` for the `FluxProfileEstimator` and the original default  `n_jobs=1` on the `FluxPointEstimator`.  I don't think it's a bug , my guess is that the child processes don't see the modifications of the global variables as they store a copy of them.

Same problem for the `LightCurveEstiamtor`.            


Discussion:

Resolving patch:
diff --git a/gammapy/estimators/points/lightcurve.py b/gammapy/estimators/points/lightcurve.py
--- a/gammapy/estimators/points/lightcurve.py
+++ b/gammapy/estimators/points/lightcurve.py
@@ -166,6 +166,7 @@ def run(self, datasets):
                 parallel_datasets.append(datasets_to_fit)
 
         if self.n_jobs > 1:
+            self._update_child_jobs()
             rows = parallel.run_multiprocessing(
                 self.estimate_time_bin_flux,
                 zip(
@@ -225,7 +226,9 @@ def estimate_time_bin_flux(self, datasets, dataset_names=None):
             Resulting flux points.
         """
 
-        fp = super().run(datasets)
+        estimator = self.copy()
+        estimator.n_jobs = self._n_child_jobs
+        fp = estimator._run_flux_points(datasets)
 
         if dataset_names:
             for name in ["counts", "npred", "npred_excess"]:
@@ -233,3 +236,6 @@ def estimate_time_bin_flux(self, datasets, dataset_names=None):
                     fp._data[name], dataset_names=dataset_names
                 )
         return fp
+
+    def _run_flux_points(self, datasets):
+        return super().run(datasets)
diff --git a/gammapy/estimators/points/profile.py b/gammapy/estimators/points/profile.py
--- a/gammapy/estimators/points/profile.py
+++ b/gammapy/estimators/points/profile.py
@@ -171,7 +171,12 @@ def _run_region(self, datasets, region):
                 .data
             )
         datasets_to_fit.models = SkyModel(self.spectrum, name="test-source")
-        return super().run(datasets_to_fit)
+        estimator = self.copy()
+        estimator.n_jobs = self._n_child_jobs
+        return estimator._run_flux_points(datasets_to_fit)
+
+    def _run_flux_points(self, datasets):
+        return super().run(datasets)
 
     @property
     def config_parameters(self):
diff --git a/gammapy/utils/parallel.py b/gammapy/utils/parallel.py
--- a/gammapy/utils/parallel.py
+++ b/gammapy/utils/parallel.py
@@ -44,6 +44,7 @@ class PoolMethodEnum(Enum):
 
 BACKEND_DEFAULT = ParallelBackendEnum.multiprocessing
 N_JOBS_DEFAULT = 1
+ALLOW_CHILD_JOBS = False
 POOL_KWARGS_DEFAULT = dict(processes=N_JOBS_DEFAULT)
 METHOD_DEFAULT = PoolMethodEnum.starmap
 METHOD_KWARGS_DEFAULT = {}
@@ -151,6 +152,8 @@ def __exit__(self, type, value, traceback):
 class ParallelMixin:
     """Mixin class to handle parallel processing."""
 
+    _n_child_jobs = 1
+
     @property
     def n_jobs(self):
         """Number of jobs as an integer."""
@@ -169,6 +172,21 @@ def n_jobs(self, value):
             )
 
         self._n_jobs = value
+        if ALLOW_CHILD_JOBS:
+            self._n_child_jobs = value
+
+    def _update_child_jobs(self):
+        """needed because we can update only in the main process
+        otherwise global ALLOW_CHILD_JOBS has default value"""
+        if ALLOW_CHILD_JOBS:
+            self._n_child_jobs = self.n_jobs
+        else:
+            self._n_child_jobs = 1
+
+    @property
+    def _get_n_child_jobs(self):
+        """Number of allowed child jobs as an integer."""
+        return self._n_child_jobs
 
     @property
     def parallel_backend(self):
@@ -246,9 +264,8 @@ def run_multiprocessing(
             processes = cpu_count
 
         if multiprocessing.current_process().name != "MainProcess":
-            # subprocesses cannot have childs
+            # with multiprocessing subprocesses cannot have childs (but possible with ray)
             processes = 1
-    # TODO: check for ray
 
     if processes == 1:
         return run_loop(


Test patch:
diff --git a/gammapy/estimators/points/tests/test_lightcurve.py b/gammapy/estimators/points/tests/test_lightcurve.py
--- a/gammapy/estimators/points/tests/test_lightcurve.py
+++ b/gammapy/estimators/points/tests/test_lightcurve.py
@@ -10,6 +10,7 @@
 from gammapy.datasets import Datasets
 from gammapy.datasets.actors import DatasetsActor
 from gammapy.estimators import FluxPoints, LightCurveEstimator
+from gammapy.estimators.points.lightcurve import parallel as parallel
 from gammapy.estimators.points.tests.test_sed import (
     simulate_map_dataset,
     simulate_spectrum_dataset,
@@ -749,6 +750,15 @@ def test_lightcurve_parallel_multiprocessing():
     assert_allclose(
         lightcurve.dnde_ul.data[0], [[[3.260703e-13]], [[1.159354e-14]]], rtol=1e-3
     )
+    assert estimator._get_n_child_jobs == 1
+
+    parallel.ALLOW_CHILD_JOBS = True
+    lightcurve = estimator.run(datasets)
+    assert_allclose(
+        lightcurve.dnde_ul.data[0], [[[3.260703e-13]], [[1.159354e-14]]], rtol=1e-3
+    )
+    assert estimator._get_n_child_jobs == 2
+    parallel.ALLOW_CHILD_JOBS = False
 
 
 @requires_data()
@@ -765,7 +775,17 @@ def test_lightcurve_parallel_ray():
         parallel_backend="ray",
     )
 
+    assert estimator.n_jobs == 2
+    lightcurve = estimator.run(datasets)
+    assert_allclose(
+        lightcurve.dnde_ul.data[0], [[[3.260703e-13]], [[1.159354e-14]]], rtol=1e-3
+    )
+    assert estimator._get_n_child_jobs == 1
+
+    parallel.ALLOW_CHILD_JOBS = True
     lightcurve = estimator.run(datasets)
     assert_allclose(
         lightcurve.dnde_ul.data[0], [[[3.260703e-13]], [[1.159354e-14]]], rtol=1e-3
     )
+    assert estimator._get_n_child_jobs == 2
+    parallel.ALLOW_CHILD_JOBS = False