Repository: gammapy/gammapy

Issue:
Problem of fit with sherpa
Hi @cdeil and @joleroi , 
I've got some problem to do a spectral fit with sherpa on CTA simulated data. I don't really understand why it's happening. Everything worked fine ~2 weeks ago.

Here is a small example to illustrate the problem: 
```
import astropy.units as u

from gammapy.spectrum.models import PowerLaw
from gammapy.scripts.cta_utils import CTAObservationSimulation, Target, ObservationParameters

from gammapy.scripts import CTAPerf

# observation parameters
alpha = 0.2 * u.Unit('')
livetime = 100 * u.h
emin = 0.05 * u.TeV
emax = 5. * u.TeV
obs_param = ObservationParameters(alpha=alpha,
                                  livetime=livetime,
                                  emin=emin,
                                  emax=emax)

# target
pwl = PowerLaw(amplitude=1e-11 * u.Unit('1/(s cm2 TeV)'),
               index=2.,
               reference=1. * u.TeV)
target = Target(name='bob', model=pwl)

# perf
dir_south = './CTA-Performance-South-20170323/'
perf = CTAPerf.read('$GAMMAPY_EXTRA/datasets/cta/perf_prod2/point_like_non_smoothed/South_5h.fits.gz')

# simu
obs_id = 0
simu = CTAObservationSimulation.simulate_obs(perf=perf,
                                             target=target,
                                             obs_param=obs_param,
                                             obs_id=obs_id)

# simu output
print(simu)
stats = simu.stats_table()
print(stats[['energy_min', 'energy_max', 'background', 'excess', 'sigma']])
simu.write(outdir='./', use_sherpa=True)


from sherpa.astro.datastack import DataStack
import sherpa.astro.datastack as sh
from sherpa_model import Absorption

simu_output_dir = './'
filename = simu_output_dir + 'pha_obs' + str(obs_id) + '.fits'
sh.load_data(id=obs_id, filename=filename)
sherpa_model = 'powlaw1d.pwl'
sh.set_source(obs_id, sherpa_model)
    
pwl.ampl = 1.e-12 * u.Unit('1/(s cm2 TeV)').to('1/(s cm2 keV)')
pwl.ref = 1 * u.TeV.to('keV')
pwl.gamma = 2.

sh.notice(emin.to('keV')*0.99,emax.to('keV')*1.01)
print('JLK: threshold for spectra: ({},{})'.format(emin.to('keV')*0.99,emax.to('keV')*1.01))

sh.set_stat('WStat')
sh.fit()
sh.conf()
```

which results in weird NaN values and implies problem for the fit:
```
Dataset               = 0
Method                = levmar
Statistic             = wstat
Initial fit statistic = nan
Final fit statistic   = nan at function evaluation 33
Data points           = 12
Degrees of freedom    = 10
Reduced statistic     = nan
Change in statistic   = nan
   pwl.gamma      2           
   pwl.ampl       1e-21       
```
 
Do you have any idea what could cause that? OGIP format writting or something like that?

Thanks in advance ++


Discussion:
I poked at this a bit, but didn't find the issue.

@joleroi - Favor attender!
(Julien and I don't know how alpha is supposed to be stored, and likely that's the issue ... see below)

Here's what I know:
* The simulation runs OK, results look good: https://gist.github.com/cdeil/4ccdbd47a0997bc9371431c47b55c1e6
* I write and read AEFF, EDISP, on and off counts, all look good.
* The @jjlk mentioned on Slack that fitting works if passing data directly to Sherpa (not writing it do disk). @jjlk - Please post that working example also, e.g. in a https://gist.github.com/ or notebook. For me, changing the stat to `cash` and the fit works.
* My guess is that writing the alpha information from Gammapy broke (or something changed on the Sherpa side). @joleroi - Can you add a test that writes and reads on-off data and checks that alpha comes back OK?
* There's an example here: https://nbviewer.jupyter.org/github/gammapy/gammapy-extra/blob/master/notebooks/spectrum_fitting_with_sherpa.ipynb But since that reads files from gammapy-extra, I guess it could be broken, and one would only see that when reproducing the files with latest Gammapy. Suggest to keep notebooks independent, and to add pytest tests.

Resolving patch:
diff --git a/gammapy/scripts/cta_utils.py b/gammapy/scripts/cta_utils.py
--- a/gammapy/scripts/cta_utils.py
+++ b/gammapy/scripts/cta_utils.py
@@ -1,4 +1,4 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
+
 import numpy as np
 import astropy.units as u
 from ..spectrum import SpectrumObservation
@@ -132,22 +132,21 @@ def simulate_obs(perf, target, obs_param, obs_id=0):
 
         on_counts += bkg_counts  # evts in ON region
 
-        meta = dict(EXPOSURE=livetime.to('s').value)
-
         on_vector = PHACountsSpectrum(
             data=on_counts,
             backscal=1,
             energy_lo=reco_energy.lo,
             energy_hi=reco_energy.hi,
-            meta=meta,
         )
 
+        on_vector.livetime = livetime
         off_vector = PHACountsSpectrum(energy_lo=reco_energy.lo,
                                        energy_hi=reco_energy.hi,
                                        data=off_counts,
                                        backscal=1. / alpha,
                                        is_bkg=True,
                                        )
+        off_vector.livetime = livetime
 
         obs = SpectrumObservation(on_vector=on_vector,
                                   off_vector=off_vector,


Test patch:
diff --git a/gammapy/spectrum/tests/test_fit.py b/gammapy/spectrum/tests/test_fit.py
--- a/gammapy/spectrum/tests/test_fit.py
+++ b/gammapy/spectrum/tests/test_fit.py
@@ -293,22 +293,20 @@ def test_run(self, tmpdir):
         actual = read_result.model.evaluate_error(1 * u.TeV)
         assert_quantity_allclose(actual, desired)
 
-
-@requires_dependency('sherpa')
-@requires_data('gammapy-extra')
-def test_sherpa_fit(tmpdir):
-    # this is to make sure that the written PHA files work with sherpa
-    import sherpa.astro.ui as sau
-    from sherpa.models import PowLaw1D
-
-    filename = gammapy_extra.filename("datasets/hess-crab4_pha/pha_obs23592.fits")
-    sau.load_pha(filename)
-    sau.set_stat('wstat')
-    model = PowLaw1D('powlaw1d.default')
-    model.ref = 1e9
-    model.ampl = 1
-    model.gamma = 2
-    sau.set_model(model * 1e-20)
-    sau.fit()
-    assert_allclose(model.pars[0].val, 2.0033101181778026)
-    assert_allclose(model.pars[2].val, 2.2991681244938498)
+    def test_sherpa_fit(self, tmpdir):
+        # this is to make sure that the written PHA files work with sherpa
+        import sherpa.astro.ui as sau
+        from sherpa.models import PowLaw1D
+
+        self.obs_list.write(tmpdir, use_sherpa=True)
+        filename = tmpdir / 'pha_obs23523.fits'
+        sau.load_pha(str(filename))
+        sau.set_stat('wstat')
+        model = PowLaw1D('powlaw1d.default')
+        model.ref = 1e9
+        model.ampl = 1
+        model.gamma = 2
+        sau.set_model(model * 1e-20)
+        sau.fit()
+        assert_allclose(model.pars[0].val, 2.0881699260935838)
+        assert_allclose(model.pars[2].val, 1.6234222129479836)