import ROOT
import numpy as np
from collections import defaultdict
from gna import datapath
import gna.parameters.ibd
import gna.parameters.oscillation
import itertools
from gna.exp import baseexp
from gna.env import env
import gna.constructors as C

year=365*24*60*60.0

spectrumfiles = {
    'Pu239': 'reactor_anu_spectra/Huber/Huber_smooth_extrap_Pu239_13MeV0.01MeVbin.dat',
    'Pu241': 'reactor_anu_spectra/Huber/Huber_smooth_extrap_Pu241_13MeV0.01MeVbin.dat',
    'U235': 'reactor_anu_spectra/Huber/Huber_smooth_extrap_U235_13MeV0.01MeVbin.dat',
    'U238': 'reactor_anu_spectra/Mueller/Mueller_smooth_extrap_U238_13MeV0.01MeVbin.dat',
}

eperfission = {
    'Pu239': (209.99, 0.60),
    'Pu241': (213.60, 0.65),
    'U235': (201.92, 0.46),
    'U238': (205.52, 0.96),
}

geo_flux_files = {
    'U238': 'AntineutrinoSpectrum_238U.knt',
    'Th232': 'AntineutrinoSpectrum_232Th.knt',
}

geo_flux_normalizations = {
    'U238': (2.7e3/5, 0.3),
    'Th232': (0.8e3/5, 0.3),
}

class Reactor(object):
    def __init__(self, name, location, fission_fractions,
                 power=None, power_rate=None):
        self.name = name
        self.location = location

        self.power = power

        self.power_rate = power_rate

        self.fission_fractions = {isoname: frac
                                  for isoname, frac in fission_fractions.items()}

    def assign(self, ns):
        self.ns = ns("reactors")(self.name)
        self.ns.reqparameter("ThermalPower", central=self.power, sigma=0)

    def calcdistance(self, detector):
        return np.sqrt(np.sum((np.array(self.location) - np.array(detector.location))**2))

    def initdistance(self, ns, detector):
        dist = self.calcdistance(detector)
        pair_ns = detector.ns(self.name)
        pair_ns.reqparameter("L_{0}".format(self.name), central=dist, sigma=0)
        pair_ns.defparameter("L", target=pair_ns.ref("L_{0}".format(self.name)))

class ReactorGroup(object):
    def __init__(self, name, reactors):
        self.name = name
        self.reactors = reactors

        self.power = 1
        self.power_rate = reactors[0].power_rate

        self.fission_fractions = reactors[0].fission_fractions

    def assign(self, ns):
        self.ns = ns("reactorgroups")(self.name)

    def initdistance(self, ns, detector):
        pair_ns = detector.ns(self.name)
        bindings = {}
        for i, reactor in enumerate(self.reactors):
            bindings["L_{0}".format(i)] = detector.ns(reactor.name)["L"]
            bindings["P_{0}".format(i)] = reactor.ns["ThermalPower"]
        ROOT.ReactorGroup(len(self.reactors), ns=pair_ns, bindings=bindings)

        pair_ns.defparameter("L", target=pair_ns.ref("Lavg"))
        pair_ns.defparameter("ThermalPower", target=pair_ns.ref("Pavg"))

class Detector(object):
    def __init__(self, name, edges, location,
                 orders=None, protons=None, livetime=None, edges_final=None):
        self.name = name
        self.edges = edges
        self.edges_final = edges_final
        self.orders = orders

        self.location = location

        self.protons = protons

        self.livetime = livetime

        self.unoscillated = None
        self.components = defaultdict(lambda: defaultdict(ROOT.Sum))
        self.backgrounds = defaultdict(ROOT.Sum)
        self.intermediates = {}
        self.intermediates_bkg = {}
        self.hists = defaultdict(dict)
        self.back_hists = defaultdict(dict)

    def assign(self, ns):
        self.ns = ns("detectors")(self.name)

        if self.protons is not None:
            self.ns.defparameter("TargetProtons", central=self.protons, relsigma=0.01)

        self.ns.reqparameter("Eres_a", central=0.0, sigma=0)
        self.ns.reqparameter("Eres_b", central=0.03, sigma=0)
        self.ns.reqparameter("Eres_c", central=0.0, sigma=0)
        self.ns.reqparameter("rho_C14", central=1e-16, sigma=1e-19)
        self.ns.reqparameter("CoincidenceWindow", central=300, sigma=1)

class GeoNeutrinoIsotope(object):
    def __init__(self, name):
        self.name = name

        try:
            Es_keV, self.ys = np.loadtxt(datapath(geo_flux_files[name]), unpack=True, skiprows=5)
        except OSError:
            raise Exception("Failed to load spectrum of {0} geo isotope from {1}".format(name, geo_flux_files[name]))
        self.Es = Es_keV*1e-3
        self.spectrum = ROOT.LinearInterpolator(len(self.Es), self.Es.copy(), self.ys.copy(), "use_zero")

class Isotope(object):
    def __init__(self, ns, name):
        self.ns = ns("isotopes")(name)
        self.name = name

        try:
            self.Es, self.ys = np.loadtxt(datapath(spectrumfiles[name]), unpack=True)
        except OSError:
            raise Exception("Failed to load spectrum of {0} reactor isotope from {1}".format(name, datapath(spectrumfiles[name])))

        self.spectrum = ROOT.LinearInterpolator(len(self.Es), self.Es.copy(), self.ys.copy(), "use_zero")

class ReactorExperimentModel(baseexp):
    oscprob_classes = {
            'standard': ROOT.OscProbPMNS,
            'decoh': ROOT.OscProbPMNSDecoh,
            }

    @classmethod
    def initparser(self, parser, env):
        """Initializes arguments parser with args common to all reactor experiments"""
        #  def OscProb(name):
            #  return {
                #  'standard': ROOT.OscProbPMNS,
                #  'decoh': ROOT.OscProbPMNSDecoh,
            #  }[name]
        parser.add_argument('--name', required=True)
        parser.add_argument('--ibd', choices=['zero', 'first'], default='zero')
        parser.add_argument('--backgrounds', choices=['geo'], action='append',
                            default=[], help='Choose backgrounds you want to add')
        parser.add_argument('--oscprob', choices=self.oscprob_classes.keys(),
                            default='standard')
        parser.add_argument('--binning', nargs=4, metavar=('DETECTOR', 'EMIN', 'EMAX', 'NBINS'), action='append', default=[])
        parser.add_argument('--binning-final', nargs=6, metavar=('DETECTOR', 'EMIN', 'E0', 'E1', 'NBINS01', 'EMAX'), action='append', default=[])
        parser.add_argument('--integration-order', type=int, default=4)
        parser.add_argument('--no-reactor-groups', action='store_true')
        parser.add_argument('--with-C14', action='store_true')

    def __init__(self, opts, ns=None, reactors=None, detectors=None):
        """Initialize a reactor experiment

        opts -- object with parsed common arguments returned by argparse
        ns -- namespace where experiment will create missing parameters, if is not provided ``opts.name`` will be used
        reactors -- iterable over Reactor objects, self.makereactors() will be called if None
        detectors -- iterable over Detector objects, self.makedetoctrs() will be called if None
        """
        super(ReactorExperimentModel, self).__init__(None, opts)
        self._oscprobs = {}
        self._oscprobcls = self.oscprob_classes[self.opts.oscprob]
        self._isotopes = defaultdict(list)
        self._Enu_inputs = defaultdict(set)
        self.oscprobs_comps = defaultdict(dict)
        self._geo_isotopes = defaultdict(list)

        self.ns = ns or env.ns(opts.name)
        self.reqparameters(self.ns)

        self.detectors = list(detectors if detectors is not None else self.makedetectors())
        for binopt in self.opts.binning:
            for det in self.detectors:
                if det.name == binopt[0]:
                    break
            else:
                raise Exception("can't find detector {}".format(binopt[0]))
            det.edges = np.linspace(float(binopt[1]), float(binopt[2]), int(binopt[3]))
        for (detname, emin, e0, e1, ne, emax) in self.opts.binning_final:
            for det in self.detectors:
                if det.name == detname:
                    break
            else:
                raise Exception("can't find detector {}".format(detname))
            det.edges_final = np.concatenate(([float(emin)],
                                              np.linspace(float(e0), float(e1), int(ne)),
                                              [float(emax)]))
        for det in self.detectors:
            det.assign(self.ns)
            if det.orders is None:
                det.orders = np.full(len(det.edges)-1, self.opts.integration_order, int)

        self.reactors = list(reactors if reactors is not None else self.makereactors())
        for reactor in self.reactors:
            reactor.assign(self.ns)

        self.linkpairs(self.reactors, self.detectors)

        for detector in self.detectors:
            self.make_backgrounds(detector, opts.backgrounds)
            self.setibd(detector, opts.ibd)
            self.setupobservations(detector)

    def make_backgrounds(self, detector, background_list):
        """Produce backgrounds and store them to detector components,
        currently only geoneutrino flux. Normalize it latter in makeibd"""

        #  pass
        for bkg in background_list:
            if bkg == 'geo':
                for iso_name in geo_flux_files.keys():
                    geo_isotope = GeoNeutrinoIsotope(iso_name)
                    self._isotopes["geo_"+iso_name].append(geo_isotope)
                    self._Enu_inputs[detector].add(geo_isotope.spectrum.f.inputs.x)
                    detector.intermediates_bkg['geo_flux_{}'.format(iso_name)] = geo_isotope.spectrum

    def _getoscprobcls(self, reactor, detector):
        """Returns oscillation probability class for given (reactor, detector) pair"""
        if isinstance(reactor, ReactorGroup):
            return (ROOT.OscProbPMNSMult, ROOT.OscProbPMNS)
        keys = [frozenset([reactor, detector]), reactor, detector, None]
        for key in keys:
            try:
                return (self._oscprobs[key], self._oscprobs[key])
            except KeyError:
                pass
        return (self._oscprobcls, self._oscprobcls)

    def _getnormtype(self, reactor, detector):
        """Returns normalization type applicable for given reactor and detector

        if power, power rate, protons and livetime are provided, 'calc' is returned to calculate normalization
        otherwise 'manual' is returned meaning that normalization should be done by the caller with external Norm parameters
        """
        if all(x is not None for x in [reactor.power, reactor.power_rate, detector.protons, detector.livetime]):
            return 'calc'
        return 'manual'

    def groupreactors(self, reactors, detector, precision=1e-2):
        """Merges some Reactors of reactors to ReactorGroup if their distances to detector are similar enough

        Faster approximated formula is used. Only reactors with standard OscProbPMNS formula are considered for grouping.
        Approximation of the probability is of the order of order precision**3.
        """
        groups = [[reactor] for reactor in reactors]
        groupable = []
        if not self.opts.no_reactor_groups:
            for reactor in reactors:
                if self._getoscprobcls(reactor, detector)[0] is ROOT.OscProbPMNS:
                    if self._getnormtype(reactor, detector) == 'calc':
                        groupable.append(reactor)
                        groups.remove([reactor])
        if not groupable:
            return reactors
        groups.append([])
        for reactor in sorted(groupable, key=lambda r: r.calcdistance(detector)):
            groups[-1].append(reactor)
            L = np.array([r.calcdistance(detector) for r in groups[-1]])
            P = np.array([r.power for r in groups[-1]])
            Lavg = sum(P/L)/sum(P/L**2)
            if not all(abs(L/Lavg-1) < precision):
                groups.append([groups[-1].pop()])
        grouped = []
        cnt = 0
        for group in groups:
            if len(group) > 1:
                rgroup = ReactorGroup("group{}".format(cnt), group)
                rgroup.assign(self.ns)
                rgroup.initdistance(self.ns, detector)
                grouped.append(rgroup)
                cnt += 1
            else:
                grouped.append(group[0])
        return grouped

    def normalization(self, reactor, detector, normtype):
        """Returns normalization object for given reactor/detector pair and norm type"""
        def vec(lst):
            v = ROOT.vector("std::string")()
            for s in lst:
                v.push_back(s)
            return v

        if normtype == 'calc':
            bindings = {}
            for isoname in reactor.fission_fractions:
                bindings["EnergyPerFission_{0}".format(isoname)] = self.ns("isotopes")(isoname)["EnergyPerFission"]
            norm = ROOT.ReactorNorm(vec(reactor.fission_fractions.keys()), bindings=bindings)
            lt=C.Points(detector.livetime)
            lt.points.setLabel('livetime: | '+detector.name)
            norm.isotopes.livetime(lt)
            pr=C.Points(reactor.power_rate)
            pr.points.setLabel('power: | '+reactor.name)
            norm.isotopes.power_rate(pr)
            norm.isotopes.setLabel('norm: | {} to {}'.format(reactor.name, detector.name))
            for isoname, frac in reactor.fission_fractions.items():
                ff=C.Points(frac)
                ff.points.setLabel('fission frac: | {} at {}'.format(isoname, reactor.name))
                norm.isotopes['fission_fraction_{0}'.format(isoname)](ff)
        elif normtype == 'manual':
            norm = ROOT.ReactorNormAbsolute(vec(reactor.fission_fractions.keys()))
            for isoname, frac in reactor.fission_fractions.items():
                norm.isotopes['fission_fraction_{0}'.format(isoname)](frac)
        return norm

    def linkpairs(self, reactors, detectors):
        """Setup oscillations and normalizations of all reactors and detectors.

        Resulting components are stored in detector.components.
        """
        for reactor, detector in itertools.product(reactors, detectors):
            reactor.initdistance(self.ns, detector)

        for detector in detectors:
            detector.unoscillated = ROOT.Sum()
            detector.unoscillated.sum.setLabel('unosc flux: | '+detector.name)
            grouped = self.groupreactors(reactors, detector)
            for rgroup in grouped:
                pair_ns = detector.ns(rgroup.name)
                normtype = self._getnormtype(rgroup, detector)
                with detector.ns, rgroup.ns, pair_ns:
                    norm = self.norm = self.normalization(rgroup, detector, normtype)
                    oscprobcls, weightscls = self._getoscprobcls(rgroup, detector)
                    with self.ns("oscillation"):
                        oscprob = oscprobcls(ROOT.Neutrino.ae(), ROOT.Neutrino.ae())

                normedflux = ROOT.Sum()
                normedflux.sum.setLabel('normed flux: | {}'.format(rgroup.name))

                for isoname in rgroup.fission_fractions.keys():
                    isotope = Isotope(self.ns, isoname)
                    isotope.spectrum.f.setLabel('spectrum: | {} at {}'.format(isoname, rgroup.name))
                    self._isotopes[(detector, rgroup)].append(isotope)
                    self._Enu_inputs[detector].add(isotope.spectrum.f.inputs.x)
                    subflux = ROOT.Product()
                    subflux.product.setLabel('flux | {}'.format(isoname))
                    subflux.multiply(isotope.spectrum)
                    subflux.multiply(norm.isotopes['norm_{0}'.format(isoname)])
                    detector.intermediates['flux_{}'.format(isoname)] = subflux
                    normedflux.add(subflux)
                detector.intermediates['flux'] = normedflux

                compnames = set(oscprob.probsum.inputs)

                detector.unoscillated.add(normedflux)
                if 'comp0' in compnames:
                    csum = detector.components['rate'][(weightscls, 'comp0')]
                    csum.add(normedflux)
                    csum.sum.setLabel('normed flux | comp0 at '+detector.name)
                    ones = ROOT.FillLike(1.0)
                    ones.fill.setLabel('1: comp0')
                    ones.fill.inputs(normedflux)
                    opsum=detector.components['oscprob'][(weightscls, 'comp0')]
                    opsum.add(ones)
                    opsum.sum.setLabel('osc flux | {} at {}'.format('comp0', detector.name))
                    self.oscprobs_comps[(detector, rgroup)][(weightscls, 'comp0')] = ones
                    compnames.remove('comp0')
                for osccomps in oscprob.transformations.values():
                    osccomps.setLabel('oscprob {} | {}-\\>{}'.format(osccomps.label(), rgroup.name, detector.name))
                    for compname, osccomp in osccomps.outputs.items():
                        if compname not in compnames:
                            continue
                        product = ROOT.Product()
                        product.product.setLabel('osc flux: | {} from {}'.format(compname, rgroup.name))
                        product.multiply(normedflux)
                        product.multiply(osccomp)
                        csum=detector.components['rate'][(weightscls, compname)]
                        csum.add(product)
                        csum.sum.setLabel('normed flux | {} at {}'.format(compname, detector.name))
                        opsum=detector.components['oscprob'][(weightscls, compname)]
                        opsum.add(osccomp)
                        opsum.sum.setLabel('osc flux | {} at {}'.format(compname, detector.name))
                        self.oscprobs_comps[(detector, rgroup)][(weightscls, compname)] = osccomp
                        if compname not in compnames:
                            raise Exception("overriden component {}".format(compname))
                        compnames.remove(compname)

                        if 'Enu' in osccomps.inputs:
                            self._Enu_inputs[detector].add(osccomps.inputs.Enu)
                if compnames:
                    raise Exception("components not found: {}".format(compnames))

    def setibd(self, detector, ibdtype):
        """Setup input energy values, integration and IBD calculation object for the given detector according to the ibdtype .
        ibdtype may be 'zero' or 'first' for the corresponding order.
        Resulting components are stored in detector.components.
        The backgrounds histos are also constructed here for now.
        """
        Evis_edges = detector.edges
        orders = detector.orders
        with self.ns("ibd"):
            econv = self.econv = ROOT.EvisToEe()
        if ibdtype == 'zero':
            with self.ns("ibd"):
                ibd = self.ibd = ROOT.IbdZeroOrder()
            integrator = self.integrator = ROOT.GaussLegendre(Evis_edges, orders, len(orders))
            integrator.points.setLabel('integrator 1d')
            histcls = ROOT.GaussLegendreHist
            econv.Ee.Evis(integrator.points.x)
            ibd.xsec.Ee(econv.Ee.Ee)
            ibd.Enu.Ee(econv.Ee.Ee)
            eventsparts = [ibd.xsec]
        elif ibdtype == 'first':
            with self.ns("ibd"):
                ibd = self.ibd = ROOT.IbdFirstOrder()
            integrator = self.integrator = ROOT.GaussLegendre2d(Evis_edges, orders, len(orders), -1.0, 1.0, 5)
            integrator.points.setLabel('integrator 2d')
            histcls = ROOT.GaussLegendre2dHist
            econv.Ee.Evis(integrator.points.x)
            ibd.Enu.Ee(econv.Ee.Ee)
            ibd.Enu.ctheta(integrator.points.y)
            detector.intermediates['ctheta'] = integrator.points.y
            ibd.xsec.Enu(ibd.Enu)
            ibd.xsec.ctheta(integrator.points.y)
            ibd.jacobian.Enu(ibd.Enu)
            # ibd.jacobian.Ee(integrator.points.x) # legacy: incorrect. X is Evis, not Ee
            ibd.jacobian.Ee(econv.Ee.Ee)
            ibd.jacobian.ctheta(integrator.points.y)
            eventsparts = [ibd.xsec, ibd.jacobian]
        else:
            raise Exception("unknown ibd type {0!r}".format(ibdtype))
        ibd.xsec.setLabel('IBD xsec')

        detector.intermediates['Enu'] = ibd.Enu
        detector.intermediates['xsec'] = ibd.xsec
        for inp in self._Enu_inputs.get(detector, []):
            inp.connect(ibd.Enu.Enu)

        for detector in self.detectors:
            for resname, comps in detector.components.items():
                for compid, comp in comps.items():
                    res = None
                    if resname == 'rate':
                        res = ROOT.Product()
                        res.product.setLabel('count rate: | {} at {}'.format(compid[1], detector.name))
                        res.multiply(comp)
                        for part in eventsparts:
                            res.multiply(part)
                    elif resname == 'oscprob':
                        res = comp
                    hist = detector.hists[resname][compid] = histcls(integrator)
                    hist.hist.setLabel('{} hist: | {} at {}'.format(resname, compid[1], detector.name))
                    detector.hists[resname][compid].hist.inputs(res)

            # Below we construct backgrounds histos
            if self.opts.backgrounds:
                bkg_summary = ROOT.Sum()
                bkg_summary.sum.setLabel('bkg: | '+detector.name)
                for bkg_name, bkg in detector.intermediates_bkg.items():
                    prod = ROOT.Product()
                    prod.product.setLabel('bkg: | '+detector.name)
                    prod.multiply(bkg)
                    unosc_bkg_name = 'unosc_' + bkg_name
                    for part in eventsparts:
                        prod.multiply(part)
                    hist = detector.back_hists[unosc_bkg_name] = histcls(integrator)
                    hist.hist.setLabel('bkg hist: | '+detector.name)
                    detector.back_hists[unosc_bkg_name].hist.inputs(prod)

                    # normalize isotope flux now
                    iso_name = bkg_name.split('_')[-1]
                    with self.ns("geo_isotopes")(iso_name):
                        norm_geo = ROOT.GeoNeutrinoFluxNormed(detector.livetime[0]/year)
                    norm_geo.flux_norm.flux(detector.back_hists[unosc_bkg_name].hist)

                    with self.ns("oscillation"):
                        aver_oscprob = ROOT.OscProbAveraged(ROOT.Neutrino.ae(), ROOT.Neutrino.ae())
                    aver_oscprob.average_oscillations.inputs(norm_geo.flux_norm)
                    detector.back_hists['geo_'+iso_name] = norm_geo.flux_norm.normed_flux
                    bkg_summary.add(aver_oscprob.average_oscillations.flux_averaged_osc)

            detector.unoscillated_hist = histcls(integrator)
            res = ROOT.Product()
            res.product.setLabel('unosc IBD: | '+detector.name)
            detector.unoscillated_hist.hist.setLabel('unoscillated hist: | '+detector.name)
            res.multiply(detector.unoscillated)
            for part in eventsparts:
                res.multiply(part)
            detector.unoscillated_hist.hist.inputs(res)

            if self.opts.backgrounds:
                detector.unoscillated_with_bkg = ROOT.Sum()
                detector.unoscillated_with_bkg.sum.setLabel('noosc+bkg | '+detector.name)
                detector.unoscillated_with_bkg.add(bkg_summary)
                detector.unoscillated_with_bkg.add(detector.unoscillated_hist)
                detector.back_hists['sum_bkg'] = bkg_summary

    def _sumcomponents(self, components, name):
        oscprobs = {}
        for oscprobcls, compname in components.keys():
            if oscprobcls in oscprobs:
                continue
            with self.ns("oscillation"):
                oscprob = oscprobcls(ROOT.Neutrino.ae(), ROOT.Neutrino.ae(),
                                     freevars=['L'])
            oscprobs[oscprobcls] = oscprob

        for compid, comp in components.items():
            ps = oscprobs[compid[0]].probsum
            ps.setLabel('osc prob: | '+name)
            ps[compid[1]](comp)

        probsums = [oscprob.probsum for oscprob in oscprobs.values()]
        if len(probsums) > 1:
            finalsum = ROOT.Sum()
            finalsum.sum.setLabel('finalsum | '+name)
            for probsum in probsums:
                finalsum.add(probsum)
            return finalsum
        else:
            return probsums[0]

    def setupobservations(self, detector):
        """Sum over the components and setup the observables to namespace"""

        self.ns.addobservable("{0}_unoscillated".format(detector.name), detector.unoscillated_hist.hist, export=False)
        if self.opts.backgrounds:
            self.ns.addobservable("{0}_unoscillated_with_bkg".format(detector.name),
                           detector.unoscillated_with_bkg, export=False)

        sums = {resname: self._sumcomponents(detector.hists[resname], detector.name)
                for resname in detector.hists}

        if 'oscprob' in sums:
            detector.intermediates["oscprob"] = sums['oscprob']
        if 'rate' in sums:
            inter_sum = ROOT.Sum()
            inter_sum.sum.setLabel('Observed IBD: | '+detector.name)
            sum_without_bkg = sums['rate']
            inter_sum.add(sum_without_bkg)
            if self.opts.backgrounds:
                inter_sum.add(detector.back_hists['sum_bkg'])

            self.ns.addobservable("{0}_noeffects".format(detector.name), inter_sum, export=False)

            if self.opts.with_C14:
                with self.ns('ibd'), detector.ns:
                    detector.c14 = ROOT.C14Spectrum(8, 5)
                    detector.c14.smear.inputs(inter_sum)
                    finalsum = detector.c14.smear

                    self.ns.addobservable("{0}_c14".format(detector.name),
                                          finalsum, export=True)
            else:
                finalsum = inter_sum

            with detector.ns:
                detector.eres = ROOT.EnergyResolution()
            detector.eres.matrix.Edges(self.integrator.points.xhist)
            detector.eres.smear.setLabel('Eres')
            detector.eres.smear.Ntrue(finalsum)
            finalsum = detector.eres.smear

            if detector.edges_final is not None:
                self.ns.addobservable("{0}_fine".format(detector.name), finalsum)

                detector.rebin = C.Rebin(detector.edges_final, 6)
                finalsum >> detector.rebin
                finalsum = detector.rebin
                self.ns.addobservable("{0}".format(detector.name), finalsum.single())
            else:
                self.ns.addobservable("{0}".format(detector.name), finalsum)

        det_ns = self.ns("detectors")(detector.name)

        for name in detector.intermediates:
            det_ns.addobservable(name, detector.intermediates[name], export=False)

        for name, bkg in detector.back_hists.items():
            det_ns.addobservable("bkg_{}".format(name), bkg, export=False)

        for pair, comps in self.oscprobs_comps.items():
            pair_ns = det_ns(pair[1].name)
            pair_ns.addobservable('oscprob', self._sumcomponents(comps, detector.name), export=False)

    @classmethod
    def reqparameters(cls, ns):
        """Setup the parameters"""
        gna.parameters.ibd.reqparameters(ns("ibd"))
        gna.parameters.oscillation.reqparameters(ns("oscillation"))

        for isoname, (central, sigma) in eperfission.items():
            isons = ns("isotopes")(isoname)
            isons.reqparameter("EnergyPerFission", central=central, sigma=sigma)
        for geo_isoname, (central, relsigma) in geo_flux_normalizations.items():
            geo_isons = ns("geo_isotopes")(geo_isoname)
            geo_isons.reqparameter("FluxNorm", central=central, relsigma=sigma)


#!/usr/bin/env python
"""Minimises the energy or the gap penalty function of a molecule in a cluster.

Template files are needed as well as an xyz file for the molecule and another
one for the surrounding molecules. Overall the use of subprocess is ugly as it
is repeated 3 or 4 times but it was found to handle memory better than Pool
when interfacing with Gaussian.

Energy units are Hartree inside the program but are printed in eV. Distances are
kept as Angstrom throughout and are converted from Bohr if necessary before
reaching this module

"""
import numpy as np
import subprocess
import os
from datetime import datetime
from scipy.optimize import minimize

from fromage.io import read_file as rf
from fromage.utils import array_operations as ao
from fromage.utils import calc
from fromage.io.parse_config_file import bool_cast

def sequence(in_pos):
    """
    Run Gaussian calculations in parallel and write and return results

    This function is designed to work with the scipy.optimise.minimize function.
    This is why it can only receive one array of floats as input and return two
    arrays of floats. As a result some variables in this function are defined
    elsewhere in the module which is a necessary evil.

    Parameters
    ----------
    in_pos : list of floats
        Input coordinates in array form
    Returns
    -------
    en_out : float
        Combined energy or penalty function value in Hartree
    gr_out : list of floats
        Gradients of en_out in Hartree/Angstrom

    References
    ----------
    Levine, B. G., Coe, J. D. & Martinez, T. J. Optimizing conical intersections
    without derivative coupling vectors: Application to multistate
    multireference second-order perturbation theory (MS-CASPT2).
    J. Phys. Chem. B 112, 405-413 (2008).

    """
    # initialise calculation objects
    rl = calc.setup_calc("rl", low_level)
    ml = calc.setup_calc("ml", low_level)
    mh = calc.setup_calc("mh", high_level)
    if bool_ci:
        mg = calc.setup_calc("mg", high_level)

    # Run the calculations as subprocesses with a maximum of 2 simultameous ones
    # at the same time. This order is optimised for the mh calculation being
    # the longest
    mh_proc = mh.run(ao.array2atom(mol_atoms, in_pos))
    if bool_ci and high_level != "gaussian_cas":
        mg_proc = mg.run(ao.array2atom(mol_atoms, in_pos))
    rl_proc = rl.run(ao.array2atom(mol_atoms, in_pos))
    rl_proc.wait()
    ml_proc = ml.run(ao.array2atom(mol_atoms, in_pos))
    ml_proc.wait()
    mh_proc.wait()
    if bool_ci and high_level != "gaussian_cas":
        mg_proc.wait()

    # read results. Each x_en_gr is a tuple (energy,gradients,scf_energy)
    rl_en_gr = rl.read_out(in_pos, mol_atoms, shell_atoms)
    ml_en_gr = ml.read_out(in_pos)

    if high_level == "gaussian_cas":
        mh_en_gr = mh.read_out(in_pos)[0:3]
        if bool_ci:
            mg_en_gr = (mh.read_out(in_pos)[2], mh.read_out(
                in_pos)[3], mh.read_out(in_pos)[2])
    elif high_level == "molcas":
        mh_en_gr = mh.read_out(in_pos)
        if bool_ci:
            # in molcas the first read out is S_1, gr, S_0 even if the target
            # state is 0
            mg_en_gr = mg.read_out(in_pos)[::-1]
    else:
        mh_en_gr = mh.read_out(in_pos)
        if bool_ci:
            mg_en_gr = mg.read_out(in_pos)

    # combine results
    en_combo = rl_en_gr[0] - ml_en_gr[0] + mh_en_gr[0]
    gr_combo = rl_en_gr[1] - ml_en_gr[1] + mh_en_gr[1]
    scf_combo = rl_en_gr[2] - ml_en_gr[2] + mh_en_gr[2]

    if bool_ci:
        # corresponding ground state energy and gradients
        en_combo_g = rl_en_gr[0] - ml_en_gr[0] + mg_en_gr[0]
        gr_combo_g = rl_en_gr[1] - ml_en_gr[1] + mg_en_gr[1]

        # Penalty function parameters and calculation
        alpha = 0.02
        e_mean = (en_combo + en_combo_g) / 2
        e_diff = en_combo - en_combo_g
        g_ij = e_diff**2 / (e_diff + alpha)
        en_out = e_mean + sigma * g_ij
        gr_out = 0.5 * (gr_combo + gr_combo_g) + sigma * ((e_diff**2 + 2 *
                                                           alpha * e_diff) / (e_diff + alpha)**2) * (gr_combo - gr_combo_g)
    else:
        en_out = en_combo
        gr_out = gr_combo

    # print some updates in the output
    out_file.write("------------------------------\n")
    global iteration
    iteration += 1
    out_file.write("Iteration: " + str(iteration) + "\n")
    out_file.write("Real low energy: {:>30.8f} eV\n".format(
        rl_en_gr[0] * evconv))
    out_file.write("Model low energy: {:>29.8f} eV\n".format(
        ml_en_gr[0] * evconv))
    out_file.write("Model high energy: {:>28.8f} eV\n".format(
        mh_en_gr[0] * evconv))
    out_file.write(
        "ONIOM Total energy: {:>27.8f} eV\n".format(en_combo * evconv))
    out_file.write(
        "ONIOM SCF energy: {:>29.8f} eV\n".format(scf_combo * evconv))
    out_file.write(
        "Energy grad. norm: {:>28.8f} eV/A\n".format(np.linalg.norm(gr_combo * evconv)))
    if bool_ci:
        out_file.write(
            "Penalty function value: {:>23.8f} eV\n".format(en_out * evconv))
        out_file.write("Penalty function grad. norm: {:>18.8f} eV\n".format(
            np.linalg.norm(gr_out * evconv)))

    out_file.write("Gap: {:>42.8f} eV\n".format(
        (en_combo - scf_combo) * evconv))
    out_file.flush()
    return (en_out, gr_out)

if __name__ == '__main__':

    evconv = 27.2114  # Something in Hartree * evconv = Something in eV

    # default settings

    def_inputs = {
        "mol_file": "mol.init.xyz",
        "shell_file": "shell.xyz",
        "out_file": "fromage.out",
        "bool_ci": "0",
        "high_level": "gaussian",
        "low_level": "gaussian",
        "sigma": "3.5",
        "gtol": 1e-5,
        "single_point": "0"}

    inputs = def_inputs.copy()

    # read user inputs
    if os.path.isfile("fromage.in"):
        new_inputs = rf.read_config("fromage.in")
        inputs.update(new_inputs)

    mol_file = inputs["mol_file"]
    shell_file = inputs["shell_file"]
    out_file = inputs["out_file"]
    bool_ci = bool_cast(inputs["bool_ci"])
    high_level = inputs["high_level"]
    low_level = inputs["low_level"]
    gtol = inputs["gtol"]
    single_point = bool_cast(inputs["single_point"])
    # sigma is called lambda in some papers but that is a bad variable name
    # in Python
    sigma = float(inputs["sigma"])
    # output
    out_file = open(out_file, "w", 1)
    # print start time
    start_time = datetime.now()
    out_file.write("STARTING TIME: " + str(start_time) + "\n")

    iteration = 0

    # clean up the last output
    if os.path.exists("geom_mol.xyz"):
        subprocess.call("rm geom_mol.xyz", shell=True)
    if os.path.exists("geom_cluster.xyz"):
        subprocess.call("rm geom_cluster.xyz", shell=True)

    # read initial coordniates
    mol_atoms = rf.read_xyz(mol_file)[0]

    # read shell atoms
    shell_atoms = rf.read_xyz(shell_file)[0]

    # make the initial coordinates into a flat list
    atoms_array = []
    for atom in mol_atoms:
        atoms_array.append(atom.x)
        atoms_array.append(atom.y)
        atoms_array.append(atom.z)

    # make the list into an array
    atoms_array = np.array(atoms_array)
    if single_point:
        sequence(atoms_array)
    else:
        res = minimize(sequence, atoms_array, jac=True,
                       options={'disp': True, 'gtol': gtol})

    out_file.write("DONE\n")
    end_time = datetime.now()
    out_file.write("ELAPSED TIME: " + str(end_time - start_time) + "\n")
    out_file.write("ENDING TIME: " + str(end_time) + "\n")
    out_file.close()


#!/usr/bin/env python
from scipy import *
import struct, sys, re
from scipy import linalg
import optparse
import rdU
from utils import W2kEnvironment, Ry_in_eV
import indmffile
import copy

class Struct:
    def __init__(self, case):
        self.case = case
        self.extn = 'struct'
        self.parse()

    def parse(self):
        '''The file is structured for reading by fortran code,
        so data is positioned by line and by column.'''
        f = open(self.case + '.' + self.extn, 'r')
        self.title = f.next().strip()

        line = f.next()
        self.lattice = line[0:4].strip()
        self.nat = int(line[27:30])
        self.latticename = line[30:].strip()

        self.mode = f.next()[13:17]

        line = f.next()
        self.a, self.b, self.c, self.alpha, self.beta, self.gamma = [float(line[i:i+10]) for i in range(0,60,10)]

        self.iatom  = []
        self.mult   = []
        self.isplit = []
        self.aname  = []
        self.npt    = []
        self.r0     = []
        self.rmt    = []
        self.Znuc   = []
        self.pos    = []
        self.rotloc = []

        for iat in range(self.nat):
            line = f.next()
            self.iatom.append( int(line[4:8]) )
            pos = [[float(line[col:col+10]) for col in 12+13*arange(3)]]

            line = f.next()
            mult = int(line[15:17])
            self.mult.append(mult)

            self.isplit.append( int(line[34:37]) )

            for mu in range(mult-1):
                line = f.next()
                pos.append( [float(line[col:col+10]) for col in 12+13*arange(3)] )

            self.pos.append(pos)

            line = f.next()
            self.aname.append( line[0:10].strip() )
            self.npt.append( int(line[15:20]) )
            self.r0.append( float(line[25:35]) )
            self.rmt.append( float(line[40:50]) )
            self.Znuc.append( int(float(line[55:65])) )

            rt=[]
            for i in range(3):
                line = f.next()
                rt.append( [float(line[col:col+10]) for col in 20+10*arange(3)] )
            self.rotloc.append(rt)

        self.Ng = int(f.next().split()[0])
        self.iz=[]
        self.tau=[]
        for i in range(self.Ng):
            tiz=zeros((3,3), dtype=float)
            ttau=zeros(3, dtype=float)
            for j in range(3):
                line = f.next()
                tiz[j,:] = array([int(line[0:2]), int(line[2:4]), int(line[4:6])])
                ttau[j] = float(line[6:16])
            self.iz.append(tiz)
            self.tau.append(ttau)
            f.next()
        #print self.iz

        f.close()

    def debugprint(self, f=sys.stdout):
        print >> f, '***** Structure File Contents *****'
        print >> f, 'title =', self.title
        print >> f, 'Number of sorts, nat =', self.nat
        print >> f, 'unit cell parameters (a,b,c) =', self.a, self.b, self.c
        print >> f, 'angles (alpha, beta, gamma) =', self.alpha, self.beta, self.gamma

        printlist = [
            (self.aname,  'Atom name, aname ='),
            (self.iatom,  'Atom index, iatom ='),
            (self.mult,   'Number of atoms of this type, mult ='),
            (self.isplit, 'Symmetry of the atom, isplit ='),
            (self.npt,    'Number of radial points, npt ='),
            (self.r0,     'First point in radial mesh, r0 ='),
            (self.rmt,    'Muffin tin radius, rmt ='),
            (self.Znuc,   'Atomic number, Znuc ='),
            ]

        for i in range(self.nat):
            print >> f, '---------- atom type', i, '------------'

            for var,docstr in printlist:
                print >> f, docstr, var[i]

            print >> f, 'Position(s) inside the unit cell:'
            for m in range(self.mult[i]):
                print >> f, '    pos =', self.pos[i][m]

            print >> f, 'Local rotation matrix:'
            for row in self.rotloc[i]:
                print >> f, row

        print >> f, '***********************************'

    def flat(self, notflat):
        '''Return a flat view of given data as a list.
        Example: if w.mult = [2,4] and w.aname = ['V', 'O']
        w.flatten(w.aname) -> ['V', 'V', 'O', 'O', 'O', 'O']'''
        if notflat is self.pos:
            listoflists = self.pos
        else:
            listoflists = [[elem]*mult for elem,mult in zip(notflat, self.mult)]
        return reduce(operator.add, listoflists)

def DirectVectors(BR,RN):
    #v0 = cross(BR[1],BR[2])
    #v1 = cross(BR[2],BR[0])
    #v2 = cross(BR[0],BR[1])
    #vol = dot(v0,BR[0])
    #array([v0/vol,v1/vol,v2/vol])
    return dot(RN, linalg.inv(BR).transpose() )

def Print(fh, R, tHr):
    R0 = R # dot(DVinv,R)
    Rs=[]
    for i in range(3):
        if abs(round(R0[i])-R0[i])<1e-3:
            Rs.append( "%d" % int(round(R0[i])) )
        else:
            Rs.append("%.1f" % R0[i])

    print >> fh, 'Hopping[(%s,%s,%s)]=[' % tuple(Rs)
    for l1 in range(shape(tHr)[0]):
        print >> fh, '[',
        for l2 in range(shape(tHr)[1]):
            print >> fh, "%8.4f+%8.4fj," % (tHr[l1,l2].real, tHr[l1,l2].imag),
        print >> fh, '],'
    print >> fh, ']'

def ReadBRs(case):
    "reads BR1 and BR2 from case.rotlm"
    fr = open(case+'.rotlm')
    BR = zeros((2,3,3), dtype=float)
    for ii in range(2):
        fr.next()
        for i in range(3):
            BR[ii,:,i] = map(float,fr.next().split())

    BRX = array(linalg.inv(BR[1]) * matrix(BR[0]))
    return (BR,BRX)

def FindMaxMinBand(Ek, Emin, Emax):
    "Finds minimum and maximum band to be used in downfolding"

    istart=None
    iend=None
    for i in range(len(Ek)):
        if Ek[i]>Emin:
            istart = i
            break

    for i in range(istart,len(Ek)):
        if Ek[i]>Emax:
            iend = i
            break
    return array([istart, iend])

def TryAddBands(Nse_global, Ek, Emin, Emax, SmearN, SmearX):

    Nse = Nse_global[:]

    Em = Ek[Nse[0]-1] # closest energy below the cutoff
    Ep = Ek[Nse[1]]   # closest energy above the cutoff

    print Em, Ep

    EmCanAdd, EpCanAdd = False, False

    if Emin-Em < SmearN : EmCanAdd = True
    if Ep-Emax < SmearX : EpCanAdd = True

    Succ = True

    if EmCanAdd and EpCanAdd:
        if Emin-Em < Ep-Emax:
            Nse[0]-=1
            print 'Both could be added. Adding below.'
        else:
            Nse[1]+=1
            print 'Both could be added. Adding above.'
    elif EmCanAdd :
        Nse[0]-=1
        print 'Can add below'
    elif EpCanAdd :
        Nse[1]+=1
        print 'Can add above'
    else:
        print 'Can not find enough bands in your interval. Increase Emin, Emax or smearings'
        Succ = False

    print Nse

    return (Nse, Succ)

# RS = (i0,i1,i2)*DV[:,:]
# RS*DV^-1 = (i0,i1,i2)

def GiveRs(div, DV):
    "Gives real space vectors"
    Rs=[]
    for i0 in range(div[0]/2,-div[0]/2,-1):
        for i1 in range(div[1]/2,-div[1]/2,-1):
            for i2 in range(div[2]/2,-div[2]/2,-1):
                Rs.append(DV[0]*i0 + DV[1]*i1 + DV[2]*i2)
    return Rs

if __name__ == '__main__':
    """ Takes the DMFT projector Udmft.0 and LDA eigenvalues (case.energy) to produces downfolding hopping parameters.
    It also requires two parameters specifying the bands used in downfolding. The lower cutoff is determined by energy
    '--Emin' (counted from EF in eV), while the upped cutoff is determined by the number of bands '--nbnd'.
    """

    usage = """usage: %prog [ options ]

    The script takes the DMFT projector Udmft.0 and LDA eigenvalues (case.energy)
    and produces downfolded hopping parameters. It requires parameters
    specifying bands used in downfolding. The lower/upper cutoff is determined
    by energy '--Emin' or '-N' /'--Emax' or '-X' (counted from EF in eV).
    Additional bands are added if needed in the interval
    '--SmearN' or '-n' / '--SmerX' or '-x'

    --------------------   Emax+SmearX
      added if needed
    ---------------------  Emax

      bands always used in downfolding

    ---------------------  Emin
      added if needed
    ---------------------  Emin-SmearN
    """

    # Finds what is case
    w2k = W2kEnvironment()

    parser = optparse.OptionParser(usage)
    parser.add_option("-N", "--Emin",  dest="Emin", type="float", default=None, help="Bands above Emin (counted from EF in eV) will be consider in downfolding.")
    parser.add_option("-X", "--Emax",  dest="Emax", type="float", default=None, help="Bands below Emax (counted from EF in eV) will be consider in downfolding.")
    parser.add_option("-n", "--SmearN",  dest="SmearN", type="float", default=1., help="When determining Emin cutoff, we use smearing in eV if needed")
    parser.add_option("-x", "--SmearX",  dest="SmearX", type="float", default=1., help="When determining Emax cutoff, we use smearing in eV if needed")
    parser.add_option("-s", "--smalls",  dest="smalls", type="float", default=0.1, help="The smallest acceptable singular value")
    # Next, parse the arguments
    (options, args) = parser.parse_args()

    if options.Emin is None:
        print 'You have to specify minimum band energy! Option -E or --Emin'
        sys.exit(1)
    if options.Emax is None:
        print 'You have to specify maximum band energy! Option -M or --Emax'
        sys.exit(1)

    print 'case=%s Emin=%s, Emax=%s SmearN=%s SmearX=%s' %  (w2k.case, options.Emin, options.Emax, options.SmearN, options.SmearX)

    # File handlers to exchange files with Fortran
    # These numbers are not relevant, but need to be set
    fhp = 233
    fhe = 60
    Ntry=10

    # Reads indmf-file
    inl = indmffile.Indmfl(w2k.case)
    inl.read()
    EF = inl.EF
    print 'Fermi energy is ', EF
    # atoms in the projection for downfolding

    atms = inl.atoms.keys()
    print 'atoms in the projection for downfolding=', atms
    # Finds actual positions of these atoms
    pos = [] # zeros((max(atms),3),dtype=float)
    fi = open(w2k.case+'.outputdmfu','r')
    for line in fi:
        m = re.search('Actual position of atom *(\d+)',line)
        if m is not None:
            if int(m.group(1)) in atms:
                atom = int(m.group(1))

                print 'ATOM', atom, ' in atms=', atms

                m = re.search('Actual position of atom *(\d+) *is: (.*)',line)
                #pos[atom-1,:] = map(float,m.group(2).split())
                pos.append( map(float,m.group(2).split()) )
    pos=array(pos)
    print 'Actual position of atoms for downfolding=',  pos.tolist()

    strc = Struct(w2k.case)
    strc.parse()

    # reads BR1 and BR2 from case.rotlm
    (BR,BRX) = ReadBRs(w2k.case)
    # Computes direct vectors, knowing the reciprocal basis
    DV = DirectVectors(BR[1],BR[0])

    print 'Direct vectors:'
    for i in range(3):
        print ("%7.3f "*3) % tuple(DV[i])

    print 'Please enter the direct vectors for large Brillouin zone'
    DVn = zeros(1)
    while shape(DVn) != (3,3):
        DVn = array(eval(sys.stdin.readline()))
    #DVn = array([[0.5,0.5,0],[0.5,-0.5,0],[0,0,1]])
    print 'But switching to direct vectors of the one atom unit cell:'
    for i in range(3):
        print ("%7.3f "*3) % tuple(DVn[i])

    # Reads div from case.klist
    firstline = open(w2k.case+'.klist').next()
    m = re.search('div: *\((\s*\d+\s*\d+\s*\d+)\)', firstline)
    if m is not None:
        div = map(int,m.groups()[0].split())

    Rs = GiveRs(div, DVn)
    print 'Real space vectors used in the Fourier transform to tigh-binding:'
    for i,R in enumerate(Rs):
        print "%3d"%i, "%7.3f %7.3f %7.3f" % tuple(R)

    # Read Udmft
    filename = 'Udmft.0'
    (nkp, nsymop, norbitals) = rdU.fopen(fhp, filename)

    nindo = rdU.read1(fhp, norbitals)

    print 'nkp=', nkp, 'nsymop=', nsymop, 'norbitals=', norbitals, 'nindo=', nindo

    delta_A_B = pos[1]-pos[0]
    print 'delta_A_B=', delta_A_B

    rdU.feopen(fhe, w2k.case+'.energy', strc.nat)

    Hk=[]
    kps=[]
    wgh=[]
    for ikp in range(nkp):

        (iikp, nbands, tmaxdim2, tnorbitals, nemin) = rdU.read2(fhp)
        (k, kname, wg, ios, nen) = rdU.feread1(fhe)

        print 'delta*k=', delta_A_B*k

        if ios!=0:
            print 'Number of k-points in case.energy is smaller then in Udfmt.0'
            break

        # Read LDA energies from case.energy
        Ek = rdU.feread2(fhe, nen) * Ry_in_eV - EF

        # finds minimum and maximum band for downfolding
        #(bnd_s, bnd_e) = FindMaxMinBand(Ek, options.Emin, nemin, options.nbnd, options.dnbnd)
        #e = Ek[nemin+bnd_s-1:nemin+bnd_e-1]
        #print 'bnd_s=', bnd_s, 'bnd_e=', bnd_e, 'Emm=', e[0], e[-1]
        #print 'k=', k

        for isym in range(nsymop):

            kn = dot(strc.iz[isym],k)
            k_nat = [sum(BRX[i,:]*kn) for i in range(3)]

            kps.append(kn)
            wgh.append(wg)

            iisym = rdU.read3(fhp)
            if iisym!=isym+1:
                print 'ERROR: isym!=iisym', isym, iisym

            for iorb in range(norbitals):

                U = []
                for ind in range(nindo[iorb]):
                    Udat = rdU.read4(fhp, nbands)
                    U.append( Udat )
                U = array(U)

                if iorb==0:
                    Utot = U
                else:
                    Utot = vstack((Utot,U))

            # finds minimum and maximum band for downfolding
            Nse = FindMaxMinBand(Ek, options.Emin, options.Emax)

            print 'len(Utot)=', len(Utot)

            while (Nse[1]-Nse[0] < len(Utot) ):
                print 'Warning: Number of bands in the interval is ', Nse[1]-Nse[0], 'while number of orbitals is ', len(Utot)
                (Nse, Succ) = TryAddBands(Nse, Ek, options.Emin, options.Emax, options.SmearN, options.SmearX)
                if not Succ: sys.exit(1)

            NsingularValues=0
            while (NsingularValues<len(Utot)):
                (bnd_s,bnd_e) = Nse-(nemin-1)
                e = Ek[nemin-1+bnd_s:nemin-1+bnd_e]
                print 'bnd_s=', bnd_s, 'bnd_e=', bnd_e, 'n=', bnd_e-bnd_s, 'Emm=', e[0], e[-1]
                # Performing SVD on projector
                (u, s, v) = linalg.svd(Utot[:,bnd_s:bnd_e])
                print 'singular-values:', ikp, s.tolist()
                NsingularValues = len(filter(lambda x: x>options.smalls, s))
                print 'NsingularValues=',  NsingularValues

                if (NsingularValues<len(Utot)):
                    (Nse, Succ) = TryAddBands(Nse, Ek, options.Emin, options.Emax, options.SmearN, options.SmearX)
                    if not Succ: sys.exit(1)

            # downfolding transformation
            Rt = dot(u[:,:len(s)],v[:len(s),:])
            # downfolded hamiltonian
            tHk = dot(conjugate(Rt) * e, transpose(Rt))

            N0 = len(Utot)/2
            print 'N0=', N0

            zHk = copy.deepcopy(tHk[:N0,:N0])
            zHk += tHk[:N0,:N0]
            zHk += tHk[:N0,N0:]*exp(2*pi*1j*sum(delta_A_B*kn) ) # 1,2
            zHk += tHk[N0:,:N0]*exp(-2*pi*1j*sum(delta_A_B*kn) )# 2,1
            zHk *= 0.5
            Hk.append(zHk)

    rdU.fclose(fhp)
    rdU.feclose(fhe)

    Hk = array(Hk)

    nrm = sum(wgh)
    wgh = wgh/nrm

    # Fourier transform to produce hopping parameters
    Hr=[]
    for R in Rs:
        tHr=zeros(shape(Hk[0]),dtype=complex)
        dsum=0j
        for ik in range(len(kps)):
            expk = exp(1j*2*pi*dot(kps[ik],R)) * wgh[ik]
            tHr += expk * Hk[ik]
            dsum += expk
        print 'R=[', "%4.1f, "*3 % tuple(R), '] exp=', "%10.5f" % abs(dsum)
        Hr.append(tHr)

    # Printing of the Hamiltonian
    fh = open('hamiltonian.dat', 'w')

    print >> fh, '# Tighbinding Hamiltonian with entries: ', inl.legends
    print >> fh
    #print >> fh, "lattice_unit=[", tuple(DVn[0]), ',', tuple(DVn[1]), ',', tuple(DVn[2]), "]"
    print >> fh, "lattice_unit=", identity(3).tolist()

    print >> fh, "orbital_positions=",zeros((3,3)).tolist()
    print >> fh
    #print '['
    #for atm in inl.siginds.keys():
    #    for i in range(len(inl.siginds[atm])):
    #        print >> fh, pos[atm-1].tolist(),','
    #print
    #print >> fh, ']\n'

    # sorting such that first come larger hoppings
    for irm,R in enumerate(Rs):
        if sum(abs(Rs[irm]))<1e-10: break

    ir_index = range(len(Rs))
    ir_index.sort( lambda a,b: cmp(sum(abs(Hr[b])), sum(abs(Hr[a]))) )
    ir0 = ir_index.index(irm)
    ir_index = [irm] + ir_index[:ir0] + ir_index[ir0+1:]

    #print ir_index

    DVn_inv = linalg.inv(DVn)
    print >> fh, 'Hopping={}'
    for ir in ir_index :
        # RS*DV^-1 = (i0,i1,i2)
        Rn = dot(Rs[ir] ,DVn_inv)
        Print(fh, Rn, Hr[ir])
        print >> fh

    fh.close()


# Licensed under a 3-clause BSD style license - see LICENSE.rst
"""Model an instrument response for spectroscopic simulations.

An instrument model is usually initialized from a configuration used to create
a simulator and then accessible via its ``instrument`` attribute, for example:

    >>> import specsim.simulator
    >>> simulator = specsim.simulator.Simulator('test')  # doctest: +IGNORE_OUTPUT
    >>> print(np.round(simulator.instrument.fiber_diameter, 1))
    107.0 um

See :doc:`/api` for examples of changing model parameters defined in the
configuration. No attributes can be changed after a simulator has
been created.  File a github issue if you would like to change this.

An :class:`Instrument` includes one or more
:class:`Cameras <specsim.camera.Camera>`.
"""
from __future__ import print_function, division

import numpy as np
import os.path
import scipy.interpolate
import scipy.integrate

import astropy.constants
import astropy.units as u

import specsim.camera
import specsim.fastfiberacceptance
import specsim.config

class Instrument(object):
    """Model the instrument response of a fiber spectrograph.

    A spectrograph can have multiple :mod:`cameras <specsim.camera>` with
    different wavelength coverages. Objects representing each camera are
    contained in a list accessible from our ``cameras`` attribute, which will
    be in order of increasing effective wavelength.

    No instrument attributes can be changed after an instrument has been
    created. Create a github issue if you would like to change this.

    Parameters
    ----------
    name : str
        Descriptive name of this instrument.
    wavelength : astropy.units.Quantity
        Array of wavelength bin centers where the instrument response is
        calculated, with units.
    fiberloss_method : str
        Must be "table" or "galsim" or "fastsim".  Specifies how fiber acceptance fractions
        will be loaded or calculated.
    fiber_acceptance_dict : dict or None
        Dictionary of fiber acceptance fractions tabulated for different
        source models, with keys corresponding to source model names.
        Ignored when fiberloss_method is "galsim".
    fast_fiber_acceptance : specsim.fastfiberacceptance.FastFiberAcceptance or None
        Initialized instance to use when fiberloss_method is "fastsim".
        Ignored for other values of fiberloss_method.
    fiberloss_num_wlen : int
        Number of wavelengths where the fiberloss fraction should be tabulated
        for interpolation.  Ignored when fiberloss_method is not "galsim".
    fiberloss_num_pixels : int
        Number of pixels used to subdivide the fiber diameter for
        numerical convolution and integration calculations.
        Ignored when fiberloss_method  is not "galsim".
    blur_function : callable
        Function of field angle and wavelength that returns the corresponding
        RMS blur in length units (e.g., microns).
    offset_function : callable
        Function of focal-plane position (x,y) in angular units and wavelength
        that returns the corresponding radial centroid offset in length
        units (e.g., microns).
    cameras : list
        List of :class:`specsim.camera.Camera` instances representing the
        camera(s) of this instrument.
    primary_mirror_diameter : astropy.units.Quantity
        Diameter of the primary mirror, with units.
    obscuration_diameter : astropy.units.Quantity
        Diameter of a central obscuration of the primary mirror, with units.
    support_width : astropy.units.Quantity
        Width of the obscuring supports, with units.
    fiber_diameter : astropy.units.Quantity
        Physical diameter of the simulated fibers, with units of length.
        Converted to an on-sky diameter using the plate scale.
    field_radius : astropy.units.Quantity
        Maximum radius of the field of view in length units measured at
        the focal plane. Converted to an angular field of view using the
        plate scale.
    radial_scale : callable
        Callable function that returns the plate scale in the radial
        (meridional) direction (with appropriate units) as a function of
        focal-plane distance (with length units) from the boresight.
    azimuthal_scale : callable
        Callable function that returns the plate scale in the azimuthal
        (sagittal) direction (with appropriate units) as a function of
        focal-plane distance (with length units) from the boresight.
    """
    def __init__(self, name, wavelength, fiberloss_method,
                 fiber_acceptance_dict, fast_fiber_acceptance, fiberloss_num_wlen,
                 fiberloss_num_pixels, blur_function, offset_function, cameras,
                 primary_mirror_diameter, obscuration_diameter, support_width,
                 fiber_diameter, field_radius, radial_scale, azimuthal_scale):
        self.name = name
        self._wavelength = wavelength
        self.fiber_acceptance_dict = fiber_acceptance_dict
        self.fast_fiber_acceptance = fast_fiber_acceptance
        # Both fiber_acceptance_dict and fast_fiber_acceptance must be initialized
        # before assigning to fiberloss_method (since its setter checks their values).
        self.fiberloss_method = fiberloss_method
        self.fiberloss_num_wlen = fiberloss_num_wlen
        self.fiberloss_num_pixels = fiberloss_num_pixels
        self._blur_function = blur_function
        self._offset_function = offset_function
        self.cameras = cameras
        self.primary_mirror_diameter = primary_mirror_diameter
        self.obscuration_diameter = obscuration_diameter
        self.support_width = support_width
        self.fiber_diameter = fiber_diameter
        self.field_radius = field_radius
        self.radial_scale = radial_scale
        self.azimuthal_scale = azimuthal_scale

        # Calculate the effective area of the primary mirror.
        D = self.primary_mirror_diameter
        obs = self.obscuration_diameter
        support_area = 0.5*(D - obs) * self.support_width
        self.effective_area = (
            np.pi * ((0.5 * D) ** 2 - (0.5 * obs) ** 2) - 4 * support_area)

        # Tabulate the mapping between focal plane radius and boresight
        # opening angle by integrating the radial plate scale.
        # Use mm and radians as the canonical units.
        self._radius_unit, self._angle_unit = u.mm, u.rad
        radius = np.linspace(
            0., self.field_radius.to(self._radius_unit).value, 1000)
        dradius_dangle = self.radial_scale(radius * self._radius_unit).to(
            self._radius_unit / self._angle_unit).value
        angle = scipy.integrate.cumtrapz(
            1. / dradius_dangle, radius, initial=0.)

        # Record the maximum field angle corresponding to our field radius.
        self.field_angle = angle[-1] * self._angle_unit

        # Build dimensionless linear interpolating functions of the
        # radius <-> angle map using the canonical units.
        self._radius_to_angle = scipy.interpolate.interp1d(
            radius, angle, kind='linear', copy=True, bounds_error=True)
        self._angle_to_radius = scipy.interpolate.interp1d(
            angle, radius, kind='linear', copy=True, bounds_error=True)

        # Calculate the energy per photon at each wavelength.
        hc = astropy.constants.h * astropy.constants.c
        energy_per_photon = (hc / self._wavelength).to(u.erg)

        # Calculate the rate of photons incident on the focal plane per
        # wavelength bin per unit spectral flux density. The fiber acceptance
        # fraction is not included in this calculation.
        wavelength_bin_size = np.gradient(self._wavelength)
        self.photons_per_bin = (
            self.effective_area * wavelength_bin_size / energy_per_photon
            ).to((u.cm**2 * u.Angstrom) / u.erg)

        wave_mid = []
        for i, camera in enumerate(self.cameras):
            wave_min, wave_max = camera.wavelength_min, camera.wavelength_max
            wave_mid.append(0.5 * (wave_min + wave_max))
            if i == 0:
                self.wavelength_min = wave_min
                self.wavelength_max = wave_max
            else:
                self.wavelength_min = min(self.wavelength_min, wave_min)
                self.wavelength_max = max(self.wavelength_max, wave_max)

        # Sort cameras in order of increasing wavelength.
        self.cameras = [x for (y, x) in sorted(zip(wave_mid, self.cameras))]

    @property
    def fiberloss_method(self):
        """The current method used to calculate fiber acceptance fractions.
        """
        return self._fiberloss_method

    @fiberloss_method.setter
    def fiberloss_method(self, fiberloss_method):
        """Set the method used to calculate fiber acceptance fractions.

        Must be one of "table" or "galsim" or "fastsim".
        """
        if fiberloss_method not in ('table', 'galsim', 'fastsim' ):
            raise ValueError(
                'fiberloss_method must be "table" or "galsim" or "fastsim".')
        if fiberloss_method == 'table' and self.fiber_acceptance_dict is None:
            raise ValueError('Missing required instrument.fiberloss.table.')
        if fiberloss_method == 'fastsim' and self.fast_fiber_acceptance is None:
            raise ValueError(
                'Missing required instrument.fiberloss.fast_fiber_acceptance_path.')
        if fiberloss_method == 'galsim':
            try:
                import galsim
            except ImportError:
                raise ValueError('The galsim package is not installed.')
        self._fiberloss_method = fiberloss_method

    def field_radius_to_angle(self, radius):
        """Convert focal plane radius to an angle relative to the boresight.

        The mapping is derived from the radial (meridional) plate scale
        function :math:`dr/d\\theta(r)` via the integral:

        .. math::

            \\theta(r) = \\int_0^{r} \\frac{dr}{dr/d\\theta(r')}\\, dr'

        The input values must be within the field of view.
        Use :meth:`field_angle_to_radius` for the inverse transform.

        Parameters
        ----------
        radius : astropy.units.Quantity
            One or more radius values where the angle should be calculated.
            Values must be between 0 and ``field radius``.

        Returns
        -------
        astropy.units.Quantity
            Opening angle(s) relative to the boresight corresponding to
            the input radius value(s).

        Raises
        ------
        ValueError
            One or more input values are outside the allowed range.
        """
        return self._radius_to_angle(
            radius.to(self._radius_unit)) * self._angle_unit

    def field_angle_to_radius(self, angle):
        """Convert focal plane radius to an angle relative to the boresight.

        The mapping :math:`r(\\theta)` is calculated by numerically inverting
        the function :math:`\\theta(r)`.

        The input values must be within the field of view.
        Use :meth:`field_radius_to_angle` for the inverse transform.

        Parameters
        ----------
        angle : astropy.units.Quantity
            One or more angle values where the radius should be calculated.
            Values must be between 0 and ``field_angle``.

        Returns
        -------
        astropy.units.Quantity
            Radial coordinate(s) in the focal plane corresponding to the
            input angle value(s).

        Raises
        ------
        ValueError
            One or more input values are outside the allowed range.
        """
        return self._angle_to_radius(
            angle.to(self._angle_unit)) * self._radius_unit

    def get_blur_rms(self, wavelength, angle):
        """Get the instrument PSF blur at the specified field angle.

        Parameters
        ----------
        wavelength : astropy.units.Quantity
            Wavelength where the blur should be calculated.
        angle : astropy.units.Quantity
            Angular separation from the field center.

        Returns
        -------
        astropy.units.Quantity
            RMS blur of the instrument at this wavelength and field radius
            in length units.
        """
        return self._blur_function(angle, wavelength)

    def get_centroid_offset(self, angle_x, angle_y, wavelength):
        """Get the instrument centroid offset at the specified field angles.

        This method does not make any assumptions about how the x and y
        axes are defined, as long as (0, 0) is the field center.

        Note that the focal-plane position is input as angles relative to
        the field center, while the offsets are returned as lengths relative
        to the nominal fiber center.

        Parameters
        ----------
        angle_x : astropy.units.Quantity
            Angular separation from the field center along x.
        angle_y : astropy.units.Quantity
            Angular separation from the field center along y.
        wavelength : astropy.units.Quantity
            Wavelength where the blur should be calculated.

        Returns
        -------
        tuple
            Tuple (dx, dy) of astropy quantities giving the spot centroid
            offset components at this wavelength and position in the focal
            plane.  Offsets are given in length units, e.g., microns.
        """
        return self._offset_function(angle_x, angle_y, wavelength)

    def get_focal_plane_optics(self, focal_x, focal_y, wlen_grid):
        """Calculate the optical parameters at a set of focal-plane positions.

        Uses :meth:`get_centroid_offset`, :meth:`get_blur_rms`, and
        :meth:`field_radius_to_angle` to calculate the optics at each focal
        plane location.

        This method does not make any assumptions about how the x and y
        axes are defined, as long as (0, 0) is the field center.  However
        radial symmetry is broken by the (dx, dy) offsets calculated by
        :meth:`get_centroid_offset`.

        Note that units are required for the input arrays and included with
        the returned arrays.

        Parameters
        ----------
        focal_x : :class:`astropy.units.Quantity`
            1D array of X coordinates in the focal plane relative to the
            boresight, with length units.
        focal_y : :class:`astropy.units.Quantity`
            1D array of Y coordinates in the focal plane relative to the
            boresight, with length units.
        wlen_grid : :class:`astropy.units.Quantity`
            1D array of wavelengths where parameters should be tabulated,
            with length units.

        Returns
        -------
        tuple
            Tuple of arrays scale, blur, offset with shapes (N,2), (N,M) and
            (N,M,2) where N is the size of the 1D input (x,y) arrays, M is
            the size of the input wavelength grid, and axes of length 2
            correspond to radial and azimuthal axes (not the input x,y!).
            All output arrays have units.
        """
        # Check for valid units on the input arrays.
        try:
            focal_x_mm = focal_x.to(u.mm).value
            focal_y_mm = focal_y.to(u.mm).value
            wlen_grid_ang = wlen_grid.to(u.Angstrom).value
        except astropy.units.UnitConversionError:
            raise ValueError('Input arrays have invalid units.')
        except AttributeError:
            raise ValueError('Input arrays are missing required units.')

        # Check for expected input array shapes.
        if len(focal_x_mm.shape) != 1 or len(wlen_grid_ang.shape) != 1:
            raise ValueError('Input arrays must be 1D.')
        if focal_x_mm.shape != focal_y_mm.shape:
            raise ValueError('Input (x,y) arrays have different shapes.')

        # Allocate output arrays.
        n_xy = len(focal_x_mm)
        n_wlen = len(wlen_grid_ang)
        scale = np.empty((n_xy, 2))
        blur = np.empty((n_xy, n_wlen))
        offset = np.empty((n_xy, n_wlen, 2))

        # Convert x, y offsets in length units to field angles.
        focal_r = np.sqrt(focal_x**2+focal_y**2)
        angle_r = self.field_radius_to_angle(focal_r)
        angle_x = np.zeros(focal_x.shape) * angle_r.unit
        angle_y = np.zeros(focal_y.shape) * angle_r.unit
        positive_radius = focal_r>0
        angle_x[positive_radius] = (
            angle_r[positive_radius] / focal_r[positive_radius]
            ) * focal_x[positive_radius]
        angle_y[positive_radius] = (
            angle_r[positive_radius] / focal_r[positive_radius]
            ) * focal_y[positive_radius]

        # Calculate the radial and azimuthal plate scales at each location.
        scale[:, 0] = self.radial_scale(focal_r).to(u.um / u.arcsec).value
        scale[:, 1] = self.azimuthal_scale(focal_r).to(u.um / u.arcsec).value

        # Calculate the transformations between polar and Cartesian coordinates.
        phi = np.arctan2(focal_y_mm, focal_x_mm)
        cos_phi = np.cos(phi)
        sin_phi = np.sin(phi)

        # Lookup the instrument blur and centroid offset at each
        # wavelength for this focal-plane position.
        for i, wlen in enumerate(wlen_grid):
            # Lookup the RMS blurs in focal-plane microns.
            blur[:, i] = self.get_blur_rms(wlen, angle_r).to(u.um).value
            # Lookup the radial centroid offsets in focal-plane microns.
            dx, dy = self.get_centroid_offset(angle_x, angle_y, wlen)
            dx_um = dx.to(u.um).value
            dy_um = dy.to(u.um).value
            # Rotate to polar coordinates.
            offset[:, i, 0] =  cos_phi * dx_um + sin_phi * dy_um
            offset[:, i, 1] =  -sin_phi * dx_um + cos_phi * dy_um

        return scale * (u.um / u.arcsec), blur * u.um, offset * u.um

    def plot_field_distortion(self):
        """Plot focal plane distortions over the field of view.

        Requires that the matplotlib package is installed.
        """
        import matplotlib.pyplot as plt

        # Tabulate the field radius - angle mapping.
        radius = np.linspace(0., self.field_radius.to(u.mm).value, 500) * u.mm
        angle = self.field_radius_to_angle(radius).to(u.deg)

        # Calculate the r**2 weighted mean inverse radial scale by minimizing
        # angle - mean_inv_radial_scale * radius with respect to
        # mean_inv_radial_scale.
        mean_inv_radial_scale = (
            np.sum(radius ** 3 * angle) / np.sum(radius ** 4))
        mean_radial_scale = (1. / mean_inv_radial_scale).to(u.um / u.arcsec)

        # Calculate the angular distortion relative to the mean radial scale.
        distortion = (angle - radius * mean_inv_radial_scale).to(u.arcsec)

        # Eliminate round off error so that the zero distortion case is
        # correctly recovered.
        distortion = np.round(distortion, decimals=5)

        # Calculate the fiber area as a function of radius.
        radial_size = (
            0.5 * self.fiber_diameter / self.radial_scale(radius))
        azimuthal_size = (
            0.5 * self.fiber_diameter / self.azimuthal_scale(radius))
        fiber_area = (np.pi * radial_size * azimuthal_size).to(u.arcsec ** 2)

        # Calculate the r**2 weighted mean fiber area.
        mean_fiber_area = np.sum(radius ** 2 * fiber_area) / np.sum(radius ** 2)

        # Calculate the dimensionless fiber area ratio.
        fiber_area_ratio = (fiber_area / mean_fiber_area).si.value

        # Calculate the dimensionless ratio of azimuthal / radial plate scales
        # which is the ratio of the on-sky radial / azimuthal extends.
        shape_ratio = (self.azimuthal_scale(radius) /
                       self.radial_scale(radius)).si.value

        # Make the plots.
        fig, (ax1, ax2) = plt.subplots(2, sharex=True, figsize=(8, 8))

        ax1.plot(angle, distortion, 'b-', lw=2)
        ax1.set_ylabel('Field angle distortion [arcsec]', fontsize='large')
        ax1.set_xlim(0., self.field_angle.to(u.deg).value)
        ax1.grid()

        ax1.axhline(0., color='r')
        xy = 0.5 * self.field_angle.to(u.deg).value, 0.
        label = '{0:.1f}'.format(mean_radial_scale)
        ax1.annotate(label, xy, xy, color='r', horizontalalignment='center',
                     verticalalignment='bottom', fontsize='large')

        ax2.plot(angle, fiber_area_ratio, 'b', lw=2, label='Area ratio')
        ax2.plot(angle, shape_ratio, 'k', lw=2, ls='--',
                 label='Radial/azimuthal')
        ax2.set_ylabel('Fiber sky area and shape ratios', fontsize='large')
        ax2.grid()
        ax2.legend(loc='upper right')

        ax2.axhline(1., color='r')
        xy = 0.5 * self.field_angle.to(u.deg).value, 1.
        label = '{0:.3f}'.format(mean_fiber_area)
        ax2.annotate(label, xy, xy, color='r', horizontalalignment='center',
                     verticalalignment='bottom', fontsize='large')

        ax2.set_xlabel('Field angle [deg]', fontsize='large')
        plt.subplots_adjust(
            left=0.10, right=0.98, bottom=0.07, top=0.97, hspace=0.05)

    def plot(self, flux=1e-17 * u.erg / (u.cm**2 * u.s * u.Angstrom),
             exposure_time=1000 * u.s, cmap='nipy_spectral'):
        """Plot a summary of this instrument's model.

        Requires that the matplotlib package is installed.

        Parameters
        ----------
        flux : astropy.units.Quantity
            Constant source flux to use for displaying the instrument response.
        exposure_time : astropy.units.Quantity
            Exposure time to use for displaying the instrument response.
        cmap : str or matplotlib.colors.Colormap
            Matplotlib colormap name or instance to use for displaying the
            instrument response.  Colors are selected for each camera
            according to its central wavelength, so a spectral color map
            will give reasonably intuitive results.
        """
        import matplotlib.pyplot as plt
        import matplotlib.cm as cm

        fig, (ax1, ax2) = plt.subplots(2, sharex=True, figsize=(8, 8))
        ax1_rhs = ax1.twinx()
        ax2_rhs = ax2.twinx()
        cmap = cm.get_cmap(cmap)

        wave = self._wavelength.value
        wave_unit = self._wavelength.unit
        dwave = np.gradient(wave)

        if self.fiber_acceptance_dict:
            for source_type in self.fiber_acceptance_dict:
                # Plot fiber acceptance fractions without labels.
                ax1.plot(wave, self.fiber_acceptance_dict[source_type], 'k--')
        for camera in self.cameras:
            cwave = camera._wavelength

            # Use an approximate spectral color for each band.
            mid_wave = 0.5 * (camera.wavelength_min + camera.wavelength_max)
            color = cmap(
                (mid_wave - self.wavelength_min) /
                (self.wavelength_max - self.wavelength_min))

            # Calculate number of photons with perfect fiber acceptance.
            nphot = (flux * self.photons_per_bin * exposure_time *
                     camera.throughput / dwave)
            dark_noise = np.sqrt(
                (camera.dark_current_per_bin * exposure_time).value)
            total_noise = np.sqrt(
                dark_noise ** 2 + camera.read_noise_per_bin.value ** 2)

            ax1.plot(cwave, camera.throughput, ls='-', color=color)

            ax1_rhs.plot(cwave, nphot.value, ls=':', color=color)
            ax1_rhs.fill_between(
                cwave, total_noise / dwave, lw=0, color=color, alpha=0.2)
            ax1_rhs.fill_between(
                cwave, dark_noise / dwave, lw=0, color=color, alpha=0.2)
            ax1_rhs.plot(cwave, total_noise / dwave, ls='-.', color=color)

            ax2.plot(
                cwave, camera.rms_resolution.to(wave_unit).value,
                ls='-', color=color)
            ax2.plot(
                cwave, camera.row_size.to(wave_unit / u.pixel).value,
                ls='--', color=color)

            ax2_rhs.plot(
                cwave, camera.neff_spatial.to(u.pixel), ls=':', color=color)

        ax1.plot([], [], 'k--', label='Fiber Acceptance')
        ax1.plot([], [], 'k-', label='Camera Throughput')
        ax1.plot([], [], 'k:', label='{0}'.format(flux))
        ax1.plot([], [], 'k-.', label='Dark + Readout Noise')
        ax1.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3,
                   ncol=2, mode="expand", borderaxespad=0.)

        ax2.plot([], [], 'k-', label='RMS Resolution')
        ax2.plot([], [], 'k--', label='Row Size')
        ax2.plot([], [], 'k:', label='Column Size')
        ax2.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3,
                   ncol=3, mode="expand", borderaxespad=0.)

        ax1.set_ylim(0., None)
        ax1.set_ylabel('Fiber, Camera Throughput')
        ax1_rhs.set_ylim(0., None)
        ax1_rhs.set_ylabel(
            'Photons, Electrons / Exposure / {0}'.format(wave_unit))
        ax2.set_ylim(0., None)
        ax2.set_ylabel('RMS Resolution, Row Size [{0}]'.format(wave_unit))
        ax2_rhs.set_ylim(0., None)
        ax2_rhs.set_ylabel('Effective Column Size [pixels]')
        ax2.set_xlabel('Wavelength [{0}]'.format(wave_unit))
        ax2.set_xlim(wave[0], wave[-1])

def initialize(config, camera_output=True):
    """Initialize the instrument model from configuration parameters.

    This method is responsible for creating a new :class:`Instrument` as
    well as the :class:`Cameras <specsim.camera.Camera>` it includes.

    Parameters
    ----------
    config : :class:`specsim.config.Configuration`
        The configuration parameters to use.
    camera_output : bool
        Initialize support for resolution convolution and downsampling for
        each camera when True.

    Returns
    -------
    Instrument
        An initialized instrument model including one or more
        :class:`cameras <specsim.camera.Camera>`.
    """
    name = config.instrument.name
    cameras = config.instrument.cameras
    camera_names = cameras.keys()
    initialized_cameras = []
    for camera_name in camera_names:
        camera = getattr(cameras, camera_name)
        ccd = config.load_table(
            camera.ccd, ['row_size', 'fwhm_resolution', 'neff_spatial'])
        throughput = config.load_table(camera.throughput, 'throughput')
        constants = config.get_constants(camera,
            ['read_noise', 'dark_current', 'gain', 'num_sigmas_clip',
             'output_pixel_size'])
        initialized_cameras.append(specsim.camera.Camera(
            camera_name, config.wavelength, throughput,
            ccd['row_size'], ccd['fwhm_resolution'],
            ccd['neff_spatial'], constants['read_noise'],
            constants['dark_current'], constants['gain'],
            constants['num_sigmas_clip'], constants['output_pixel_size'],
            allow_convolution=camera_output))

    constants = config.get_constants(
        config.instrument,
        ['primary_mirror_diameter', 'obscuration_diameter',
         'support_width', 'fiber_diameter', 'field_radius'])

    try:
        # Try to read a tabulated plate scale first.
        plate_scale = config.load_table(
            config.instrument.plate_scale,
            ['radius', 'radial_scale', 'azimuthal_scale'], interpolate=False)
        r_vec = plate_scale['radius']
        sr_vec = plate_scale['radial_scale']
        sa_vec = plate_scale['azimuthal_scale']
        # Build dimensionless linear interpolators for the radial and azimuthal
        # scales using the native units from the tabulated data.
        sr_interpolate = scipy.interpolate.interp1d(
            r_vec.value, sr_vec.value, kind='linear', copy=True)
        sa_interpolate = scipy.interpolate.interp1d(
            r_vec.value, sa_vec.value, kind='linear', copy=True)
        # Wrap interpolators in lambdas that take care of units.
        radial_scale = lambda r: (
            sr_interpolate(r.to(r_vec.unit).value) * sr_vec.unit)
        azimuthal_scale = lambda r: (
            sa_interpolate(r.to(r_vec.unit).value) * sa_vec.unit)
    except AttributeError:
        # Fall back to a constant value.
        plate_scale_constant = config.get_constants(
            config.instrument.plate_scale, ['value'])
        value = plate_scale_constant['value']
        # Create lambdas that return the constant plate scale with units.
        # Use np.ones_like to ensure correct broadcasting.
        radial_scale = lambda r: value * np.ones_like(r.value)
        azimuthal_scale = lambda r: value * np.ones_like(r.value)

    # Initialize for both fiberloss methods so that method can be changed
    # at run time.
    fiberloss_method = config.instrument.fiberloss.method
    fiberloss_num_wlen = config.instrument.fiberloss.num_wlen
    fiberloss_num_pixels = config.instrument.fiberloss.num_pixels
    if hasattr(config.instrument.fiberloss, 'table'):
        fiber_acceptance_dict = config.load_table(
            config.instrument.fiberloss, 'fiber_acceptance', as_dict=True)
    else:
        fiber_acceptance_dict = None
    if hasattr(config.instrument.fiberloss, 'fast_fiber_acceptance_path'):
        filename = os.path.join(
            config.abs_base_path,
            config.instrument.fiberloss.fast_fiber_acceptance_path)
        if not os.path.isfile(filename) :
            raise RuntimeError(
            'Cannot find file {}. May need to update desimodel svn ?'
            .format(filename))
        fast_fiber_acceptance = specsim.fastfiberacceptance.FastFiberAcceptance(
            filename)
    else:
        fast_fiber_acceptance = None

    blur_value = getattr(config.instrument.blur, 'value', None)
    if blur_value:
        blur_value = specsim.config.parse_quantity(blur_value, u.micron)
        blur_function = lambda angle, wlen: blur_value
    else:
        blur_function = config.load_table2d(
            config.instrument.blur, 'wavelength', 'r=')

    offset_value = getattr(config.instrument.offset, 'value', None)
    if offset_value:
        offset_value = specsim.config.parse_quantity(offset_value, u.micron)
        offset_function = (
            lambda angle_x, angle_y, wlen: (offset_value, 0 * u.um))
    else:
        # Build an interpolator in (r, wlen) of radial chromatic offsets.
        radial_offset_function = config.load_table2d(
            config.instrument.offset, 'wavelength', 'r=')
        # Look for an optional file of achromatic offsets.
        # Static achromatic term, correlated in focal plane
        if hasattr(config.instrument.offset, 'static'):
            # Build an interpolator in (x, y).
            interpolators = config.load_fits2d(
                config.instrument.offset.static, xy_unit=u.deg,
                dx='XOFFSET', dy='YOFFSET')
            static_fx = interpolators['dx']
            static_fy = interpolators['dy']
        else:
            static_fx  = None
            static_fy  = None
        # Random uncorrelated achromatic term
        if hasattr(config.instrument.offset, 'sigma1d'):
            if hasattr(config.instrument.offset, 'seed'):
                offset_gen = np.random.RandomState(config.instrument.offset.seed)
            else:
                offset_gen = np.random.RandomState()
            sigma1d=specsim.config.parse_quantity(config.instrument.offset.sigma1d)
            random_fx = lambda angle_x, angle_y: offset_gen.normal(size=angle_x.shape) * sigma1d
            random_fy = lambda angle_x, angle_y: offset_gen.normal(size=angle_x.shape) * sigma1d
        else :
            random_fx = None
            random_fy = None
        # Combine the interpolators into a function of (x, y, wlen) that
        # returns (dx, dy).  Use default parameter values to capture the
        # necessary state in the inner function's closure.
        def offset_function(angle_x, angle_y, wlen,
                            fr=radial_offset_function,
                            static_fx=static_fx,
                            static_fy=static_fy,
                            random_fx=random_fx,
                            random_fy=random_fy) :
            angle_r = np.sqrt(angle_x ** 2 + angle_y ** 2)
            dr = fr(angle_r, wlen)
            # Special handling of the origin.
            not_at_origin = (angle_r > 0.)
            ux = np.ones(shape=dr.shape, dtype=float)
            uy = np.ones(shape=dr.shape, dtype=float)
            ux[not_at_origin] = angle_x / angle_r
            uy[not_at_origin] = angle_y / angle_r
            dx = dr * ux
            dy = dr * uy
            # Add interpolated offsets if any.
            if static_fx is not None : dx += static_fx(angle_x, angle_y)
            if static_fy is not None : dy += static_fy(angle_x, angle_y)
            if random_fx is not None : dx += random_fx(angle_x, angle_y)
            if random_fy is not None : dy += random_fy(angle_x, angle_y)
            return dx , dy

    instrument = Instrument(
        name, config.wavelength, fiberloss_method, fiber_acceptance_dict,
        fast_fiber_acceptance, fiberloss_num_wlen, fiberloss_num_pixels,
        blur_function, offset_function, initialized_cameras,
        constants['primary_mirror_diameter'], constants['obscuration_diameter'],
        constants['support_width'], constants['fiber_diameter'],
        constants['field_radius'], radial_scale, azimuthal_scale)

    if config.verbose:
        # Print some derived quantities.
        print('Telescope effective area: {0:.3f}'
              .format(instrument.effective_area))
        print('Field of view diameter: {0:.1f} = {1:.2f}.'
              .format(2 * instrument.field_radius.to(u.mm),
                      2 * instrument.field_angle.to(u.deg)))
        if fiber_acceptance_dict is not None:
            print('Fiberloss source types: {0}.'
                  .format(instrument.fiber_acceptance_dict.keys()))

    return instrument

import numpy,import scipy,import astropy


# -*- mode: python; coding: utf-8 -*-
# Copyright 2018 Peter Williams and collaborators.
# Licensed under the MIT License.

"""This module provides helpers for doing radiative transfer integrations with
`grtrans <https://github.com/jadexter/grtrans>`_, including a simple
framework for running end-to-end tests.

In order to use this functionality, the Python module ``radtrans_integrate``
must be importable. Sadly `grtrans <https://github.com/jadexter/grtrans>`_
doesn't install itself like a regular Python package, so getting this working
can be a pain. Documenting the installation procedure is beyond the scope of
this project.

"""
from __future__ import absolute_import, division, print_function

import numpy as np
import pandas as pd

# grtrans' integration methods:
METHOD_LSODA_YES_LINEAR_STOKES = 0 # LSODA with IS_LINEAR_STOKES=1
METHOD_DELO = 1 # DELO method from Rees+ (1989ApJ...339.1093R)
METHOD_FORMAL = 2 # "formal" method = "matricant (O-matrix) method from Landi Degl'Innocenti"
METHOD_LSODA_NO_LINEAR_STOKES = 3 # LSODA with IS_LINEAR_STOKES=0 -- this is "under development" spherical stokes

def integrate_ray_formal(x, j, K):
    """Use `grtrans <https://github.com/jadexter/grtrans>`_ to integrate one ray
    using its "formal" (matricant / O-matrix) method.

    **Call signature**

    *x*
      1D array, shape (n,). Path length along ray, starting from zero, in cm.
    *j*
      Array, shape (n, 4). Emission coefficients: ``j_{IQUV}``, in that order.
    *K*
      Array, shape (n, 7). Absorption coefficients and Faraday mixing coefficients:
      ``alpha_{IQUV}, rho_{QUV}``.
    Return value
      Array of shape (4, m): Stokes intensities ``IQUV`` along parts of the
      ray with non-zero total emissivities; m <= n.

    """
    # A correct version of the fully-specified absorption matrix is in Leung+
    # 2011 (10.1088/0004-637X/737/1/21). From looking at the function
    # `radtrans_jac_form` of grtrans' `radtrans_integrate.f90` file, one can
    # see that the K vector is indeed packed in the way described above.
    from radtrans_integrate import radtrans_integrate

    n = x.size

    # The formal method doesn't do the same kind of clipping as LSODA *inside*
    # grtrans, but here we do the same clipping for consistency, and since it
    # is genuinely true that the clipped samples are superfluous.

    if np.all(j[:,0] == 0.):
        return np.zeros((4, n))

    i0 = 0
    i1 = n - 1

    while j[i0,0] == 0.:
        i0 += 1
    while j[i1,0] == 0.:
        i1 -= 1

    n = i1 + 1 - i0
    x = x[i0:i1+1]
    j = j[i0:i1+1]
    K = K[i0:i1+1]

    # OK we can go.

    radtrans_integrate.init_radtrans_integrate_data(
        METHOD_FORMAL, # method selector
        4, # number of equations
        n, # number of input data points
        n, # number of output data points
        10., # maximum optical depth; not used by "formal"
        1., # maximum absolute step size; not used by "formal"
        0.1, # absolute tolerance; not used by "formal"
        0.1, # relative tolerance; not used by "formal"
        1e-2, # "thin" parameter for DELO method; not used by "formal"
        1, # maximum number of steps; not used by "formal"
    )

    try:
        tau = x # not used by "formal"
        radtrans_integrate.integrate(x[::-1], j[::-1], K[::-1], tau[::-1], 4)
        i = radtrans_integrate.intensity.copy()
    finally:
        # If we exit without calling this, the next init call causes an abort
        radtrans_integrate.del_radtrans_integrate_data()
    return i

def integrate_ray_lsoda(x, j, K, atol=1e-8, rtol=1e-6, max_step_size=None,
                        frac_max_step_size=1e-3, max_steps=100000):
    """Use `grtrans <https://github.com/jadexter/grtrans>`_ to integrate one ray
    using its LSODA method.

    **Call signature**

    *x*
      1D array, shape (n,). Path length along ray, starting from zero, in cm.
    *j*
      Array, shape (n, 4). Emission coefficients: ``j_{IQUV}``, in that order.
    *K*
      Array, shape (n, 7). Absorption coefficients and Faraday mixing coefficients:
      ``alpha_{IQUV}, rho_{QUV}``.
    *atol*
      Some kind of tolerance parameter.
    *rtol*
      Some kind of tolerance parameter.
    *max_step_size*
      The maximum absolute step size. Overrides *frac_max_step_size*.
    *frac_max_step_size*
      If *max_step_size* is not specified, the maximum step size passed to the
      integrator is ``x.max()`` multiplied by this parameter. Experience shows
      that (for LSODA at least) this parameter must be pretty small to get
      good convergence!
    *max_steps*
      The maximum number of steps to take.
    Return value
      Array of shape (4, m): Stokes intensities IQUV along parts of the ray with
      non-zero total emissivities; m <= n.

    """
    n = x.size

    if max_step_size is None:
        max_step_size = frac_max_step_size * x.max()

    # the LSODA method clips its input arrays based on "tau" and zero emission
    # coefficients. It's hard for us to find out how it clipped, though, so we
    # reproduce its logic. LSODA doesn't use "tau" for anything besides this
    # clipping, so we pass it all zeros.

    if np.all(j[:,0] == 0.):
        return np.zeros((4, n))

    i0 = 0
    i1 = n - 1

    while j[i0,0] == 0.:
        i0 += 1
    while j[i1,0] == 0.:
        i1 -= 1

    n = i1 + 1 - i0
    x = x[i0:i1+1]
    j = j[i0:i1+1]
    K = K[i0:i1+1]

    # OK we can go.

    radtrans_integrate.init_radtrans_integrate_data(
        METHOD_LSODA_YES_LINEAR_STOKES, # method selector
        4, # number of equations
        n, # number of input data points
        n, # number of output data points
        10., # maximum optical depth; defused here (see comment above)
        max_step_size, # maximum absolute step size
        atol, # absolute tolerance
        rtol, # relative tolerance
        1e-2, # "thin" parameter for DELO method ... to be researched
        max_steps, # maximum number of steps
    )

    try:
        tau = np.zeros(n)
        radtrans_integrate.integrate(x[::-1], j, K, tau, 4)
        i = radtrans_integrate.intensity.copy()
    finally:
        # If we exit without calling this, the next init call causes an abort
        radtrans_integrate.del_radtrans_integrate_data()
    return i

def integrate(d, coeffs, psi):
    """Integrate a ray with `grtrans <https://github.com/jadexter/grtrans>`_,
    using reasonable defaults.

    **Call signature**

    *d*
      An array giving the location of each sample along the ray, starting from zero, in cm.
    *coeffs*
      An array of shape (N, 8) of RT coefficients in the basis where the
      Stokes U coefficients are always zero. Such arrays are returned by
      :meth:`neurosynchro.impl.PhysicalApproximator.compute_all_nontrivial`.
    *psi*
      An array of angles between the local magnetic field and the observer’s Stokes U
      axis, in radians.
    Return value
      An array of shape (4,), giving the Stokes IQUV at the end of the ray.

    This function is mainly intended to test what happens if the passed-in
    coefficients are slightly different due to the neural network
    approximation. So we don't provide many knobs or diagnostics here.

    """
    from . import detrivialize_stokes_basis

    xformed = detrivialize_stokes_basis(coeffs, psi)
    j = np.empty(coeffs.shape[:-1] + (4,))
    j[...,0] = xformed[...,0]
    j[...,1] = xformed[...,2]
    j[...,2] = xformed[...,4]
    j[...,3] = xformed[...,6]
    K = np.empty(coeffs.shape[:-1] + (7,))
    K[...,0] = xformed[...,1]
    K[...,1] = xformed[...,3]
    K[...,2] = xformed[...,5]
    K[...,3:] = xformed[...,7:]

    iquv = integrate_ray_formal(d, j, K)
    return iquv[:,-1]

def make_parser(ap=None):
    if ap is None:
        ap = argparse.ArgumentParser()

    ap.add_argument('--frequency', '-f', type=float, metavar='<ghz>',
                    default=10.0, help='The observing frequency to simulate')
    ap.add_argument('nndir', type=str, metavar='<nndir>',
                    help='The path to the neural-net directory.')
    ap.add_argument('testdata', type=str, metavar='<testdata>',
                    help='A file containing test data')
    return ap

def grtrans_cli(settings):
    from pwkit import cgs
    from pwkit.cli import die
    from time import time
    from .impl import PhysicalApproximator, hardcoded_nu_ref, hardcoded_ne_ref

    # Read and validate the test dataset.

    testdata = pd.read_table(settings.testdata)

    psi = testdata.get('psi(meta)')
    if psi is None:
        die('the test dataset must contain a column of field-to-Stokes-U angles \"psi(meta)\"')

    d = testdata.get('d(meta)')
    if d is None:
        die('the test dataset must contain a column of integration path lengths \"d(meta)\"')

    n_e = testdata.get('n_e(meta)')
    if n_e is None:
        die('the test dataset must contain a column of particle densities \"n_e(meta)\"')

    time_ms = testdata.get('time_ms(meta)')
    if time_ms is None:
        die('the test dataset must contain a column of computation times \"time_ms(meta)\"')

    s = None
    theta = None
    others = {}

    for col in testdata.columns:
        if col.startswith('s('):
            s = testdata[col]
        elif col.startswith('theta('):
            theta = testdata[col]
        elif col.endswith('(lin)') or col.endswith('(log)'):
            others[col.split('(')[0]] = testdata[col]

    if s is None:
        die('the test dataset must have an input parameter of the harmonic number \"s\"')

    if theta is None:
        die('the test dataset must have an input parameter of the field-to-LOS angle \"theta\"')

    # Get the coefficients into physical units, packed in our standard format.

    nu_hz = settings.frequency * 1e9
    freq_scale = nu_hz / hardcoded_nu_ref
    n_e_scale = n_e / hardcoded_ne_ref

    coeffs = np.empty((psi.size, 8))
    coeffs[...,0] = testdata['j_I(res)'] * freq_scale
    coeffs[...,1] = testdata['alpha_I(res)'] / freq_scale
    coeffs[...,2] = testdata['j_Q(res)'] * freq_scale
    coeffs[...,3] = testdata['alpha_Q(res)'] / freq_scale
    coeffs[...,4] = testdata['j_V(res)'] * freq_scale
    coeffs[...,5] = testdata['alpha_V(res)'] / freq_scale
    coeffs[...,6] = testdata['rho_Q(res)'] / freq_scale
    coeffs[...,7] = testdata['rho_V(res)'] / freq_scale
    coeffs *= n_e_scale.values.reshape((-1, 1))

    # Ground truth:

    iquv_precise = integrate(d, coeffs, psi)
    ctime_precise = time_ms.sum()
    print('Precise computation: I={:.4e}  Q={:.4e}  U={:.4e}  V={:.4e}  calc_time={:.0f} ms'.format(
          iquv_precise[0], iquv_precise[1], iquv_precise[2], iquv_precise[3], ctime_precise
    ))

    # Now set up the approximator and do the same thing. (Note that often the
    # timing seems backwards, because the time spent doing the precise
    # calculation has already been spent, whereas we have a lot of overhead to
    # set up the neural networks.)

    B = 2 * np.pi * cgs.me * cgs.c * nu_hz / (s * cgs.e)
    approx = PhysicalApproximator(settings.nndir)
    t0 = time()
    coeffs, oos = approx.compute_all_nontrivial(nu_hz, B, n_e, theta, **others)
    ctime_approx = 1000 * (time() - t0)

    if np.any(oos != 0):
        print('WARNING: some of the approximations were out-of-sample')

    iquv_approx = integrate(d, coeffs, psi)
    print('Approx. computation: I={:.4e}  Q={:.4e}  U={:.4e}  V={:.4e}  calc_time={:.0f} ms'.format(
          iquv_approx[0], iquv_approx[1], iquv_approx[2], iquv_approx[3], ctime_approx
    ))

    acc = np.abs((iquv_approx - iquv_precise) / iquv_precise)
    print('Accuracy: I={:.3f}  Q={:.3f}  U={:.3f}  V={:.3f}'.format(acc[0], acc[1], acc[2], acc[3]))

    print('Speedup: {:.1f}'.format(ctime_precise / ctime_approx))