from multiprocessing import Pool<EOL>fittype = '<STR_LIT>'<EOL>if not magsarefluxes:<EOL><INDENT>raise NotImplementedError('<STR_LIT>')<EOL><DEDENT>if not samplesavpath:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>'<EOL>)<EOL><DEDENT>if not mandel_agol_dependencies:<EOL><INDENT>raise ImportError(<EOL>'<STR_LIT>'<EOL>)<EOL><DEDENT>stimes, smags, serrs = sigclip_magseries(times, mags, errs,<EOL>sigclip=sigclip,<EOL>magsarefluxes=magsarefluxes)<EOL>nzind = np.nonzero(serrs)<EOL>stimes, smags, serrs = stimes[nzind], smags[nzind], serrs[nzind]<EOL>init_period = _get_value('<STR_LIT>', fitparams, fixedparams)<EOL>init_epoch = _get_value('<STR_LIT>', fitparams, fixedparams)<EOL>init_rp = _get_value('<STR_LIT>', fitparams, fixedparams)<EOL>init_sma = _get_value('<STR_LIT>', fitparams, fixedparams)<EOL>init_incl = _get_value('<STR_LIT>', fitparams, fixedparams)<EOL>init_ecc = _get_value('<STR_LIT>', fitparams, fixedparams)<EOL>init_omega = _get_value('<STR_LIT>', fitparams, fixedparams)<EOL>limb_dark = _get_value('<STR_LIT>', fitparams, fixedparams)<EOL>init_u = _get_value('<STR_LIT:u>', fitparams, fixedparams)<EOL>init_poly_order0 = _get_value('<STR_LIT>', fitparams, fixedparams)<EOL>init_poly_order1 = _get_value('<STR_LIT>', fitparams, fixedparams)<EOL>if not limb_dark == '<STR_LIT>':<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>'<EOL>)<EOL><DEDENT>init_params, init_m = _transit_model(<EOL>stimes, init_epoch, init_period, init_rp, init_sma, init_incl,<EOL>init_ecc, init_omega, init_u, limb_dark)<EOL>init_flux = (<EOL>init_m.light_curve(init_params) +<EOL>init_poly_order0 + init_poly_order1*stimes<EOL>)<EOL>theta, fitparamnames = [], []<EOL>for k in np.sort(list(fitparams.keys())):<EOL><INDENT>if isinstance(fitparams[k], float) or isinstance(fitparams[k], int):<EOL><INDENT>theta.append(fitparams[k])<EOL>fitparamnames.append(fitparams[k])<EOL><DEDENT>elif isinstance(fitparams[k], list):<EOL><INDENT>if not len(fitparams[k]) == <NUM_LIT:2>:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>theta.append(fitparams[k][<NUM_LIT:0>])<EOL>theta.append(fitparams[k][<NUM_LIT:1>])<EOL>fitparamnames.append(fitparams[k][<NUM_LIT:0>])<EOL>fitparamnames.append(fitparams[k][<NUM_LIT:1>])<EOL><DEDENT><DEDENT>n_dim = len(theta)<EOL>if not skipsampling:<EOL><INDENT>backend = emcee.backends.HDFBackend(samplesavpath)<EOL>if overwriteexistingsamples:<EOL><INDENT>LOGWARNING(<EOL>'<STR_LIT>'.format(samplesavpath)<EOL>)<EOL>backend.reset(n_walkers, n_dim)<EOL><DEDENT>def nll(*args):<EOL><INDENT>return -_log_likelihood_transit_plus_line(*args)<EOL><DEDENT>soln = spminimize(<EOL>nll, theta, method='<STR_LIT>',<EOL>args=(init_params, init_m, stimes, smags, serrs, priorbounds)<EOL>)<EOL>theta_ml = soln.x<EOL>ml_poly_order0 = theta_ml[<NUM_LIT:0>]<EOL>ml_poly_order1 = theta_ml[<NUM_LIT:1>]<EOL>ml_rp = theta_ml[<NUM_LIT:2>]<EOL>ml_t0 = theta_ml[<NUM_LIT:3>]<EOL>ml_params, ml_m = _transit_model(stimes, ml_t0, init_period,<EOL>ml_rp, init_sma, init_incl,<EOL>init_ecc, init_omega, init_u,<EOL>limb_dark)<EOL>ml_mags = (<EOL>ml_m.light_curve(ml_params) +<EOL>ml_poly_order0 + ml_poly_order1*stimes<EOL>)<EOL>initial_position_vec = [theta_ml + eps*np.random.randn(n_dim)<EOL>for i in range(n_walkers)]<EOL>starting_positions = initial_position_vec<EOL>isfirstrun = True<EOL>if os.path.exists(backend.filename):<EOL><INDENT>if backend.iteration > <NUM_LIT:1>:<EOL><INDENT>starting_positions = None<EOL>isfirstrun = False<EOL><DEDENT><DEDENT>if verbose and isfirstrun:<EOL><INDENT>LOGINFO(<EOL>'<STR_LIT>'.<EOL>format(fittype, n_dim, n_mcmc_steps, n_walkers) +<EOL>'<STR_LIT>'.format(nworkers)<EOL>)<EOL><DEDENT>elif verbose and not isfirstrun:<EOL><INDENT>LOGINFO(<EOL>'<STR_LIT>'.<EOL>format(fittype, n_dim, n_mcmc_steps, n_walkers) +<EOL>'<STR_LIT>'.format(nworkers)<EOL>)<EOL><DEDENT>with Pool(nworkers) as pool:<EOL><INDENT>sampler = emcee.EnsembleSampler(<EOL>n_walkers, n_dim, log_posterior_transit_plus_line,<EOL>args=(init_params, init_m, stimes, smags, serrs, priorbounds),<EOL>pool=pool,<EOL>backend=backend<EOL>)<EOL>sampler.run_mcmc(starting_positions, n_mcmc_steps,<EOL>progress=mcmcprogressbar)<EOL><DEDENT>if verbose:<EOL><INDENT>LOGINFO(<EOL>'<STR_LIT>'.format(<EOL>fittype, n_mcmc_steps, n_walkers<EOL>) + '<STR_LIT>'.format(nworkers)<EOL>)<EOL><DEDENT><DEDENT>reader = emcee.backends.HDFBackend(samplesavpath)<EOL>n_to_discard = int(burninpercent*n_mcmc_steps)<EOL>samples = reader.get_chain(discard=n_to_discard, flat=True)<EOL>log_prob_samples = reader.get_log_prob(discard=n_to_discard, flat=True)<EOL>log_prior_samples = reader.get_blobs(discard=n_to_discard, flat=True)<EOL>fit_statistics = list(<EOL>map(lambda v: (v[<NUM_LIT:1>], v[<NUM_LIT:2>]-v[<NUM_LIT:1>], v[<NUM_LIT:1>]-v[<NUM_LIT:0>]),<EOL>list(zip( *np.percentile(samples, [<NUM_LIT>, <NUM_LIT:50>, <NUM_LIT>], axis=<NUM_LIT:0>))))<EOL>)<EOL>medianparams, std_perrs, std_merrs = {}, {}, {}<EOL>for ix, k in enumerate(np.sort(list(priorbounds.keys()))):<EOL><INDENT>medianparams[k] = fit_statistics[ix][<NUM_LIT:0>]<EOL>std_perrs[k] = fit_statistics[ix][<NUM_LIT:1>]<EOL>std_merrs[k] = fit_statistics[ix][<NUM_LIT:2>]<EOL><DEDENT>stderrs = {'<STR_LIT>':std_perrs, '<STR_LIT>':std_merrs}<EOL>per = _get_value('<STR_LIT>', medianparams, fixedparams)<EOL>t0 = _get_value('<STR_LIT>', medianparams, fixedparams)<EOL>rp = _get_value('<STR_LIT>', medianparams, fixedparams)<EOL>sma = _get_value('<STR_LIT>', medianparams, fixedparams)<EOL>incl = _get_value('<STR_LIT>', medianparams, fixedparams)<EOL>ecc = _get_value('<STR_LIT>', medianparams, fixedparams)<EOL>omega = _get_value('<STR_LIT>', medianparams, fixedparams)<EOL>limb_dark = _get_value('<STR_LIT>', medianparams, fixedparams)<EOL>try:<EOL><INDENT>u = fixedparams['<STR_LIT:u>']<EOL><DEDENT>except Exception as e:<EOL><INDENT>u = [medianparams['<STR_LIT>'], medianparams['<STR_LIT>']]<EOL><DEDENT>poly_order0 = _get_value('<STR_LIT>', medianparams, fixedparams)<EOL>poly_order1 = _get_value('<STR_LIT>', medianparams, fixedparams)<EOL>fit_params, fit_m = _transit_model(stimes, t0, per, rp, sma, incl, ecc,<EOL>omega, u, limb_dark)<EOL>fitmags = (<EOL>fit_m.light_curve(fit_params) +<EOL>poly_order0 + poly_order1*stimes<EOL>)<EOL>fepoch = t0<EOL>medianparams['<STR_LIT>'] += timeoffset<EOL>returndict = {<EOL>'<STR_LIT>':fittype,<EOL>'<STR_LIT>':{<EOL>'<STR_LIT>':fitparams,<EOL>'<STR_LIT>':init_flux,<EOL>'<STR_LIT>':fixedparams,<EOL>'<STR_LIT>':medianparams,<EOL>'<STR_LIT>':stderrs,<EOL>'<STR_LIT>':fitmags,<EOL>'<STR_LIT>':fepoch+timeoffset,<EOL>},<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':{<EOL>'<STR_LIT>':stimes+timeoffset,<EOL>'<STR_LIT>':smags,<EOL>'<STR_LIT>':serrs,<EOL>'<STR_LIT>':magsarefluxes,<EOL>},<EOL>}<EOL>if plotcorner:<EOL><INDENT>fig = corner.corner(<EOL>samples,<EOL>labels=['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>'.format(timeoffset)],<EOL>truths=[ml_poly_order0, ml_poly_order1, ml_rp, ml_t0],<EOL>quantiles=[<NUM_LIT>, <NUM_LIT:0.5>, <NUM_LIT>], show_titles=True<EOL>)<EOL>plt.savefig(plotcorner, dpi=<NUM_LIT>)<EOL>if verbose:<EOL><INDENT>LOGINFO('<STR_LIT>'.format(plotcorner))<EOL><DEDENT><DEDENT>if plotfit and isinstance(plotfit, str):<EOL><INDENT>plt.close('<STR_LIT:all>')<EOL>f, (a0, a1) = plt.subplots(nrows=<NUM_LIT:2>, ncols=<NUM_LIT:1>, sharex=True,<EOL>figsize=(<NUM_LIT:8>,<NUM_LIT:5>),<EOL>gridspec_kw={'<STR_LIT>':[<NUM_LIT:3>, <NUM_LIT:1>]})<EOL>a0.scatter(stimes, smags, c='<STR_LIT:k>', alpha=<NUM_LIT>, label='<STR_LIT:data>', zorder=<NUM_LIT:1>,<EOL>s=<NUM_LIT:10>, rasterized=True, linewidths=<NUM_LIT:0>)<EOL>DEBUGGING = False<EOL>if DEBUGGING:<EOL><INDENT>a0.scatter(stimes, init_flux, c='<STR_LIT:r>', alpha=<NUM_LIT:1>, s=<NUM_LIT>, zorder=<NUM_LIT:2>,<EOL>rasterized=True, linewidths=<NUM_LIT:0>,<EOL>label='<STR_LIT>')<EOL>a0.scatter(stimes, ml_mags, c='<STR_LIT:g>', alpha=<NUM_LIT:1>, s=<NUM_LIT>, zorder=<NUM_LIT:2>,<EOL>rasterized=True, linewidths=<NUM_LIT:0>, label='<STR_LIT>')<EOL><DEDENT>a0.plot(<EOL>stimes, fitmags, c='<STR_LIT:b>',<EOL>zorder=<NUM_LIT:0>, rasterized=True, lw=<NUM_LIT:2>, alpha=<NUM_LIT>,<EOL>label='<STR_LIT>'.format(fittype, len(fitparamnames))<EOL>)<EOL>a1.scatter(<EOL>stimes, smags-fitmags, c='<STR_LIT:k>', alpha=<NUM_LIT>,<EOL>rasterized=True, s=<NUM_LIT:10>, linewidths=<NUM_LIT:0><EOL>)<EOL>if scatterxdata and scatteryaxes:<EOL><INDENT>import matplotlib.transforms as transforms<EOL>for a in [a0, a1]:<EOL><INDENT>transform = transforms.blended_transform_factory(<EOL>a.transData, a.transAxes<EOL>)<EOL>a.scatter(scatterxdata, scatteryaxes, c='<STR_LIT:r>', alpha=<NUM_LIT>,<EOL>zorder=<NUM_LIT:2>, s=<NUM_LIT:10>, rasterized=True, linewidths=<NUM_LIT:0>,<EOL>marker="<STR_LIT>", transform=transform)<EOL><DEDENT><DEDENT>a1.set_xlabel('<STR_LIT>')<EOL>a0.set_ylabel('<STR_LIT>')<EOL>a1.set_ylabel('<STR_LIT>')<EOL>a0.legend(loc='<STR_LIT>', fontsize='<STR_LIT>')<EOL>for a in [a0, a1]:<EOL><INDENT>a.get_yaxis().set_tick_params(which='<STR_LIT>', direction='<STR_LIT>')<EOL>a.get_xaxis().set_tick_params(which='<STR_LIT>', direction='<STR_LIT>')<EOL><DEDENT>f.tight_layout(h_pad=<NUM_LIT:0>, w_pad=<NUM_LIT:0>)<EOL>f.savefig(plotfit, dpi=<NUM_LIT>, bbox_inches='<STR_LIT>')<EOL>if verbose:<EOL><INDENT>LOGINFO('<STR_LIT>'.format(plotfit))<EOL><DEDENT>returndict['<STR_LIT>'] = plotfit<EOL><DEDENT>return returndict<EOL>

The model fit by this function is: a Mandel & Agol (2002) transit, PLUS a
    line. You can fit and fix whatever parameters you want.

    A typical use case: you want to measure transit times of individual SNR >~
    50 transits. You fix all the transit parameters except for the mid-time,
    and also fit for a line locally.

    NOTE: this only works for flux time-series at the moment.

    NOTE: Between the `fitparams`, `priorbounds`, and `fixedparams` dicts, you
    must specify all of the planetary transit parameters required by BATMAN and
    the parameters for the line fit: `['t0', 'rp', 'sma', 'incl', 'u', 'rp',
    'ecc', 'omega', 'period', 'poly_order0', poly_order1']`, or the BATMAN model
    will fail to initialize.

    Parameters
    ----------

    times,mags,errs : np.array
        The input flux time-series to fit a Fourier cosine series to.

    fitparams : dict
        This is the initial parameter guesses for MCMC, found e.g., by
        BLS. The key string format must not be changed, but any parameter can be
        either "fit" or "fixed". If it is "fit", it must have a corresponding
        prior. For example::

            fitparams = {'t0':1325.9,
                         'poly_order0':1,
                         'poly_order1':0.}

        where `t0` is the time of transit-center for a reference transit.
        `poly_order0` corresponds to the intercept of the line, `poly_order1` is
        the slope.

    priorbounds : dict
        This sets the lower & upper bounds on uniform prior, e.g.::

            priorbounds = {'t0':(np.min(time), np.max(time)),
                            'poly_order0':(0.5,1.5),
                            'poly_order1':(-0.5,0.5) }

    fixedparams : dict
        This sets which parameters are fixed, and their values. For example::

            fixedparams = {'ecc':0.,
                           'omega':90.,
                           'limb_dark':'quadratic',
                           'period':fitd['period'],
                           'rp':np.sqrt(fitd['transitdepth']),
                           'sma':6.17, 'incl':85, 'u':[0.3, 0.2]}

        `limb_dark` must be "quadratic".  It's "fixed", because once you
        choose your limb-darkening model, it's fixed.

    trueparams : list of floats
        The true parameter values you're fitting for, if they're known (e.g., a
        known planet, or fake data). Only for plotting purposes.

    burninpercent : float
        The percent of MCMC samples to discard as burn-in.

    plotcorner : str or False
        If this is a str, points to the path of output corner plot that will be
        generated for this MCMC run.

    timeoffset : float
        If input times are offset by some constant, and you want saved pickles
        to fix that.

    samplesavpath : str
        This must be provided so `emcee` can save its MCMC samples to disk as
        HDF5 files. This will set the path of the output HDF5file written.

    n_walkers : int
        The number of MCMC walkers to use.

    n_mcmc_steps : int
        The number of MCMC steps to take.

    eps : float
        The radius of the `n_walkers-dimensional` Gaussian ball used to
        initialize the MCMC.

    skipsampling : bool
        If you've already collected MCMC samples, and you do not want any more
        sampling (e.g., just make the plots), set this to be True.

    overwriteexistingsamples : bool
        If you've collected samples, but you want to overwrite them, set this to
        True. Usually, it should be False, which appends samples to
        `samplesavpath` HDF5 file.

    mcmcprogressbar : bool
        If True, will show a progress bar for the MCMC process.

    plotfit: str or bool
        If a str, indicates the path of the output fit plot file. If False, no
        fit plot will be made.

    scatterxdata : np.array or None
        Use this to overplot x,y scatter points on the output model/data
        lightcurve (e.g., to highlight bad data, or to indicate an ephemeris),
        this can take a `np.ndarray` with the same units as `times`.

    scatteryaxes : np.array or None
        Use this to provide the y-values for scatterxdata, in units of fraction
        of an axis.

    magsarefluxes : bool
        This indicates if the input measurements in `mags` are actually fluxes.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    verbose : bool
        If True, will indicate MCMC progress.

    nworkers : int
        The number of parallel workers to launch for MCMC.

    Returns
    -------

    dict
        This function returns a dict containing the model fit parameters and
        other fit information. The form of this dict is mostly standardized
        across all functions in this module::

            {
                'fittype':'mandelagol_and_line',
                'fitinfo':{
                    'initialparams':the initial transit params provided,
                    'fixedparams':the fixed transit params provided,
                    'finalparams':the final model fit transit params,
                    'finalparamerrs':formal errors in the params,
                    'fitmags': the model fit mags,
                    'fitepoch': the epoch of minimum light for the fit,
                },
                'fitplotfile': the output fit plot if fitplot is not None,
                'magseries':{
                    'times':input times in phase order of the model,
                    'phase':the phases of the model mags,
                    'mags':input mags/fluxes in the phase order of the model,
                    'errs':errs in the phase order of the model,
                    'magsarefluxes':input value of magsarefluxes kwarg
                }
            }


<EOL>if errs is None:<EOL><INDENT>errs = npfull_like(mags, <NUM_LIT>)<EOL><DEDENT>stimes, smags, serrs = sigclip_magseries(times, mags, errs,<EOL>sigclip=sigclip,<EOL>magsarefluxes=magsarefluxes)<EOL>nzind = npnonzero(serrs)<EOL>stimes, smags, serrs = stimes[nzind], smags[nzind], serrs[nzind]<EOL>phase, pmags, perrs, ptimes, mintime = (<EOL>get_phased_quantities(stimes, smags, serrs, period)<EOL>)<EOL>nobs = len(phase)<EOL>nknots = int(npfloor(knotfraction*nobs))<EOL>nknots = maxknots if nknots > maxknots else nknots<EOL>splineknots = nplinspace(phase[<NUM_LIT:0>] + <NUM_LIT>,<EOL>phase[-<NUM_LIT:1>] - <NUM_LIT>,<EOL>num=nknots)<EOL>phase_diffs_ind = npdiff(phase) > <NUM_LIT:0.0><EOL>incphase_ind = npconcatenate((nparray([True]), phase_diffs_ind))<EOL>phase, pmags, perrs = (phase[incphase_ind],<EOL>pmags[incphase_ind],<EOL>perrs[incphase_ind])<EOL>spl = LSQUnivariateSpline(phase, pmags, t=splineknots, w=<NUM_LIT:1.0>/perrs)<EOL>fitmags = spl(phase)<EOL>fitchisq = npsum(<EOL>((fitmags - pmags)*(fitmags - pmags)) / (perrs*perrs)<EOL>)<EOL>fitredchisq = fitchisq/(len(pmags) - nknots - <NUM_LIT:1>)<EOL>if verbose:<EOL><INDENT>LOGINFO(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(nknots, fitchisq, fitredchisq)<EOL>)<EOL><DEDENT>if not magsarefluxes:<EOL><INDENT>fitmagminind = npwhere(fitmags == npmax(fitmags))<EOL><DEDENT>else:<EOL><INDENT>fitmagminind = npwhere(fitmags == npmin(fitmags))<EOL><DEDENT>if len(fitmagminind[<NUM_LIT:0>]) > <NUM_LIT:1>:<EOL><INDENT>fitmagminind = (fitmagminind[<NUM_LIT:0>][<NUM_LIT:0>],)<EOL><DEDENT>magseriesepoch = ptimes[fitmagminind]<EOL>returndict = {<EOL>'<STR_LIT>':'<STR_LIT>',<EOL>'<STR_LIT>':{<EOL>'<STR_LIT>':nknots,<EOL>'<STR_LIT>':fitmags,<EOL>'<STR_LIT>':magseriesepoch<EOL>},<EOL>'<STR_LIT>':fitchisq,<EOL>'<STR_LIT>':fitredchisq,<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':{<EOL>'<STR_LIT>':ptimes,<EOL>'<STR_LIT>':phase,<EOL>'<STR_LIT>':pmags,<EOL>'<STR_LIT>':perrs,<EOL>'<STR_LIT>':magsarefluxes<EOL>},<EOL>}<EOL>if plotfit and isinstance(plotfit, str):<EOL><INDENT>make_fit_plot(phase, pmags, perrs, fitmags,<EOL>period, mintime, magseriesepoch,<EOL>plotfit,<EOL>magsarefluxes=magsarefluxes)<EOL>returndict['<STR_LIT>'] = plotfit<EOL><DEDENT>return returndict<EOL>

This fits a univariate cubic spline to the phased light curve.

    This fit may be better than the Fourier fit for sharply variable objects,
    like EBs, so can be used to distinguish them from other types of variables.

    Parameters
    ----------

    times,mags,errs : np.array
        The input mag/flux time-series to fit a spline to.

    period : float
        The period to use for the spline fit.

    knotfraction : float
        The knot fraction is the number of internal knots to use for the
        spline. A value of 0.01 (or 1%) of the total number of non-nan
        observations appears to work quite well, without over-fitting. maxknots
        controls the maximum number of knots that will be allowed.

    maxknots : int
        The maximum number of knots that will be used even if `knotfraction`
        gives a value to use larger than `maxknots`. This helps dealing with
        over-fitting to short time-scale variations.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    magsarefluxes : bool
        If True, will treat the input values of `mags` as fluxes for purposes of
        plotting the fit and sig-clipping.

    plotfit : str or False
        If this is a string, this function will make a plot for the fit to the
        mag/flux time-series and writes the plot to the path specified here.

    ignoreinitfail : bool
        If this is True, ignores the initial failure to find a set of optimized
        Fourier parameters using the global optimization function and proceeds
        to do a least-squares fit anyway.

    verbose : bool
        If True, will indicate progress and warn of any problems.

    Returns
    -------

    dict
        This function returns a dict containing the model fit parameters, the
        minimized chi-sq value and the reduced chi-sq value. The form of this
        dict is mostly standardized across all functions in this module::

            {
                'fittype':'spline',
                'fitinfo':{
                    'nknots': the number of knots used for the fit
                    'fitmags': the model fit mags,
                    'fitepoch': the epoch of minimum light for the fit,
                },
                'fitchisq': the minimized value of the fit's chi-sq,
                'fitredchisq':the reduced chi-sq value,
                'fitplotfile': the output fit plot if fitplot is not None,
                'magseries':{
                    'times':input times in phase order of the model,
                    'phase':the phases of the model mags,
                    'mags':input mags/fluxes in the phase order of the model,
                    'errs':errs in the phase order of the model,
                    'magsarefluxes':input value of magsarefluxes kwarg
                }
            }


stimes, smags, serrs = sigclip_magseries(times, mags, errs,<EOL>sigclip=sigclip,<EOL>magsarefluxes=magsarefluxes)<EOL>nzind = npnonzero(serrs)<EOL>stimes, smags, serrs = stimes[nzind], smags[nzind], serrs[nzind]<EOL>phase, pmags, perrs, ptimes, mintime = (<EOL>get_phased_quantities(stimes, smags, serrs, period)<EOL>)<EOL>if fourierorder and fourierorder > <NUM_LIT:0> and not fourierparams:<EOL><INDENT>fourieramps = [<NUM_LIT>] + [<NUM_LIT>]*(fourierorder - <NUM_LIT:1>)<EOL>fourierphas = [<NUM_LIT:0.1>] + [<NUM_LIT:0.1>]*(fourierorder - <NUM_LIT:1>)<EOL>fourierparams = fourieramps + fourierphas<EOL><DEDENT>elif not fourierorder and fourierparams:<EOL><INDENT>fourierorder = int(len(fourierparams)/<NUM_LIT:2>)<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>')<EOL>fourierorder = <NUM_LIT:3><EOL>fourieramps = [<NUM_LIT>] + [<NUM_LIT>]*(fourierorder - <NUM_LIT:1>)<EOL>fourierphas = [<NUM_LIT:0.1>] + [<NUM_LIT:0.1>]*(fourierorder - <NUM_LIT:1>)<EOL>fourierparams = fourieramps + fourierphas<EOL><DEDENT>if verbose:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (fourierorder,<EOL>len(phase),<EOL>period,<EOL>mintime))<EOL><DEDENT>initialfit = spminimize(_fourier_chisq,<EOL>fourierparams,<EOL>method='<STR_LIT>',<EOL>args=(phase, pmags, perrs))<EOL>if initialfit.success or ignoreinitfail:<EOL><INDENT>if verbose:<EOL><INDENT>LOGINFO('<STR_LIT>')<EOL><DEDENT>leastsqparams = initialfit.x<EOL>try:<EOL><INDENT>leastsqfit = spleastsq(_fourier_residual,<EOL>leastsqparams,<EOL>args=(phase, pmags))<EOL><DEDENT>except Exception as e:<EOL><INDENT>leastsqfit = None<EOL><DEDENT>if leastsqfit and leastsqfit[-<NUM_LIT:1>] in (<NUM_LIT:1>,<NUM_LIT:2>,<NUM_LIT:3>,<NUM_LIT:4>):<EOL><INDENT>finalparams = leastsqfit[<NUM_LIT:0>]<EOL>fitmags = _fourier_func(finalparams, phase, pmags)<EOL>fitchisq = npsum(<EOL>((fitmags - pmags)*(fitmags - pmags)) / (perrs*perrs)<EOL>)<EOL>fitredchisq = fitchisq/(len(pmags) - len(finalparams) - <NUM_LIT:1>)<EOL>if verbose:<EOL><INDENT>LOGINFO(<EOL>'<STR_LIT>' %<EOL>(fitchisq,fitredchisq)<EOL>)<EOL><DEDENT>if not magsarefluxes:<EOL><INDENT>fitmagminind = npwhere(fitmags == npmax(fitmags))<EOL><DEDENT>else:<EOL><INDENT>fitmagminind = npwhere(fitmags == npmin(fitmags))<EOL><DEDENT>if len(fitmagminind[<NUM_LIT:0>]) > <NUM_LIT:1>:<EOL><INDENT>fitmagminind = (fitmagminind[<NUM_LIT:0>][<NUM_LIT:0>],)<EOL><DEDENT>returndict = {<EOL>'<STR_LIT>':'<STR_LIT>',<EOL>'<STR_LIT>':{<EOL>'<STR_LIT>':fourierorder,<EOL>'<STR_LIT>':finalparams,<EOL>'<STR_LIT>':initialfit,<EOL>'<STR_LIT>':leastsqfit,<EOL>'<STR_LIT>':fitmags,<EOL>'<STR_LIT>':mintime,<EOL>'<STR_LIT>':ptimes[fitmagminind]<EOL>},<EOL>'<STR_LIT>':fitchisq,<EOL>'<STR_LIT>':fitredchisq,<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':{<EOL>'<STR_LIT>':ptimes,<EOL>'<STR_LIT>':phase,<EOL>'<STR_LIT>':pmags,<EOL>'<STR_LIT>':perrs,<EOL>'<STR_LIT>':magsarefluxes<EOL>},<EOL>}<EOL>if plotfit and isinstance(plotfit, str):<EOL><INDENT>make_fit_plot(phase, pmags, perrs, fitmags,<EOL>period, mintime, mintime,<EOL>plotfit,<EOL>magsarefluxes=magsarefluxes)<EOL>returndict['<STR_LIT>'] = plotfit<EOL><DEDENT>return returndict<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>')<EOL>return {<EOL>'<STR_LIT>':'<STR_LIT>',<EOL>'<STR_LIT>':{<EOL>'<STR_LIT>':fourierorder,<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':initialfit,<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':None<EOL>},<EOL>'<STR_LIT>':npnan,<EOL>'<STR_LIT>':npnan,<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':{<EOL>'<STR_LIT>':ptimes,<EOL>'<STR_LIT>':phase,<EOL>'<STR_LIT>':pmags,<EOL>'<STR_LIT>':perrs,<EOL>'<STR_LIT>':magsarefluxes<EOL>}<EOL>}<EOL><DEDENT><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% initialfit.message)<EOL>return {<EOL>'<STR_LIT>':'<STR_LIT>',<EOL>'<STR_LIT>':{<EOL>'<STR_LIT>':fourierorder,<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':initialfit,<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':None<EOL>},<EOL>'<STR_LIT>':npnan,<EOL>'<STR_LIT>':npnan,<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':{<EOL>'<STR_LIT>':ptimes,<EOL>'<STR_LIT>':phase,<EOL>'<STR_LIT>':pmags,<EOL>'<STR_LIT>':perrs,<EOL>'<STR_LIT>':magsarefluxes<EOL>}<EOL>}<EOL><DEDENT>

This fits a Fourier series to a mag/flux time series.

    Parameters
    ----------

    times,mags,errs : np.array
        The input mag/flux time-series to fit a Fourier cosine series to.

    period : float
        The period to use for the Fourier fit.

    fourierorder : None or int
        If this is an int, will be interpreted as the Fourier order of the
        series to fit to the input mag/flux times-series. If this is None and
        `fourierparams` is specified, `fourierparams` will be used directly to
        generate the fit Fourier series. If `fourierparams` is also None, this
        function will try to fit a Fourier cosine series of order 3 to the
        mag/flux time-series.

    fourierparams : list of floats or None
        If this is specified as a list of floats, it must be of the form below::

            [fourier_amp1, fourier_amp2, fourier_amp3,...,fourier_ampN,
             fourier_phase1, fourier_phase2, fourier_phase3,...,fourier_phaseN]

        to specify a Fourier cosine series of order N. If this is None and
        `fourierorder` is specified, the Fourier order specified there will be
        used to construct the Fourier cosine series used to fit the input
        mag/flux time-series. If both are None, this function will try to fit a
        Fourier cosine series of order 3 to the input mag/flux time-series.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    magsarefluxes : bool
        If True, will treat the input values of `mags` as fluxes for purposes of
        plotting the fit and sig-clipping.

    plotfit : str or False
        If this is a string, this function will make a plot for the fit to the
        mag/flux time-series and writes the plot to the path specified here.

    ignoreinitfail : bool
        If this is True, ignores the initial failure to find a set of optimized
        Fourier parameters using the global optimization function and proceeds
        to do a least-squares fit anyway.

    verbose : bool
        If True, will indicate progress and warn of any problems.

    Returns
    -------

    dict
        This function returns a dict containing the model fit parameters, the
        minimized chi-sq value and the reduced chi-sq value. The form of this
        dict is mostly standardized across all functions in this module::

            {
                'fittype':'fourier',
                'fitinfo':{
                    'finalparams': the list of final model fit params,
                    'leastsqfit':the full tuple returned by scipy.leastsq,
                    'fitmags': the model fit mags,
                    'fitepoch': the epoch of minimum light for the fit,
                    ... other fit function specific keys ...
                },
                'fitchisq': the minimized value of the fit's chi-sq,
                'fitredchisq':the reduced chi-sq value,
                'fitplotfile': the output fit plot if fitplot is not None,
                'magseries':{
                    'times':input times in phase order of the model,
                    'phase':the phases of the model mags,
                    'mags':input mags/fluxes in the phase order of the model,
                    'errs':errs in the phase order of the model,
                    'magsarefluxes':input value of magsarefluxes kwarg
                }
            }

        NOTE: the returned value of 'fitepoch' in the 'fitinfo' dict returned by
        this function is the time value of the first observation since this is
        where the LC is folded for the fit procedure. To get the actual time of
        minimum epoch as calculated by a spline fit to the phased LC, use the
        key 'actual_fitepoch' in the 'fitinfo' dict.


def _fourier_func(fourierparams, phase, mags):

<EOL>order = int(len(fourierparams)/<NUM_LIT:2>)<EOL>f_amp = fourierparams[:order]<EOL>f_pha = fourierparams[order:]<EOL>f_orders = [f_amp[x]*npcos(<NUM_LIT>*pi_value*x*phase + f_pha[x])<EOL>for x in range(order)]<EOL>total_f = npmedian(mags)<EOL>for fo in f_orders:<EOL><INDENT>total_f += fo<EOL><DEDENT>return total_f<EOL>

This returns a summed Fourier cosine series.

    Parameters
    ----------

    fourierparams : list
        This MUST be a list of the following form like so::

            [period,
             epoch,
             [amplitude_1, amplitude_2, amplitude_3, ..., amplitude_X],
             [phase_1, phase_2, phase_3, ..., phase_X]]

        where X is the Fourier order.

    phase,mags : np.array
        The input phase and magnitude areas to use as the basis for the cosine
        series. The phases are used directly to generate the values of the
        function, while the mags array is used to generate the zeroth order
        amplitude coefficient.

    Returns
    -------

    np.array
        The Fourier cosine series function evaluated over `phase`.


def get_phased_quantities(stimes, smags, serrs, period):

<EOL>mintime = np.min(stimes)<EOL>iphase = (stimes - mintime)/period - np.floor((stimes - mintime)/period)<EOL>phasesortind = np.argsort(iphase)<EOL>phase = iphase[phasesortind]<EOL>pmags = smags[phasesortind]<EOL>perrs = serrs[phasesortind]<EOL>ptimes = stimes[phasesortind]<EOL>return phase, pmags, perrs, ptimes, mintime<EOL>

Does phase-folding for the mag/flux time-series given a period.

    Given finite and sigma-clipped times, magnitudes, and errors, along with the
    period at which to phase-fold the data, perform the phase-folding and
    return the phase-folded values.

    Parameters
    ----------

    stimes,smags,serrs : np.array
        The sigma-clipped and finite input mag/flux time-series arrays to
        operate on.

    period : float
        The period to phase the mag/flux time-series at. stimes.min() is used as
        the epoch value to fold the times-series around.

    Returns
    -------

    (phase, pmags, perrs, ptimes, mintime) : tuple
        The tuple returned contains the following items:

        - `phase`: phase-sorted values of phase at each of stimes
        - `pmags`: phase-sorted magnitudes at each phase
        - `perrs`: phase-sorted errors
        - `ptimes`: phase-sorted times
        - `mintime`: earliest time in stimes.


binnedpkls = sorted(glob.glob(os.path.join(binnedpkldir, binnedpklglob)))<EOL>textlcs = []<EOL>for bpkl in binnedpkls:<EOL><INDENT>objectid = HATIDREGEX.findall(bpkl)<EOL>if objectid is not None:<EOL><INDENT>objectid = objectid[<NUM_LIT:0>]<EOL><DEDENT>searchpath = os.path.join(textlcdir, '<STR_LIT>' % (objectid, textlcglob))<EOL>textlcf = glob.glob(searchpath)<EOL>if textlcf:<EOL><INDENT>textlcs.append(textlcf)<EOL><DEDENT>else:<EOL><INDENT>textlcs.append(None)<EOL><DEDENT><DEDENT>

This generates the binnedlc pkls for a directory of such files.

FIXME: finish this


concatlcd = concatenate_textlcs_for_objectid(lcbasedir,<EOL>objectid,<EOL>aperture=aperture,<EOL>sortby=sortby,<EOL>normalize=normalize,<EOL>recursive=recursive)<EOL>if not outdir:<EOL><INDENT>outdir = '<STR_LIT>'<EOL><DEDENT>if not os.path.exists(outdir):<EOL><INDENT>os.mkdir(outdir)<EOL><DEDENT>outfpath = os.path.join(outdir, '<STR_LIT>' % (concatlcd['<STR_LIT>'],<EOL>aperture))<EOL>pklc = lcdict_to_pickle(concatlcd, outfile=outfpath)<EOL>return pklc<EOL>

This concatenates all text LCs for the given object and writes to a pklc.

    Basically a rollup for the concatenate_textlcs_for_objectid and
    lcdict_to_pickle functions.


if not outdir:<EOL><INDENT>outdir = '<STR_LIT>'<EOL><DEDENT>if not os.path.exists(outdir):<EOL><INDENT>os.mkdir(outdir)<EOL><DEDENT>tasks = [(lcbasedir, x, {'<STR_LIT>':aperture,<EOL>'<STR_LIT>':postfix,<EOL>'<STR_LIT>':sortby,<EOL>'<STR_LIT>':normalize,<EOL>'<STR_LIT>':outdir,<EOL>'<STR_LIT>':recursive}) for x in objectidlist]<EOL>pool = mp.Pool(nworkers, maxtasksperchild=maxworkertasks)<EOL>results = pool.map(parallel_concat_worker, tasks)<EOL>pool.close()<EOL>pool.join()<EOL>return {x:y for (x,y) in zip(objectidlist, results)}<EOL>

This concatenates all text LCs for the given objectidlist.


if '<STR_LIT>' in lcfile:<EOL><INDENT>thiscoldefs = COLDEFS + [('<STR_LIT>',float)]<EOL><DEDENT>elif '<STR_LIT>' in lcfile:<EOL><INDENT>thiscoldefs = COLDEFS + [('<STR_LIT>',float)]<EOL><DEDENT>elif '<STR_LIT>' in lcfile:<EOL><INDENT>thiscoldefs = COLDEFS + [('<STR_LIT>',float)]<EOL><DEDENT>LOGINFO('<STR_LIT>' % lcfile)<EOL>if lcfile.endswith('<STR_LIT>'):<EOL><INDENT>infd = gzip.open(lcfile,'<STR_LIT:r>')<EOL><DEDENT>else:<EOL><INDENT>infd = open(lcfile,'<STR_LIT:r>')<EOL><DEDENT>with infd:<EOL><INDENT>lclines = infd.read().decode().split('<STR_LIT:\n>')<EOL>lclines = [x.split() for x in lclines if ('<STR_LIT:#>' not in x and len(x) > <NUM_LIT:0>)]<EOL>ndet = len(lclines)<EOL>if ndet > <NUM_LIT:0>:<EOL><INDENT>lccols = list(zip(*lclines))<EOL>lcdict = {x[<NUM_LIT:0>]:y for (x,y) in zip(thiscoldefs, lccols)}<EOL>for col in thiscoldefs:<EOL><INDENT>lcdict[col[<NUM_LIT:0>]] = np.array([col[<NUM_LIT:1>](x) for x in lcdict[col[<NUM_LIT:0>]]])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>lcdict = {}<EOL>LOGWARNING('<STR_LIT>' % lcfile)<EOL>for col in thiscoldefs:<EOL><INDENT>lcdict[col[<NUM_LIT:0>]] = np.array([])<EOL><DEDENT><DEDENT>hatid = HATIDREGEX.findall(lcfile)<EOL>lcdict['<STR_LIT>'] = hatid[<NUM_LIT:0>] if hatid else '<STR_LIT>'<EOL>lcdict['<STR_LIT>'] = [x[<NUM_LIT:0>] for x in thiscoldefs]<EOL>lcdict['<STR_LIT>'] = {<EOL>'<STR_LIT>':ndet,<EOL>'<STR_LIT>':hatid[<NUM_LIT:0>] if hatid else '<STR_LIT>',<EOL>'<STR_LIT>':'<STR_LIT>',<EOL>}<EOL>framekeyelems = FRAMEREGEX.findall('<STR_LIT:\n>'.join(lcdict['<STR_LIT>']))<EOL>lcdict['<STR_LIT>'] = np.array([(int(x[<NUM_LIT:0>]) if x[<NUM_LIT:0>].isdigit() else np.nan)<EOL>for x in framekeyelems])<EOL>lcdict['<STR_LIT>'] = np.array([(int(x[<NUM_LIT:1>]) if x[<NUM_LIT:0>].isdigit() else np.nan)<EOL>for x in framekeyelems])<EOL>lcdict['<STR_LIT>'] = np.array([x[<NUM_LIT:2>] for x in framekeyelems])<EOL>lcdict['<STR_LIT>'] = np.array([(int(x[<NUM_LIT:3>]) if x[<NUM_LIT:0>].isdigit() else np.nan)<EOL>for x in framekeyelems])<EOL>lcdict['<STR_LIT>'].extend(['<STR_LIT>','<STR_LIT>','<STR_LIT>','<STR_LIT>'])<EOL>lcdict['<STR_LIT>']['<STR_LIT>'] = '<STR_LIT>'<EOL>lcdict['<STR_LIT>']['<STR_LIT>'] = [<EOL>'<STR_LIT>' % x for x in np.unique(lcdict['<STR_LIT>']).tolist()<EOL>]<EOL><DEDENT>return lcdict<EOL>

This reads in a textlc that is complete up to the TFA stage.


try:<EOL><INDENT>if lcfile.endswith('<STR_LIT>'):<EOL><INDENT>infd = gzip.open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>infd = open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>lcdict = pickle.load(infd)<EOL>infd.close()<EOL>return lcdict<EOL><DEDENT>except UnicodeDecodeError:<EOL><INDENT>if lcfile.endswith('<STR_LIT>'):<EOL><INDENT>infd = gzip.open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>infd = open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % lcfile)<EOL>lcdict = pickle.load(infd, encoding='<STR_LIT>')<EOL>infd.close()<EOL>return lcdict<EOL><DEDENT>

This just reads a pickle LC. Returns an lcdict.


lcbasedir, objectid, kwargs = task<EOL>try:<EOL><INDENT>return concat_write_pklc(lcbasedir, objectid, **kwargs)<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>'<EOL>% (objectid, lcbasedir))<EOL>return None<EOL><DEDENT>

This is a worker for the function below.

task[0] = lcbasedir
task[1] = objectid
task[2] = {'aperture','postfix','sortby','normalize','outdir','recursive'}


LOGINFO('<STR_LIT>'.format(lcpath))<EOL>N_lines_to_parse_comments = <NUM_LIT:50><EOL>with open(lcpath, '<STR_LIT:rb>') as file:<EOL><INDENT>head = [next(file) for ind in range(N_lines_to_parse_comments)]<EOL><DEDENT>N_comment_lines = len([l for l in head if l.decode('<STR_LIT>')[<NUM_LIT:0>] == '<STR_LIT:#>'])<EOL>if N_comment_lines < N_lines_to_parse_comments:<EOL><INDENT>LOGERROR(<EOL>'<STR_LIT>'.format(fpath=lcpath)<EOL>)<EOL>return None<EOL><DEDENT>first_data_line = list(<EOL>filter(None, head[N_comment_lines].decode('<STR_LIT>').split())<EOL>)<EOL>N_cols = len(first_data_line)<EOL>if N_cols == <NUM_LIT>:<EOL><INDENT>colformat = '<STR_LIT>'<EOL><DEDENT>elif N_cols == <NUM_LIT:20>:<EOL><INDENT>colformat = '<STR_LIT>'<EOL><DEDENT>elif N_cols == <NUM_LIT:32>:<EOL><INDENT>colformat = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>"<EOL>"<STR_LIT>".format(fpath=lcpath,<EOL>ncols=N_cols))<EOL>return None<EOL><DEDENT>if colformat == '<STR_LIT>':<EOL><INDENT>col_names = ['<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>']<EOL>col_dtypes = ['<STR_LIT>',float,<EOL>float,float,'<STR_LIT>',<EOL>float,float,'<STR_LIT>',<EOL>float,float,'<STR_LIT>',<EOL>float,float,float,<EOL>float,float,float]<EOL>dtype_pairs = [el for el in zip(col_names, col_dtypes)]<EOL>data = np.genfromtxt(lcpath, names=col_names, dtype=col_dtypes,<EOL>skip_header=N_comment_lines, delimiter=None)<EOL>out = {}<EOL>for ix in range(len(data.dtype.names)):<EOL><INDENT>out[data.dtype.names[ix]] = data[data.dtype.names[ix]]<EOL><DEDENT><DEDENT>elif colformat == '<STR_LIT>':<EOL><INDENT>col_names = ['<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>']<EOL>col_dtypes = ['<STR_LIT>',float,<EOL>float,float,'<STR_LIT>',<EOL>float,float,'<STR_LIT>',<EOL>float,float,'<STR_LIT>',<EOL>float,float,float,<EOL>float,float,float,<EOL>float,float,float]<EOL>dtype_pairs = [el for el in zip(col_names, col_dtypes)]<EOL>data = np.genfromtxt(lcpath, names=col_names, dtype=col_dtypes,<EOL>skip_header=N_comment_lines, delimiter=None)<EOL>out = {}<EOL>for ix in range(len(data.dtype.names)):<EOL><INDENT>out[data.dtype.names[ix]] = data[data.dtype.names[ix]]<EOL><DEDENT><DEDENT>elif colformat == '<STR_LIT>':<EOL><INDENT>col_names = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>out = astascii.read(lcpath, names=col_names, comment='<STR_LIT:#>')<EOL><DEDENT>return out<EOL>

Read .epdlc, and .tfalc light curves and return a corresponding labelled
dict (if LC from <2012) or astropy table (if >=2012). Each has different
keys that can be accessed via .keys()

Input:
lcpath: path (string) to light curve data, which is a textfile with HAT
LC data.

Example:
dat = read_original_textlc('HAT-115-0003266.epdlc')


<EOL>headerlines = header.split('<STR_LIT:\n>')<EOL>headerlines = [x.lstrip('<STR_LIT>') for x in headerlines]<EOL>objectstart = headerlines.index('<STR_LIT>')<EOL>metadatastart = headerlines.index('<STR_LIT>')<EOL>camfilterstart = headerlines.index('<STR_LIT>')<EOL>photaperturestart = headerlines.index('<STR_LIT>')<EOL>columnstart = headerlines.index('<STR_LIT>')<EOL>lcstart = headerlines.index('<STR_LIT>')<EOL>objectinfo = headerlines[objectstart+<NUM_LIT:1>:metadatastart-<NUM_LIT:1>]<EOL>metadatainfo = headerlines[metadatastart+<NUM_LIT:1>:camfilterstart-<NUM_LIT:1>]<EOL>camfilterinfo = headerlines[camfilterstart+<NUM_LIT:1>:photaperturestart-<NUM_LIT:1>]<EOL>photapertureinfo = headerlines[photaperturestart+<NUM_LIT:1>:columnstart-<NUM_LIT:1>]<EOL>columninfo = headerlines[columnstart+<NUM_LIT:1>:lcstart-<NUM_LIT:1>]<EOL>metadict = {'<STR_LIT>':{}}<EOL>objectinfo = [x.split('<STR_LIT:;>') for x in objectinfo]<EOL>for elem in objectinfo:<EOL><INDENT>for kvelem in elem:<EOL><INDENT>key, val = kvelem.split('<STR_LIT>',<NUM_LIT:1>)<EOL>metadict['<STR_LIT>'][key.strip()] = (<EOL>_smartcast(val, METAKEYS[key.strip()])<EOL>)<EOL><DEDENT><DEDENT>metadict['<STR_LIT>'] = metadict['<STR_LIT>']['<STR_LIT>'][:]<EOL>del metadict['<STR_LIT>']['<STR_LIT>']<EOL>metadatainfo = [x.split('<STR_LIT:;>') for x in metadatainfo]<EOL>for elem in metadatainfo:<EOL><INDENT>for kvelem in elem:<EOL><INDENT>try:<EOL><INDENT>key, val = kvelem.split('<STR_LIT>',<NUM_LIT:1>)<EOL>if key.strip() == '<STR_LIT>':<EOL><INDENT>val = json.loads(val)<EOL><DEDENT>if key.strip() in ('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>val = int(val)<EOL><DEDENT>if key.strip() == '<STR_LIT>':<EOL><INDENT>val = float(val)<EOL><DEDENT>metadict[key.strip()] = val<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>' % kvelem)<EOL><DEDENT><DEDENT><DEDENT>metadict['<STR_LIT>'] = []<EOL>for row in camfilterinfo:<EOL><INDENT>filterid, filtername, filterdesc = row.split('<STR_LIT>')<EOL>metadict['<STR_LIT>'].append((int(filterid),<EOL>filtername,<EOL>filterdesc))<EOL><DEDENT>metadict['<STR_LIT>'] = {}<EOL>for row in photapertureinfo:<EOL><INDENT>apnum, appix = row.split('<STR_LIT>')<EOL>appix = float(appix.rstrip('<STR_LIT>'))<EOL>metadict['<STR_LIT>'][apnum.strip()] = appix<EOL><DEDENT>metadict['<STR_LIT>'] = []<EOL>for row in columninfo:<EOL><INDENT>colnum, colname, coldesc = row.split('<STR_LIT>')<EOL>metadict['<STR_LIT>'].append(colname)<EOL><DEDENT>return metadict<EOL>

This parses the CSV header from the CSV HAT sqlitecurve.

Returns a dict that can be used to update an existing lcdict with the
relevant metadata info needed to form a full LC.


<EOL>try:<EOL><INDENT>if '<STR_LIT>' in lcfile[-<NUM_LIT:4>:]:<EOL><INDENT>lcf = _uncompress_sqlitecurve(lcfile)<EOL><DEDENT>else:<EOL><INDENT>lcf = lcfile<EOL><DEDENT>db = sql.connect(lcf)<EOL>cur = db.cursor()<EOL>query = ("<STR_LIT>")<EOL>cur.execute(query)<EOL>objectinfo = cur.fetchone()<EOL>query = ("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>cur.execute(query)<EOL>lcinfo = cur.fetchone()<EOL>(lcversion, lcdatarelease, lccols, lcsortcol,<EOL>lcapertures, lcbestaperture,<EOL>objinfocols, objidcol,<EOL>lcunixtime, lcgitrev, lccomment) = lcinfo<EOL>lcapertures = json.loads(lcapertures)<EOL>lcbestaperture = json.loads(lcbestaperture)<EOL>objectinfokeys = objinfocols.split('<STR_LIT:U+002C>')<EOL>objectinfodict = {x:y for (x,y) in zip(objectinfokeys, objectinfo)}<EOL>objectid = objectinfodict[objidcol]<EOL>query = ("<STR_LIT>")<EOL>cur.execute(query)<EOL>filterinfo = cur.fetchall()<EOL>if columns and all([x in lccols.split('<STR_LIT:U+002C>') for x in columns]):<EOL><INDENT>LOGINFO('<STR_LIT>' % columns)<EOL>proceed = True<EOL><DEDENT>elif columns is None:<EOL><INDENT>columns = lccols.split('<STR_LIT:U+002C>')<EOL>proceed = True<EOL><DEDENT>else:<EOL><INDENT>proceed = False<EOL><DEDENT>if not proceed:<EOL><INDENT>if '<STR_LIT>' in lcfile[-<NUM_LIT:4>:] and lcf:<EOL><INDENT>_compress_sqlitecurve(lcf, force=forcerecompress)<EOL><DEDENT>LOGERROR('<STR_LIT>')<EOL>return None, "<STR_LIT>"<EOL><DEDENT>lcdict = {'<STR_LIT>':objectid,<EOL>'<STR_LIT>':objectinfodict,<EOL>'<STR_LIT>':objectinfokeys,<EOL>'<STR_LIT>':lcversion,<EOL>'<STR_LIT>':lcdatarelease,<EOL>'<STR_LIT>':columns,<EOL>'<STR_LIT>':lcsortcol,<EOL>'<STR_LIT>':lcapertures,<EOL>'<STR_LIT>':lcbestaperture,<EOL>'<STR_LIT>':lcunixtime,<EOL>'<STR_LIT>':lcgitrev,<EOL>'<STR_LIT>':lccomment,<EOL>'<STR_LIT>':filterinfo}<EOL>if ((sqlfilters is not None) and<EOL>(isinstance(sqlfilters,str) or<EOL>isinstance(sqlfilters, unicode))):<EOL><INDENT>validatedfilters = _validate_sqlitecurve_filters(sqlfilters,<EOL>lccols.split('<STR_LIT:U+002C>'))<EOL>if validatedfilters is not None:<EOL><INDENT>LOGINFO('<STR_LIT>' % validatedfilters)<EOL>filtersok = True<EOL><DEDENT>else:<EOL><INDENT>filtersok = False<EOL><DEDENT><DEDENT>else:<EOL><INDENT>validatedfilters = None<EOL>filtersok = None<EOL><DEDENT>if validatedfilters is not None:<EOL><INDENT>query = (<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>).format(<EOL>columns='<STR_LIT:U+002C>'.join(columns),  <EOL>sqlfilter=validatedfilters,<EOL>sortcol=lcsortcol<EOL>)<EOL>lcdict['<STR_LIT>'] = validatedfilters<EOL><DEDENT>else:<EOL><INDENT>query = ("<STR_LIT>") % (<EOL>'<STR_LIT:U+002C>'.join(columns),<EOL>lcsortcol<EOL>)<EOL><DEDENT>cur.execute(query)<EOL>lightcurve = cur.fetchall()<EOL>if lightcurve and len(lightcurve) > <NUM_LIT:0>:<EOL><INDENT>lightcurve = list(zip(*lightcurve))<EOL>lcdict.update({x:y for (x,y) in zip(lcdict['<STR_LIT>'],<EOL>lightcurve)})<EOL>lcok = True<EOL>lcdict['<STR_LIT>']['<STR_LIT>'] = len(lightcurve[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>' % lcdict['<STR_LIT>'])<EOL>lcdict.update({x:y for (x,y) in<EOL>zip(lcdict['<STR_LIT>'],<EOL>[[] for x in lcdict['<STR_LIT>']])})<EOL>lcok = False<EOL><DEDENT>if filtersok is True and lcok:<EOL><INDENT>statusmsg = '<STR_LIT>'<EOL><DEDENT>elif filtersok is None and lcok:<EOL><INDENT>statusmsg = '<STR_LIT>'<EOL><DEDENT>elif filtersok is False and lcok:<EOL><INDENT>statusmsg = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>statusmsg = '<STR_LIT>'<EOL><DEDENT>returnval = (lcdict, statusmsg)<EOL>if '<STR_LIT>' in lcfile[-<NUM_LIT:4>:] and lcf:<EOL><INDENT>_compress_sqlitecurve(lcf, force=forcerecompress)<EOL><DEDENT>if returnarrays:<EOL><INDENT>for column in lcdict['<STR_LIT>']:<EOL><INDENT>lcdict[column] = np.array([x if x is not None else np.nan<EOL>for x in lcdict[column]])<EOL><DEDENT><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>if not quiet:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' % lcfile)<EOL><DEDENT>returnval = (None, '<STR_LIT>')<EOL>if '<STR_LIT>' in lcfile[-<NUM_LIT:4>:] and lcf:<EOL><INDENT>_compress_sqlitecurve(lcf, force=forcerecompress)<EOL><DEDENT>if raiseonfail:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>return returnval<EOL>

This reads a HAT sqlitecurve and optionally filters it.

    Parameters
    ----------

    lcfile : str
        The path to the HAT sqlitecurve file.

    columns : list
        A list of columns to extract from the ligh curve file. If None, then
        returns all columns present in the latest `columnlist` in the light
        curve.

    sqlfilters : list of str
        If no None, it must be a list of text SQL filters that apply to the
        columns in the lightcurve.

    raiseonfail : bool
        If this is True, an Exception when reading the LC will crash the
        function instead of failing silently and returning None as the result.

    returnarrays : bool
        If this is True, the output lcdict contains columns as np.arrays instead
        of lists. You generally want this to be True.

    forcerecompress : bool
        If True, the sqlitecurve will be recompressed even if a compressed
        version of it is found. This usually happens when sqlitecurve opening is
        interrupted by the OS for some reason, leaving behind a gzipped and
        un-gzipped copy. By default, this function refuses to overwrite the
        existing gzipped version so if the un-gzipped version is corrupt but
        that one isn't, it can be safely recovered.

    quiet : bool
        If True, will not warn about any problems, even if the light curve
        reading fails (the only clue then will be the return value of
        None). Useful for batch processing of many many light curves.

    Returns
    -------

    tuple : (lcdict, status_message)
        A two-element tuple is returned, with the first element being the
        lcdict.


<EOL>import signal<EOL>signal.signal(signal.SIGPIPE, signal.SIG_DFL)<EOL>import argparse<EOL>aparser = argparse.ArgumentParser(<EOL>description='<STR_LIT>'<EOL>)<EOL>aparser.add_argument(<EOL>'<STR_LIT>',<EOL>action='<STR_LIT:store>',<EOL>type=str,<EOL>help=("<STR_LIT>")<EOL>)<EOL>aparser.add_argument(<EOL>'<STR_LIT>',<EOL>action='<STR_LIT:store_true>',<EOL>default=False,<EOL>help=("<STR_LIT>")<EOL>)<EOL>args = aparser.parse_args()<EOL>filetoread = args.hatlcfile<EOL>if not os.path.exists(filetoread):<EOL><INDENT>LOGERROR("<STR_LIT>" % filetoread)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>filename = os.path.basename(filetoread)<EOL>if filename.endswith('<STR_LIT>') or filename.endswith('<STR_LIT>'):<EOL><INDENT>if args.describe:<EOL><INDENT>describe(read_csvlc(filename))<EOL>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>with gzip.open(filename,'<STR_LIT:rb>') as infd:<EOL><INDENT>for line in infd:<EOL><INDENT>print(line.decode(),end='<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT><DEDENT>elif filename.endswith('<STR_LIT>'):<EOL><INDENT>lcdict, msg = read_and_filter_sqlitecurve(filetoread)<EOL>describe(lcdict, offsetwith='<STR_LIT:#>')<EOL>if args.describe:<EOL><INDENT>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>apertures = sorted(lcdict['<STR_LIT>'].keys())<EOL>for aper in apertures:<EOL><INDENT>COLUMNDEFS.update({'<STR_LIT>' % (x, aper): COLUMNDEFS[x] for x in<EOL>LC_MAG_COLUMNS})<EOL>COLUMNDEFS.update({'<STR_LIT>' % (x, aper): COLUMNDEFS[x] for x in<EOL>LC_ERR_COLUMNS})<EOL>COLUMNDEFS.update({'<STR_LIT>' % (x, aper): COLUMNDEFS[x] for x in<EOL>LC_FLAG_COLUMNS})<EOL><DEDENT>formstr = '<STR_LIT:U+002C>'.join([COLUMNDEFS[x][<NUM_LIT:1>] for x in lcdict['<STR_LIT>']])<EOL>ndet = lcdict['<STR_LIT>']['<STR_LIT>']<EOL>for ind in range(ndet):<EOL><INDENT>line = [lcdict[x][ind] for x in lcdict['<STR_LIT>']]<EOL>formline = formstr % tuple(line)<EOL>print(formline)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>' % filetoread)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>

This is called when we're executed from the commandline.

The current usage from the command-line is described below::

    usage: hatlc [-h] [--describe] hatlcfile

    read a HAT LC of any format and output to stdout

    positional arguments:
      hatlcfile   path to the light curve you want to read and pipe to stdout

    optional arguments:
      -h, --help  show this help message and exit
      --describe  don't dump the columns, show only object info and LC metadata


<EOL>if '<STR_LIT>' in os.path.basename(lcfile):<EOL><INDENT>infd = gzip.open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>infd = open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>lctext = infd.read().decode()<EOL>infd.close()<EOL>lctextlines = lctext.split('<STR_LIT:\n>')<EOL>lcformat = lctextlines[<NUM_LIT:0>]<EOL>commentchar = lctextlines[<NUM_LIT:1>]<EOL>lcstart = lctextlines.index('<STR_LIT>' % commentchar)<EOL>headerlines = lctextlines[:lcstart+<NUM_LIT:1>]<EOL>lclines = lctextlines[lcstart+<NUM_LIT:1>:]<EOL>metadata, columns, separator = _parse_csv_header_lcc_csv_v1(headerlines)<EOL>objectid = metadata['<STR_LIT>']['<STR_LIT>']<EOL>objectinfo = {key:metadata[key]['<STR_LIT>'] for key in metadata}<EOL>colnames = []<EOL>colnum = []<EOL>coldtypes = []<EOL>for k in columns:<EOL><INDENT>coldef = columns[k]<EOL>colnames.append(k)<EOL>colnum.append(coldef['<STR_LIT>'])<EOL>coldtypes.append(coldef['<STR_LIT>'])<EOL><DEDENT>coldtypes = '<STR_LIT:U+002C>'.join(coldtypes)<EOL>recarr = np.genfromtxt(<EOL>lclines,<EOL>comments=commentchar,<EOL>delimiter=separator,<EOL>usecols=colnum,<EOL>autostrip=True,<EOL>names=colnames,<EOL>dtype=coldtypes<EOL>)<EOL>lcdict = {x:recarr[x] for x in colnames}<EOL>lcdict['<STR_LIT>'] = lcformat<EOL>lcdict['<STR_LIT>'] = objectid<EOL>lcdict['<STR_LIT>'] = objectinfo<EOL>lcdict['<STR_LIT>'] = colnames<EOL>lcdict['<STR_LIT>'] = columns<EOL>lcdict['<STR_LIT>'] = metadata<EOL>return lcdict<EOL>

This reads a CSV LC produced by an `LCC-Server
    <https://github.com/waqasbhatti/lcc-server>`_ instance.

    Parameters
    ----------

    lcfile : str
        The LC file to read.

    Returns
    -------

    dict
        Returns an lcdict that's readable by most astrobase functions for
        further processing.


<EOL>if '<STR_LIT>' in os.path.basename(lcfile):<EOL><INDENT>LOGINFO('<STR_LIT>' % lcfile)<EOL>infd = gzip.open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>LOGINFO('<STR_LIT>' % lcfile)<EOL>infd = open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>lcformat_check = infd.read(<NUM_LIT:12>).decode()<EOL>if '<STR_LIT>' in lcformat_check:<EOL><INDENT>infd.close()<EOL>return read_lcc_csvlc(lcfile)<EOL><DEDENT>else:<EOL><INDENT>infd.seek(<NUM_LIT:0>)<EOL><DEDENT>lctext = infd.read().decode()  <EOL>infd.close()<EOL>lcstart = lctext.index('<STR_LIT>')<EOL>lcheader = lctext[:lcstart+<NUM_LIT:12>]<EOL>lccolumns = lctext[lcstart+<NUM_LIT>:].split('<STR_LIT:\n>')<EOL>lccolumns = [x for x in lccolumns if len(x) > <NUM_LIT:0>]<EOL>lcdict = _parse_csv_header(lcheader)<EOL>lccolumns = [x.split('<STR_LIT:U+002C>') for x in lccolumns]<EOL>lccolumns = list(zip(*lccolumns))  <EOL>for colind, col in enumerate(lcdict['<STR_LIT>']):<EOL><INDENT>if (col.split('<STR_LIT:_>')[<NUM_LIT:0>] in LC_MAG_COLUMNS or<EOL>col.split('<STR_LIT:_>')[<NUM_LIT:0>] in LC_ERR_COLUMNS or<EOL>col.split('<STR_LIT:_>')[<NUM_LIT:0>] in LC_FLAG_COLUMNS):<EOL><INDENT>lcdict[col] = np.array([_smartcast(x,<EOL>COLUMNDEFS[col.split('<STR_LIT:_>')[<NUM_LIT:0>]][<NUM_LIT:2>])<EOL>for x in lccolumns[colind]])<EOL><DEDENT>elif col in COLUMNDEFS:<EOL><INDENT>lcdict[col] = np.array([_smartcast(x,COLUMNDEFS[col][<NUM_LIT:2>])<EOL>for x in lccolumns[colind]])<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>' % col)<EOL>continue<EOL><DEDENT><DEDENT>return lcdict<EOL>

This reads a HAT data server or LCC-Server produced CSV light curve
    into an lcdict.

    This will automatically figure out the format of the file
    provided. Currently, it can read:

    - legacy HAT data server CSV LCs (e.g. from
      https://hatsouth.org/planets/lightcurves.html) with an extension of the
      form: `.hatlc.csv.gz`.
    - all LCC-Server produced LCC-CSV-V1 LCs (e.g. from
      https://data.hatsurveys.org) with an extension of the form: `-csvlc.gz`.

    Parameters
    ----------

    lcfile : str
        The light curve file to read.

    Returns
    -------

    dict
        Returns an lcdict that can be read and used by many astrobase processing
        functions.


def _smartcast(castee, caster, subval=None):

try:<EOL><INDENT>return caster(castee)<EOL><DEDENT>except Exception as e:<EOL><INDENT>if caster is float or caster is int:<EOL><INDENT>return nan<EOL><DEDENT>elif caster is str:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>return subval<EOL><DEDENT><DEDENT>

This just tries to apply the caster function to castee.

Returns None on failure.


<EOL>if '<STR_LIT>' in lcdict and len(lcdict['<STR_LIT>']) > <NUM_LIT:0>:<EOL><INDENT>if not quiet:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL><DEDENT>return lcdict<EOL><DEDENT>if timecol in lcdict:<EOL><INDENT>times = lcdict[timecol]<EOL><DEDENT>elif '<STR_LIT>' in lcdict:<EOL><INDENT>times = lcdict['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>" %<EOL>lcdict['<STR_LIT>'])<EOL>return lcdict<EOL><DEDENT>ngroups, timegroups = find_lc_timegroups(np.array(times),<EOL>mingap=mingap)<EOL>if '<STR_LIT>' in lcdict:<EOL><INDENT>apertures = sorted(lcdict['<STR_LIT>'].keys())<EOL><DEDENT>elif '<STR_LIT>' in lcdict and '<STR_LIT>' in lcdict['<STR_LIT>']:<EOL><INDENT>apertures = sorted(lcdict['<STR_LIT>']['<STR_LIT>'].keys())<EOL><DEDENT>aimcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>armcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>aepcols = [('<STR_LIT>' % x)for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>atfcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>psimcols = [x for x in ['<STR_LIT>','<STR_LIT>','<STR_LIT>','<STR_LIT>'] if x in lcdict]<EOL>irmcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>iepcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>itfcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>if magcols == '<STR_LIT:all>':<EOL><INDENT>cols_to_normalize = (aimcols + armcols + aepcols + atfcols +<EOL>psimcols + irmcols + iepcols + itfcols)<EOL><DEDENT>elif magcols == '<STR_LIT>':<EOL><INDENT>cols_to_normalize = (irmcols + (['<STR_LIT>'] if '<STR_LIT>' in lcdict else []) +<EOL>irmcols)<EOL><DEDENT>elif magcols == '<STR_LIT>':<EOL><INDENT>cols_to_normalize = (aepcols + (['<STR_LIT>'] if '<STR_LIT>' in lcdict else []) +<EOL>iepcols)<EOL><DEDENT>elif magcols == '<STR_LIT>':<EOL><INDENT>cols_to_normalize = (atfcols + (['<STR_LIT>'] if '<STR_LIT>' in lcdict else []) +<EOL>itfcols)<EOL><DEDENT>elif magcols == '<STR_LIT>':<EOL><INDENT>cols_to_normalize = (aepcols + (['<STR_LIT>'] if '<STR_LIT>' in lcdict else []) +<EOL>iepcols + atfcols +<EOL>(['<STR_LIT>'] if '<STR_LIT>' in lcdict else []) +<EOL>itfcols)<EOL><DEDENT>else:<EOL><INDENT>cols_to_normalize = magcols.split('<STR_LIT:U+002C>')<EOL>cols_to_normalize = [x.strip() for x in cols_to_normalize]<EOL><DEDENT>colsnormalized = []<EOL>for col in cols_to_normalize:<EOL><INDENT>if col in lcdict:<EOL><INDENT>mags = lcdict[col]<EOL>mags = [(nan if x is None else x) for x in mags]<EOL>mags = np.array(mags)<EOL>colsnormalized.append(col)<EOL>finite_ind = np.isfinite(mags)<EOL>if any(finite_ind):<EOL><INDENT>global_mag_median = np.median(mags[finite_ind])<EOL>for tgind, tg in enumerate(timegroups):<EOL><INDENT>finite_ind = np.isfinite(mags[tg])<EOL>group_median = np.median((mags[tg])[finite_ind])<EOL>mags[tg] = mags[tg] - group_median<EOL>if debugmode:<EOL><INDENT>LOGDEBUG('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(col, tgind,<EOL>len(mags[tg]),<EOL>len(finite_ind),<EOL>group_median))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>' % col)<EOL>continue<EOL><DEDENT>if normto == '<STR_LIT>':<EOL><INDENT>mags = mags + global_mag_median<EOL><DEDENT>elif normto in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>if (normto in lcdict['<STR_LIT>'] and<EOL>lcdict['<STR_LIT>'][normto] is not None):<EOL><INDENT>mags = mags + lcdict['<STR_LIT>'][normto]<EOL><DEDENT>else:<EOL><INDENT>if not quiet:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>' % normto)<EOL><DEDENT>normto = '<STR_LIT>'<EOL>mags = mags + global_mag_median<EOL><DEDENT><DEDENT>lcdict[col] = mags<EOL><DEDENT>else:<EOL><INDENT>if not quiet:<EOL><INDENT>LOGWARNING('<STR_LIT>' % col)<EOL><DEDENT>continue<EOL><DEDENT><DEDENT>lcnormcols = ('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>') % (<EOL>repr(colsnormalized),<EOL>mingap,<EOL>normto<EOL>)<EOL>lcdict['<STR_LIT>'] = lcnormcols<EOL>return lcdict<EOL>

This normalizes magcols in `lcdict` using `timecol` to find timegroups.

    Parameters
    ----------

    lcdict : dict
        The input lcdict to process.

    timecol : str
        The key in the lcdict that is to be used to extract the time column.

    magcols : 'all' or list of str
        If this is 'all', all of the columns in the lcdict that are indicated to
        be magnitude measurement columns are normalized. If this is a list of
        str, must contain the keys of the lcdict specifying which magnitude
        columns will be normalized.

    mingap : float
        This defines how much the difference between consecutive measurements is
        allowed to be to consider them as parts of different timegroups. By
        default it is set to 4.0 days.

    normto : {'globalmedian', 'zero', 'jmag', 'hmag', 'kmag', 'bmag', 'vmag', 'sdssg', 'sdssr', 'sdssi'}
        This indicates which column will be the normalization target. If this is
        'globalmedian', the normalization will be to the global median of each
        LC column. If this is 'zero', will normalize to 0.0 for each LC
        column. Otherwise, will normalize to the value of one of the other keys
        in the lcdict['objectinfo'][magkey], meaning the normalization will be
        to some form of catalog magnitude.

    debugmode : bool
        If True, will indicate progress as time-groups are found and processed.

    quiet : bool
        If True, will not emit any messages when processing.

    Returns
    -------

    dict
        Returns the lcdict with the magnitude measurements normalized as
        specified. The normalization happens IN PLACE.


<EOL>if '<STR_LIT>' in os.path.basename(lcfile):<EOL><INDENT>LOGINFO('<STR_LIT>' % lcfile)<EOL>infd = gzip.open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>LOGINFO('<STR_LIT>' % lcfile)<EOL>infd = open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>lctext = infd.read().decode()<EOL>infd.close()<EOL>lcstart = lctext.index('<STR_LIT>')<EOL>lcheader = lctext[:lcstart+<NUM_LIT:12>]<EOL>lccolumns = lctext[lcstart+<NUM_LIT>:].split('<STR_LIT:\n>')<EOL>lccolumns = [x.split('<STR_LIT:U+002C>') for x in lccolumns if len(x) > <NUM_LIT:0>]<EOL>lcdict = _parse_csv_header(lcheader)<EOL>lccolumns = list(zip(*lccolumns))<EOL>for colind, col in enumerate(lcdict['<STR_LIT>']):<EOL><INDENT>lcdict[col.lower()] = np.array([COLUMNDEFS[col][<NUM_LIT:2>](x)<EOL>for x in lccolumns[colind]])<EOL><DEDENT>lcdict['<STR_LIT>'] = [x.lower() for x in lcdict['<STR_LIT>']]<EOL>return lcdict<EOL>

This reads in a K2 lightcurve in CSV format. Transparently reads gzipped
files.

Parameters
----------

lcfile : str
    The light curve file to read.

Returns
-------

dict
    Returns an lcdict.


<EOL>infof = os.path.join(simbasedir,'<STR_LIT>')<EOL>with open(infof, '<STR_LIT:rb>') as infd:<EOL><INDENT>lcinfo = pickle.load(infd)<EOL><DEDENT>lclist = lcinfo['<STR_LIT>']<EOL>varflag = lcinfo['<STR_LIT>']<EOL>vartypes = lcinfo['<STR_LIT>']<EOL>vartind = <NUM_LIT:0><EOL>varinfo = {}<EOL>for lc, varf, _lcind in zip(lclist, varflag, range(len(lclist))):<EOL><INDENT>if varf:<EOL><INDENT>thisvartype = vartypes[vartind]<EOL>if (override_paramdists and<EOL>isinstance(override_paramdists, dict) and<EOL>thisvartype in override_paramdists and<EOL>isinstance(override_paramdists[thisvartype], dict)):<EOL><INDENT>thisoverride_paramdists = override_paramdists[thisvartype]<EOL><DEDENT>else:<EOL><INDENT>thisoverride_paramdists = None<EOL><DEDENT>varlc = add_fakelc_variability(<EOL>lc, thisvartype,<EOL>override_paramdists=thisoverride_paramdists,<EOL>overwrite=overwrite_existingvar<EOL>)<EOL>varinfo[varlc['<STR_LIT>']] = {'<STR_LIT>': varlc['<STR_LIT>'],<EOL>'<STR_LIT>': varlc['<STR_LIT>']}<EOL>vartind = vartind + <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>varlc = add_fakelc_variability(<EOL>lc, None,<EOL>overwrite=overwrite_existingvar<EOL>)<EOL>varinfo[varlc['<STR_LIT>']] = {'<STR_LIT>': varlc['<STR_LIT>'],<EOL>'<STR_LIT>': varlc['<STR_LIT>']}<EOL><DEDENT><DEDENT>lcinfo['<STR_LIT>'] = varinfo<EOL>tempoutf = '<STR_LIT>' % (infof, md5(npr.bytes(<NUM_LIT:4>)).hexdigest()[-<NUM_LIT:8>:])<EOL>with open(tempoutf, '<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(lcinfo, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>if os.path.exists(tempoutf):<EOL><INDENT>shutil.copy(tempoutf, infof)<EOL>os.remove(tempoutf)<EOL><DEDENT>else:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' %<EOL>os.path.dirname(tempoutf))<EOL>raise<EOL><DEDENT>return lcinfo<EOL>

This adds variability and noise to all fake LCs in `simbasedir`.

    If an object is marked as variable in the `fakelcs-info`.pkl file in
    `simbasedir`, a variable signal will be added to its light curve based on
    its selected type, default period and amplitude distribution, the
    appropriate params, etc. the epochs for each variable object will be chosen
    uniformly from its time-range (and may not necessarily fall on a actual
    observed time). Nonvariable objects will only have noise added as determined
    by their params, but no variable signal will be added.

    Parameters
    ----------

    simbasedir : str
        The directory containing the fake LCs to process.

    override_paramdists : dict
        This can be used to override the stored variable parameters in each fake
        LC. It should be a dict of the following form::

            {'<vartype1>': {'<param1>: a scipy.stats distribution function or
                                       the np.random.randint function,
                            .
                            .
                            .
                            '<paramN>: a scipy.stats distribution function
                                       or the np.random.randint function}

        for any vartype in VARTYPE_LCGEN_MAP. These are used to override the
        default parameter distributions for each variable type.

    overwrite_existingvar : bool
        If this is True, then will overwrite any existing variability in the
        input fake LCs in `simbasedir`.

    Returns
    -------

    dict
        This returns a dict containing the fake LC filenames as keys and
        variability info for each as values.


if mags is None:<EOL><INDENT>mags = np.full_like(times, <NUM_LIT:0.0>)<EOL><DEDENT>if errs is None:<EOL><INDENT>errs = np.full_like(times, <NUM_LIT:0.0>)<EOL><DEDENT>epoch = npr.random()*(times.max() - times.min()) + times.min()<EOL>period = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>depth = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>duration = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>ingduration = npr.random()*(<NUM_LIT:0.5>*duration - <NUM_LIT>*duration) + <NUM_LIT>*duration<EOL>if magsarefluxes and depth < <NUM_LIT:0.0>:<EOL><INDENT>depth = -depth<EOL><DEDENT>elif not magsarefluxes and depth > <NUM_LIT:0.0>:<EOL><INDENT>depth = -depth<EOL><DEDENT>modelmags, phase, ptimes, pmags, perrs = (<EOL>transits.trapezoid_transit_func([period, epoch, depth,<EOL>duration, ingduration],<EOL>times,<EOL>mags,<EOL>errs)<EOL>)<EOL>timeind = np.argsort(ptimes)<EOL>mtimes = ptimes[timeind]<EOL>mmags = modelmags[timeind]<EOL>merrs = perrs[timeind]<EOL>modeldict = {<EOL>'<STR_LIT>':'<STR_LIT>',<EOL>'<STR_LIT>':{x:np.asscalar(y) for x,y in zip(['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>'],<EOL>[period,<EOL>epoch,<EOL>depth,<EOL>duration,<EOL>ingduration])},<EOL>'<STR_LIT>':mtimes,<EOL>'<STR_LIT>':mmags,<EOL>'<STR_LIT>':merrs,<EOL>'<STR_LIT>':period,<EOL>'<STR_LIT>':depth<EOL>}<EOL>return modeldict<EOL>

This generates fake planet transit light curves.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'transitperiod', 'transitdepth', 'transitduration'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The variability epoch will be automatically chosen from a uniform
        distribution between `times.min()` and `times.max()`.

        The ingress duration will be automatically chosen from a uniform
        distribution ranging from 0.05 to 0.5 of the transitduration.

        The transitdepth will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'planet',
             'params': {'transitperiod': generated value of period,
                        'transitepoch': generated value of epoch,
                        'transitdepth': generated value of transit depth,
                        'transitduration': generated value of transit duration,
                        'ingressduration': generated value of transit ingress
                                           duration},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varperiod': the generated period of variability == 'transitperiod'
             'varamplitude': the generated amplitude of
                             variability == 'transitdepth'}


lcfile, outdir, kwargs = task<EOL>try:<EOL><INDENT>fakelcresults = make_fakelc(<EOL>lcfile,<EOL>outdir,<EOL>**kwargs<EOL>)<EOL>return fakelcresults<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' % lcfile)<EOL>return None<EOL><DEDENT>

This wraps `process_fakelc` for `make_fakelc_collection` below.

Parameters
----------

task : tuple
    This is of the form::

        task[0] = lcfile
        task[1] = outdir
        task[2] = magrms
        task[3] = dict with keys: {'lcformat', 'timecols', 'magcols',
                                   'errcols', 'randomizeinfo'}

Returns
-------

tuple
    This returns a tuple of the form::

        (fakelc_fpath,
         fakelc_lcdict['columns'],
         fakelc_lcdict['objectinfo'],
         fakelc_lcdict['moments'])


if mags is None:<EOL><INDENT>mags = np.full_like(times, <NUM_LIT:0.0>)<EOL><DEDENT>if errs is None:<EOL><INDENT>errs = np.full_like(times, <NUM_LIT:0.0>)<EOL><DEDENT>nflares = npr.randint(paramdists['<STR_LIT>'][<NUM_LIT:0>],<EOL>high=paramdists['<STR_LIT>'][<NUM_LIT:1>])<EOL>flarepeaktimes = (<EOL>npr.random(<EOL>size=nflares<EOL>)*(times.max() - times.min()) + times.min()<EOL>)<EOL>params = {'<STR_LIT>':nflares}<EOL>for flareind, peaktime in zip(range(nflares), flarepeaktimes):<EOL><INDENT>amp = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>risestdev = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>decayconst = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>if magsarefluxes and amp < <NUM_LIT:0.0>:<EOL><INDENT>amp = -amp<EOL><DEDENT>elif not magsarefluxes and amp > <NUM_LIT:0.0>:<EOL><INDENT>amp = -amp<EOL><DEDENT>modelmags, ptimes, pmags, perrs = (<EOL>flares.flare_model(<EOL>[amp, peaktime, risestdev, decayconst],<EOL>times,<EOL>mags,<EOL>errs<EOL>)<EOL>)<EOL>mags = modelmags<EOL>params[flareind] = {'<STR_LIT>':peaktime,<EOL>'<STR_LIT>':amp,<EOL>'<STR_LIT>':risestdev,<EOL>'<STR_LIT>':decayconst}<EOL><DEDENT>modeldict = {<EOL>'<STR_LIT>':'<STR_LIT>',<EOL>'<STR_LIT>':params,<EOL>'<STR_LIT>':times,<EOL>'<STR_LIT>':mags,<EOL>'<STR_LIT>':errs,<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':[params[x]['<STR_LIT>']<EOL>for x in range(params['<STR_LIT>'])],<EOL>}<EOL>return modeldict<EOL>

This generates fake flare light curves.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'amplitude', 'nflares', 'risestdev', 'decayconst'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The `flare_peak_time` for each flare will be generated automatically
        between `times.min()` and `times.max()` using a uniform distribution.

        The `amplitude` will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'flare',
             'params': {'amplitude': generated value of flare amplitudes,
                        'nflares': generated value of number of flares,
                        'risestdev': generated value of stdev of rise time,
                        'decayconst': generated value of decay constant,
                        'peaktime': generated value of flare peak time},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varamplitude': the generated amplitude of
                             variability == 'amplitude'}


if fakepfpkl.endswith('<STR_LIT>'):<EOL><INDENT>infd = gzip.open(fakepfpkl,'<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>infd = open(fakepfpkl,'<STR_LIT:rb>')<EOL><DEDENT>fakepf = pickle.load(infd)<EOL>infd.close()<EOL>objectid, lcfbasename = fakepf['<STR_LIT>'], fakepf['<STR_LIT>']<EOL>lcfpath = os.path.join(simbasedir,'<STR_LIT>',lcfbasename)<EOL>if not os.path.exists(lcfpath):<EOL><INDENT>LOGERROR('<STR_LIT>' % (objectid,<EOL>lcfpath))<EOL>return None<EOL><DEDENT>fakelc = lcproc._read_pklc(lcfpath)<EOL>actual_varparams, actual_varperiod, actual_varamplitude, actual_vartype = (<EOL>fakelc['<STR_LIT>'],<EOL>fakelc['<STR_LIT>'],<EOL>fakelc['<STR_LIT>'],<EOL>fakelc['<STR_LIT>']<EOL>)<EOL>actual_moments = fakelc['<STR_LIT>']<EOL>magcols = fakelc['<STR_LIT>']<EOL>pfres = {<EOL>'<STR_LIT>':objectid,<EOL>'<STR_LIT>':simbasedir,<EOL>'<STR_LIT>':magcols,<EOL>'<STR_LIT>':os.path.abspath(lcfpath),<EOL>'<STR_LIT>':os.path.abspath(fakepfpkl),<EOL>'<STR_LIT>':actual_vartype,<EOL>'<STR_LIT>':actual_varperiod,<EOL>'<STR_LIT>':actual_varamplitude,<EOL>'<STR_LIT>':actual_varparams,<EOL>'<STR_LIT>':actual_moments,<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>}<EOL>for magcol in magcols:<EOL><INDENT>for pfm in lcproc.PFMETHODS:<EOL><INDENT>if pfm in fakepf[magcol]:<EOL><INDENT>for rpi, rp in enumerate(<EOL>fakepf[magcol][pfm]['<STR_LIT>']<EOL>):<EOL><INDENT>if ((not np.any(np.isclose(<EOL>rp,<EOL>np.array(pfres['<STR_LIT>']),<EOL>rtol=period_tolerance<EOL>))) and np.isfinite(rp)):<EOL><INDENT>pfres['<STR_LIT>'].append(rp)<EOL>pfres['<STR_LIT>'].append(pfm)<EOL>pfres['<STR_LIT>'].append(magcol)<EOL>if pfm == '<STR_LIT>':<EOL><INDENT>this_lspval = (<EOL>np.max(fakepf[magcol][pfm]['<STR_LIT>']) -<EOL>fakepf[magcol][pfm]['<STR_LIT>'][rpi]<EOL>)<EOL><DEDENT>else:<EOL><INDENT>this_lspval = (<EOL>fakepf[magcol][pfm]['<STR_LIT>'][rpi] /<EOL>np.max(fakepf[magcol][pfm]['<STR_LIT>'])<EOL>)<EOL><DEDENT>pfres['<STR_LIT>'].append(this_lspval)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>pfres['<STR_LIT>'] = np.array(pfres['<STR_LIT>'])<EOL>pfres['<STR_LIT>'] = np.array(pfres['<STR_LIT>'])<EOL>pfres['<STR_LIT>'] = np.array(pfres['<STR_LIT>'])<EOL>pfres['<STR_LIT>'] = np.array(pfres['<STR_LIT>'])<EOL>if (actual_vartype and<EOL>actual_vartype in PERIODIC_VARTYPES and<EOL>np.isfinite(actual_varperiod)):<EOL><INDENT>if pfres['<STR_LIT>'].size > <NUM_LIT:0>:<EOL><INDENT>for ri in range(pfres['<STR_LIT>'].size):<EOL><INDENT>pfres['<STR_LIT>'].append(pfres['<STR_LIT>'][ri] -<EOL>np.asscalar(actual_varperiod))<EOL>pfres['<STR_LIT>'].append(<EOL>check_periodrec_alias(actual_varperiod,<EOL>pfres['<STR_LIT>'][ri],<EOL>tolerance=period_tolerance)<EOL>)<EOL><DEDENT>pfres['<STR_LIT>'] = np.array(pfres['<STR_LIT>'])<EOL>pfres['<STR_LIT>'] = np.array(pfres['<STR_LIT>'])<EOL>rec_absdiff = np.abs(pfres['<STR_LIT>'])<EOL>best_recp_ind = rec_absdiff == rec_absdiff.min()<EOL>pfres['<STR_LIT>'] = (<EOL>pfres['<STR_LIT>'][best_recp_ind]<EOL>)<EOL>pfres['<STR_LIT>'] = (<EOL>pfres['<STR_LIT>'][best_recp_ind]<EOL>)<EOL>pfres['<STR_LIT>'] = (<EOL>pfres['<STR_LIT>'][best_recp_ind]<EOL>)<EOL>pfres['<STR_LIT>'] = (<EOL>pfres['<STR_LIT>'][best_recp_ind]<EOL>)<EOL>pfres['<STR_LIT>'] = (<EOL>pfres['<STR_LIT>'][best_recp_ind]<EOL>)<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING(<EOL>'<STR_LIT>' %<EOL>fakepfpkl<EOL>)<EOL>pfres['<STR_LIT>'] = np.array(['<STR_LIT>'])<EOL>pfres['<STR_LIT>'] = np.array([np.nan])<EOL>pfres['<STR_LIT>'] = np.array([np.nan])<EOL>pfres['<STR_LIT>'] = np.array([],dtype=np.unicode_)<EOL>pfres['<STR_LIT>'] = np.array([],dtype=np.unicode_)<EOL>pfres['<STR_LIT>'] = np.array([],dtype=np.unicode_)<EOL>pfres['<STR_LIT>'] = np.array([np.nan])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>pfres['<STR_LIT>'] = np.array(<EOL>['<STR_LIT>']*pfres['<STR_LIT>'].size<EOL>)<EOL>pfres['<STR_LIT>'] = np.zeros(pfres['<STR_LIT>'].size)<EOL>pfres['<STR_LIT>'] = np.array([np.nan])<EOL>pfres['<STR_LIT>'] = np.array([],dtype=np.unicode_)<EOL>pfres['<STR_LIT>'] = np.array([],dtype=np.unicode_)<EOL>pfres['<STR_LIT>'] = np.array(['<STR_LIT>'])<EOL>pfres['<STR_LIT>'] = np.array([np.nan])<EOL><DEDENT>return pfres<EOL>

Recovers the periodic variable status/info for the simulated PF result.

    - Uses simbasedir and the lcfbasename stored in fakepfpkl to figure out
      where the LC for this object is.
    - Gets the actual_varparams, actual_varperiod, actual_vartype,
      actual_varamplitude elements from the LC.
    - Figures out if the current objectid is a periodic variable (using
      actual_vartype).
    - If it is a periodic variable, gets the canonical period assigned to it.
    - Checks if the period was recovered in any of the five best periods
      reported by any of the period-finders, checks if the period recovered was
      a harmonic of the period.
    - Returns the objectid, actual period and vartype, recovered period, and
      recovery status.

    Parameters
    ----------

    fakepfpkl : str
        This is a periodfinding-<objectid>.pkl[.gz] file produced in the
        `simbasedir/periodfinding` subdirectory after `run_periodfinding` above
        is done.

    simbasedir : str
        The base directory where all of the fake LCs and period-finding results
        are.

    period_tolerance : float
        The maximum difference that this function will consider between an
        actual period (or its aliases) and a recovered period to consider it as
        as a 'recovered' period.

    Returns
    -------

    dict
        Returns a dict of period-recovery results.


<EOL>with open(os.path.join(simbasedir, '<STR_LIT>'),'<STR_LIT:rb>') as infd:<EOL><INDENT>siminfo = pickle.load(infd)<EOL><DEDENT>objectids = siminfo['<STR_LIT>']<EOL>varflags = siminfo['<STR_LIT>']<EOL>sdssr = siminfo['<STR_LIT>']<EOL>timecols = siminfo['<STR_LIT>']<EOL>magcols = siminfo['<STR_LIT>']<EOL>errcols = siminfo['<STR_LIT>']<EOL>fakelc_formatkey = '<STR_LIT>' % siminfo['<STR_LIT>']<EOL>lcproc.register_lcformat(<EOL>fakelc_formatkey,<EOL>'<STR_LIT>',<EOL>timecols,<EOL>magcols,<EOL>errcols,<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>magsarefluxes=siminfo['<STR_LIT>']<EOL>)<EOL>outdir = os.path.join(simbasedir, '<STR_LIT>')<EOL>if not os.path.exists(outdir):<EOL><INDENT>os.mkdir(outdir)<EOL><DEDENT>varfeaturedir = os.path.join(simbasedir, '<STR_LIT>')<EOL>varthreshinfof = os.path.join(<EOL>outdir,<EOL>'<STR_LIT>' % (magbinmedian,<EOL>stetson_stdev_min,<EOL>inveta_stdev_min)<EOL>)<EOL>varthresh = varthreshold.variability_threshold(<EOL>varfeaturedir,<EOL>varthreshinfof,<EOL>lcformat=fakelc_formatkey,<EOL>min_stetj_stdev=stetson_stdev_min,<EOL>min_inveta_stdev=inveta_stdev_min,<EOL>min_iqr_stdev=iqr_stdev_min,<EOL>verbose=False<EOL>)<EOL>magbins = varthresh['<STR_LIT>']<EOL>magbininds = np.digitize(sdssr, magbins)<EOL>binned_objectids = []<EOL>binned_actualvars = []<EOL>binned_actualnotvars = []<EOL>for mbinind, _magi in zip(np.unique(magbininds),<EOL>range(len(magbins)-<NUM_LIT:1>)):<EOL><INDENT>thisbinind = np.where(magbininds == mbinind)<EOL>thisbin_objectids = objectids[thisbinind]<EOL>thisbin_varflags = varflags[thisbinind]<EOL>thisbin_actualvars = thisbin_objectids[thisbin_varflags]<EOL>thisbin_actualnotvars = thisbin_objectids[~thisbin_varflags]<EOL>binned_objectids.append(thisbin_objectids)<EOL>binned_actualvars.append(thisbin_actualvars)<EOL>binned_actualnotvars.append(thisbin_actualnotvars)<EOL><DEDENT>recdict = {<EOL>'<STR_LIT>':simbasedir,<EOL>'<STR_LIT>':timecols,<EOL>'<STR_LIT>':magcols,<EOL>'<STR_LIT>':errcols,<EOL>'<STR_LIT>':siminfo['<STR_LIT>'],<EOL>'<STR_LIT>':stetson_stdev_min,<EOL>'<STR_LIT>':inveta_stdev_min,<EOL>'<STR_LIT>':iqr_stdev_min,<EOL>'<STR_LIT>':magbinmedian,<EOL>}<EOL>for magcol in magcols:<EOL><INDENT>magbinind = np.where(<EOL>np.array(varthresh[magcol]['<STR_LIT>']) == magbinmedian<EOL>)<EOL>magbinind = np.asscalar(magbinind[<NUM_LIT:0>])<EOL>thisbin_objectids = binned_objectids[magbinind]<EOL>thisbin_actualvars = binned_actualvars[magbinind]<EOL>thisbin_actualnotvars = binned_actualnotvars[magbinind]<EOL>stet_recoveredvars = varthresh[magcol][<EOL>'<STR_LIT>'<EOL>][magbinind]<EOL>stet_recoverednotvars = np.setdiff1d(thisbin_objectids,<EOL>stet_recoveredvars)<EOL>stet_truepositives = np.intersect1d(stet_recoveredvars,<EOL>thisbin_actualvars)<EOL>stet_falsepositives = np.intersect1d(stet_recoveredvars,<EOL>thisbin_actualnotvars)<EOL>stet_truenegatives = np.intersect1d(stet_recoverednotvars,<EOL>thisbin_actualnotvars)<EOL>stet_falsenegatives = np.intersect1d(stet_recoverednotvars,<EOL>thisbin_actualvars)<EOL>stet_recall = recall(stet_truepositives.size,<EOL>stet_falsenegatives.size)<EOL>stet_precision = precision(stet_truepositives.size,<EOL>stet_falsepositives.size)<EOL>stet_mcc = matthews_correl_coeff(stet_truepositives.size,<EOL>stet_truenegatives.size,<EOL>stet_falsepositives.size,<EOL>stet_falsenegatives.size)<EOL>inveta_recoveredvars = varthresh[magcol][<EOL>'<STR_LIT>'<EOL>][magbinind]<EOL>inveta_recoverednotvars = np.setdiff1d(thisbin_objectids,<EOL>inveta_recoveredvars)<EOL>inveta_truepositives = np.intersect1d(inveta_recoveredvars,<EOL>thisbin_actualvars)<EOL>inveta_falsepositives = np.intersect1d(inveta_recoveredvars,<EOL>thisbin_actualnotvars)<EOL>inveta_truenegatives = np.intersect1d(inveta_recoverednotvars,<EOL>thisbin_actualnotvars)<EOL>inveta_falsenegatives = np.intersect1d(inveta_recoverednotvars,<EOL>thisbin_actualvars)<EOL>inveta_recall = recall(inveta_truepositives.size,<EOL>inveta_falsenegatives.size)<EOL>inveta_precision = precision(inveta_truepositives.size,<EOL>inveta_falsepositives.size)<EOL>inveta_mcc = matthews_correl_coeff(inveta_truepositives.size,<EOL>inveta_truenegatives.size,<EOL>inveta_falsepositives.size,<EOL>inveta_falsenegatives.size)<EOL>iqr_recoveredvars = varthresh[magcol][<EOL>'<STR_LIT>'<EOL>][magbinind]<EOL>iqr_recoverednotvars = np.setdiff1d(thisbin_objectids,<EOL>iqr_recoveredvars)<EOL>iqr_truepositives = np.intersect1d(iqr_recoveredvars,<EOL>thisbin_actualvars)<EOL>iqr_falsepositives = np.intersect1d(iqr_recoveredvars,<EOL>thisbin_actualnotvars)<EOL>iqr_truenegatives = np.intersect1d(iqr_recoverednotvars,<EOL>thisbin_actualnotvars)<EOL>iqr_falsenegatives = np.intersect1d(iqr_recoverednotvars,<EOL>thisbin_actualvars)<EOL>iqr_recall = recall(iqr_truepositives.size,<EOL>iqr_falsenegatives.size)<EOL>iqr_precision = precision(iqr_truepositives.size,<EOL>iqr_falsepositives.size)<EOL>iqr_mcc = matthews_correl_coeff(iqr_truepositives.size,<EOL>iqr_truenegatives.size,<EOL>iqr_falsepositives.size,<EOL>iqr_falsenegatives.size)<EOL>stet_missed_inveta_found = np.setdiff1d(inveta_truepositives,<EOL>stet_truepositives)<EOL>stet_missed_iqr_found = np.setdiff1d(iqr_truepositives,<EOL>stet_truepositives)<EOL>inveta_missed_stet_found = np.setdiff1d(stet_truepositives,<EOL>inveta_truepositives)<EOL>inveta_missed_iqr_found = np.setdiff1d(iqr_truepositives,<EOL>inveta_truepositives)<EOL>iqr_missed_stet_found = np.setdiff1d(stet_truepositives,<EOL>iqr_truepositives)<EOL>iqr_missed_inveta_found = np.setdiff1d(inveta_truepositives,<EOL>iqr_truepositives)<EOL>if not statsonly:<EOL><INDENT>recdict[magcol] = {<EOL>'<STR_LIT>':stet_recoveredvars,<EOL>'<STR_LIT>':stet_truepositives,<EOL>'<STR_LIT>':stet_falsepositives,<EOL>'<STR_LIT>':stet_truenegatives,<EOL>'<STR_LIT>':stet_falsenegatives,<EOL>'<STR_LIT>':stet_precision,<EOL>'<STR_LIT>':stet_recall,<EOL>'<STR_LIT>':stet_mcc,<EOL>'<STR_LIT>':inveta_recoveredvars,<EOL>'<STR_LIT>':inveta_truepositives,<EOL>'<STR_LIT>':inveta_falsepositives,<EOL>'<STR_LIT>':inveta_truenegatives,<EOL>'<STR_LIT>':inveta_falsenegatives,<EOL>'<STR_LIT>':inveta_precision,<EOL>'<STR_LIT>':inveta_recall,<EOL>'<STR_LIT>':inveta_mcc,<EOL>'<STR_LIT>':iqr_recoveredvars,<EOL>'<STR_LIT>':iqr_truepositives,<EOL>'<STR_LIT>':iqr_falsepositives,<EOL>'<STR_LIT>':iqr_truenegatives,<EOL>'<STR_LIT>':iqr_falsenegatives,<EOL>'<STR_LIT>':iqr_precision,<EOL>'<STR_LIT>':iqr_recall,<EOL>'<STR_LIT>':iqr_mcc,<EOL>'<STR_LIT>':stet_missed_inveta_found,<EOL>'<STR_LIT>':stet_missed_iqr_found,<EOL>'<STR_LIT>':inveta_missed_stet_found,<EOL>'<STR_LIT>':inveta_missed_iqr_found,<EOL>'<STR_LIT>':iqr_missed_stet_found,<EOL>'<STR_LIT>':iqr_missed_inveta_found,<EOL>'<STR_LIT>':thisbin_actualvars,<EOL>'<STR_LIT>':thisbin_actualnotvars,<EOL>'<STR_LIT>':thisbin_objectids,<EOL>'<STR_LIT>':magbinind,<EOL>}<EOL><DEDENT>else:<EOL><INDENT>recdict[magcol] = {<EOL>'<STR_LIT>':stet_recoveredvars.size,<EOL>'<STR_LIT>':stet_truepositives.size,<EOL>'<STR_LIT>':stet_falsepositives.size,<EOL>'<STR_LIT>':stet_truenegatives.size,<EOL>'<STR_LIT>':stet_falsenegatives.size,<EOL>'<STR_LIT>':stet_precision,<EOL>'<STR_LIT>':stet_recall,<EOL>'<STR_LIT>':stet_mcc,<EOL>'<STR_LIT>':inveta_recoveredvars.size,<EOL>'<STR_LIT>':inveta_truepositives.size,<EOL>'<STR_LIT>':inveta_falsepositives.size,<EOL>'<STR_LIT>':inveta_truenegatives.size,<EOL>'<STR_LIT>':inveta_falsenegatives.size,<EOL>'<STR_LIT>':inveta_precision,<EOL>'<STR_LIT>':inveta_recall,<EOL>'<STR_LIT>':inveta_mcc,<EOL>'<STR_LIT>':iqr_recoveredvars.size,<EOL>'<STR_LIT>':iqr_truepositives.size,<EOL>'<STR_LIT>':iqr_falsepositives.size,<EOL>'<STR_LIT>':iqr_truenegatives.size,<EOL>'<STR_LIT>':iqr_falsenegatives.size,<EOL>'<STR_LIT>':iqr_precision,<EOL>'<STR_LIT>':iqr_recall,<EOL>'<STR_LIT>':iqr_mcc,<EOL>'<STR_LIT>':stet_missed_inveta_found.size,<EOL>'<STR_LIT>':stet_missed_iqr_found.size,<EOL>'<STR_LIT>':inveta_missed_stet_found.size,<EOL>'<STR_LIT>':inveta_missed_iqr_found.size,<EOL>'<STR_LIT>':iqr_missed_stet_found.size,<EOL>'<STR_LIT>':iqr_missed_inveta_found.size,<EOL>'<STR_LIT>':thisbin_actualvars.size,<EOL>'<STR_LIT>':thisbin_actualnotvars.size,<EOL>'<STR_LIT>':thisbin_objectids.size,<EOL>'<STR_LIT>':magbinind,<EOL>}<EOL><DEDENT><DEDENT>return recdict<EOL>

This runs variability selection for the given magbinmedian.

    To generate a full recovery matrix over all magnitude bins, run this
    function for each magbin over the specified stetson_stdev_min and
    inveta_stdev_min grid.

    Parameters
    ----------

    simbasedir : str
        The input directory of fake LCs.

    magbinmedian : float
        The magbin to run the variable recovery for. This is an item from the
        dict from `simbasedir/fakelcs-info.pkl: `fakelcinfo['magrms'][magcol]`
        list for each magcol and designates which magbin to get the recovery
        stats for.

    stetson_stdev_min : float
        The minimum sigma above the trend in the Stetson J variability index
        distribution for this magbin to use to consider objects as variable.

    inveta_stdev_min : float
        The minimum sigma above the trend in the 1/eta variability index
        distribution for this magbin to use to consider objects as variable.

    iqr_stdev_min : float
        The minimum sigma above the trend in the IQR variability index
        distribution for this magbin to use to consider objects as variable.

    statsonly : bool
        If this is True, only the final stats will be returned. If False, the
        full arrays used to generate the stats will also be returned.

    Returns
    -------

    dict
        The returned dict contains statistics for this magbin and if requested,
        the full arrays used to calculate the statistics.


try:<EOL><INDENT>with open(fakelcfile,'<STR_LIT:rb>') as infd:<EOL><INDENT>lcdict = pickle.load(infd)<EOL><DEDENT><DEDENT>except UnicodeDecodeError:<EOL><INDENT>with open(fakelcfile,'<STR_LIT:rb>') as infd:<EOL><INDENT>lcdict = pickle.load(infd, encoding='<STR_LIT>')<EOL><DEDENT><DEDENT>return lcdict<EOL>

This just reads a pickled fake LC.

Parameters
----------

fakelcfile : str
    The fake LC file to read.

Returns
-------

dict
    This returns an lcdict.


<EOL>with open(os.path.join(simbasedir, '<STR_LIT>'),'<STR_LIT:rb>') as infd:<EOL><INDENT>siminfo = pickle.load(infd)<EOL><DEDENT>lcfpaths = siminfo['<STR_LIT>']<EOL>varfeaturedir = os.path.join(simbasedir,'<STR_LIT>')<EOL>timecols = siminfo['<STR_LIT>']<EOL>magcols = siminfo['<STR_LIT>']<EOL>errcols = siminfo['<STR_LIT>']<EOL>timecols = siminfo['<STR_LIT>']<EOL>magcols = siminfo['<STR_LIT>']<EOL>errcols = siminfo['<STR_LIT>']<EOL>fakelc_formatkey = '<STR_LIT>' % siminfo['<STR_LIT>']<EOL>lcproc.register_lcformat(<EOL>fakelc_formatkey,<EOL>'<STR_LIT>',<EOL>timecols,<EOL>magcols,<EOL>errcols,<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>magsarefluxes=siminfo['<STR_LIT>']<EOL>)<EOL>varinfo = lcvfeatures.parallel_varfeatures(lcfpaths,<EOL>varfeaturedir,<EOL>lcformat=fakelc_formatkey,<EOL>mindet=mindet,<EOL>nworkers=nworkers)<EOL>with open(os.path.join(simbasedir,'<STR_LIT>'),'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(varinfo, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>return os.path.join(simbasedir,'<STR_LIT>')<EOL>

This runs `lcproc.lcvfeatures.parallel_varfeatures` on fake LCs in
    `simbasedir`.

    Parameters
    ----------

    simbasedir : str
        The directory containing the fake LCs to process.

    mindet : int
        The minimum number of detections needed to accept an LC and process it.

    nworkers : int or None
        The number of parallel workers to use when extracting variability
        features from the input light curves.

    Returns
    -------

    str
        The path to the `varfeatures` pickle created after running the
        `lcproc.lcvfeatures.parallel_varfeatures` function.


if (ntp+nfn) > <NUM_LIT:0>:<EOL><INDENT>return ntp/(ntp+nfn)<EOL><DEDENT>else:<EOL><INDENT>return np.nan<EOL><DEDENT>

This calculates recall.

https://en.wikipedia.org/wiki/Precision_and_recall

Parameters
----------

ntp : int
    The number of true positives.

nfn : int
    The number of false negatives.

Returns
-------

float
    The precision calculated using `ntp/(ntp + nfn)`.