try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(fileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if pfpickle.endswith('<STR_LIT>'):<EOL><INDENT>infd = gzip.open(pfpickle)<EOL><DEDENT>else:<EOL><INDENT>infd = open(pfpickle)<EOL><DEDENT>pf = pickle.load(infd)<EOL>infd.close()<EOL>lcfile = os.path.join(lcbasedir, pf['<STR_LIT>'])<EOL>objectid = pf['<STR_LIT>']<EOL>if '<STR_LIT>' in pf:<EOL><INDENT>kwargs = pf['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>kwargs = None<EOL><DEDENT>if kwargs and '<STR_LIT>' in kwargs and timecols is None:<EOL><INDENT>timecols = kwargs['<STR_LIT>']<EOL><DEDENT>elif not kwargs and not timecols:<EOL><INDENT>timecols = dtimecols<EOL><DEDENT>if kwargs and '<STR_LIT>' in kwargs and magcols is None:<EOL><INDENT>magcols = kwargs['<STR_LIT>']<EOL><DEDENT>elif not kwargs and not magcols:<EOL><INDENT>magcols = dmagcols<EOL><DEDENT>if kwargs and '<STR_LIT>' in kwargs and errcols is None:<EOL><INDENT>errcols = kwargs['<STR_LIT>']<EOL><DEDENT>elif not kwargs and not errcols:<EOL><INDENT>errcols = derrcols<EOL><DEDENT>if not os.path.exists(lcfile):<EOL><INDENT>LOGERROR("<STR_LIT>" % (lcfile, objectid))<EOL>return None<EOL><DEDENT>if starfeatures is not None and os.path.exists(starfeatures):<EOL><INDENT>with open(starfeatures,'<STR_LIT:rb>') as infd:<EOL><INDENT>starfeat = pickle.load(infd)<EOL><DEDENT>if starfeat['<STR_LIT>'].size > <NUM_LIT:0>:<EOL><INDENT>nbr_full_lcf = starfeat['<STR_LIT>'][<NUM_LIT:0>]<EOL>if os.path.exists(os.path.join(lcbasedir,<EOL>os.path.basename(nbr_full_lcf))):<EOL><INDENT>nbrlcf = os.path.join(lcbasedir,<EOL>os.path.basename(nbr_full_lcf))<EOL><DEDENT>elif os.path.exists(nbr_full_lcf):<EOL><INDENT>nbrlcf = nbr_full_lcf<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" %<EOL>(os.path.basename(nbr_full_lcf),<EOL>os.path.dirname(nbr_full_lcf),<EOL>lcbasedir))<EOL>nbrlcf = None<EOL><DEDENT><DEDENT>else:<EOL><INDENT>nbrlcf = None<EOL><DEDENT><DEDENT>else:<EOL><INDENT>nbrlcf = None<EOL><DEDENT>try:<EOL><INDENT>lcdict = readerfunc(lcfile)<EOL>if ( (isinstance(lcdict, (list, tuple))) and<EOL>(isinstance(lcdict[<NUM_LIT:0>], dict)) ):<EOL><INDENT>lcdict = lcdict[<NUM_LIT:0>]<EOL><DEDENT>if nbrlcf is not None:<EOL><INDENT>nbrlcdict = readerfunc(nbrlcf)<EOL>if ( (isinstance(nbrlcdict, (list, tuple))) and<EOL>(isinstance(nbrlcdict[<NUM_LIT:0>], dict)) ):<EOL><INDENT>nbrlcdict = nbrlcdict[<NUM_LIT:0>]<EOL><DEDENT><DEDENT>outfile = os.path.join(outdir, '<STR_LIT>' %<EOL>squeeze(objectid).replace('<STR_LIT:U+0020>','<STR_LIT:->'))<EOL>if normfunc is not None:<EOL><INDENT>lcdict = normfunc(lcdict)<EOL>if nbrlcf:<EOL><INDENT>nbrlcdict = normfunc(nbrlcdict)<EOL><DEDENT><DEDENT>resultdict = {}<EOL>for tcol, mcol, ecol in zip(timecols, magcols, errcols):<EOL><INDENT>if '<STR_LIT:.>' in tcol:<EOL><INDENT>tcolget = tcol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>tcolget = [tcol]<EOL><DEDENT>times = _dict_get(lcdict, tcolget)<EOL>if nbrlcf:<EOL><INDENT>nbrtimes = _dict_get(nbrlcdict, tcolget)<EOL><DEDENT>else:<EOL><INDENT>nbrtimes = None<EOL><DEDENT>if '<STR_LIT:.>' in mcol:<EOL><INDENT>mcolget = mcol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>mcolget = [mcol]<EOL><DEDENT>mags = _dict_get(lcdict, mcolget)<EOL>if nbrlcf:<EOL><INDENT>nbrmags = _dict_get(nbrlcdict, mcolget)<EOL><DEDENT>else:<EOL><INDENT>nbrmags = None<EOL><DEDENT>if '<STR_LIT:.>' in ecol:<EOL><INDENT>ecolget = ecol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>ecolget = [ecol]<EOL><DEDENT>errs = _dict_get(lcdict, ecolget)<EOL>if nbrlcf:<EOL><INDENT>nbrerrs = _dict_get(nbrlcdict, ecolget)<EOL><DEDENT>else:<EOL><INDENT>nbrerrs = None<EOL><DEDENT>finind = np.isfinite(times) & np.isfinite(mags) & np.isfinite(errs)<EOL>ftimes, fmags, ferrs = times[finind], mags[finind], errs[finind]<EOL>if nbrlcf:<EOL><INDENT>nfinind = (np.isfinite(nbrtimes) &<EOL>np.isfinite(nbrmags) &<EOL>np.isfinite(nbrerrs))<EOL>nbrftimes, nbrfmags, nbrferrs = (nbrtimes[nfinind],<EOL>nbrmags[nfinind],<EOL>nbrerrs[nfinind])<EOL><DEDENT>nzind = np.nonzero(ferrs)<EOL>ftimes, fmags, ferrs = ftimes[nzind], fmags[nzind], ferrs[nzind]<EOL>if nbrlcf:<EOL><INDENT>nnzind = np.nonzero(nbrferrs)<EOL>nbrftimes, nbrfmags, nbrferrs = (nbrftimes[nnzind],<EOL>nbrfmags[nnzind],<EOL>nbrferrs[nnzind])<EOL><DEDENT>if normfunc is None:<EOL><INDENT>ntimes, nmags = normalize_magseries(<EOL>ftimes, fmags,<EOL>magsarefluxes=magsarefluxes<EOL>)<EOL>times, mags, errs = ntimes, nmags, ferrs<EOL>if nbrlcf:<EOL><INDENT>nbrntimes, nbrnmags = normalize_magseries(<EOL>nbrftimes, nbrfmags,<EOL>magsarefluxes=magsarefluxes<EOL>)<EOL>nbrtimes, nbrmags, nbrerrs = nbrntimes, nbrnmags, nbrferrs<EOL><DEDENT>else:<EOL><INDENT>nbrtimes, nbrmags, nbrerrs = None, None, None<EOL><DEDENT><DEDENT>else:<EOL><INDENT>times, mags, errs = ftimes, fmags, ferrs<EOL><DEDENT>if times.size > <NUM_LIT>:<EOL><INDENT>available_pfmethods = []<EOL>available_pgrams = []<EOL>available_bestperiods = []<EOL>for k in pf[mcol].keys():<EOL><INDENT>if k in PFMETHODS:<EOL><INDENT>available_pgrams.append(pf[mcol][k])<EOL>if k != '<STR_LIT>':<EOL><INDENT>available_pfmethods.append(<EOL>pf[mcol][k]['<STR_LIT>']<EOL>)<EOL>available_bestperiods.append(<EOL>pf[mcol][k]['<STR_LIT>']<EOL>)<EOL><DEDENT><DEDENT><DEDENT>featkey = '<STR_LIT>' % mcol<EOL>resultdict[featkey] = {}<EOL>pgramfeat = periodicfeatures.periodogram_features(<EOL>available_pgrams, times, mags, errs,<EOL>sigclip=sigclip,<EOL>pdiff_threshold=pdiff_threshold,<EOL>sidereal_threshold=sidereal_threshold,<EOL>sampling_peak_multiplier=sampling_peak_multiplier,<EOL>sampling_startp=sampling_startp,<EOL>sampling_endp=sampling_endp,<EOL>verbose=verbose<EOL>)<EOL>resultdict[featkey].update(pgramfeat)<EOL>resultdict[featkey]['<STR_LIT>'] = available_pfmethods<EOL>for _ind, pfm, bp in zip(range(len(available_bestperiods)),<EOL>available_pfmethods,<EOL>available_bestperiods):<EOL><INDENT>resultdict[featkey][pfm] = periodicfeatures.lcfit_features(<EOL>times, mags, errs, bp,<EOL>fourierorder=fourierorder,<EOL>transitparams=transitparams,<EOL>ebparams=ebparams,<EOL>sigclip=sigclip,<EOL>magsarefluxes=magsarefluxes,<EOL>verbose=verbose<EOL>)<EOL>phasedlcfeat = periodicfeatures.phasedlc_features(<EOL>times, mags, errs, bp,<EOL>nbrtimes=nbrtimes,<EOL>nbrmags=nbrmags,<EOL>nbrerrs=nbrerrs<EOL>)<EOL>resultdict[featkey][pfm].update(phasedlcfeat)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% (mcol, pfpickle))<EOL>featkey = '<STR_LIT>' % mcol<EOL>resultdict[featkey] = None<EOL><DEDENT><DEDENT>outfile = os.path.join(outdir, '<STR_LIT>' %<EOL>squeeze(objectid).replace('<STR_LIT:U+0020>','<STR_LIT:->'))<EOL>with open(outfile,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(resultdict, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>return outfile<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' %<EOL>(pfpickle, lcfile))<EOL>if raiseonfail:<EOL><INDENT>raise<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT><DEDENT>

This gets all periodic features for the object.

    Parameters
    ----------

    pfpickle : str
        The period-finding result pickle containing period-finder results to use
        for the calculation of LC fit, periodogram, and phased LC features.

    lcbasedir : str
        The base directory where the light curve for the current object is
        located.

    outdir : str
        The output directory where the results will be written.

    fourierorder : int
        The Fourier order to use to generate sinusoidal function and fit that to
        the phased light curve.

    transitparams : list of floats
        The transit depth, duration, and ingress duration to use to generate a
        trapezoid planet transit model fit to the phased light curve. The period
        used is the one provided in `period`, while the epoch is automatically
        obtained from a spline fit to the phased light curve.

    ebparams : list of floats
        The primary eclipse depth, eclipse duration, the primary-secondary depth
        ratio, and the phase of the secondary eclipse to use to generate an
        eclipsing binary model fit to the phased light curve. The period used is
        the one provided in `period`, while the epoch is automatically obtained
        from a spline fit to the phased light curve.

    pdiff_threshold : float
        This is the max difference between periods to consider them the same.

    sidereal_threshold : float
        This is the max difference between any of the 'best' periods and the
        sidereal day periods to consider them the same.

    sampling_peak_multiplier : float
        This is the minimum multiplicative factor of a 'best' period's
        normalized periodogram peak over the sampling periodogram peak at the
        same period required to accept the 'best' period as possibly real.

    sampling_startp, sampling_endp : float
        If the `pgramlist` doesn't have a time-sampling Lomb-Scargle
        periodogram, it will be obtained automatically. Use these kwargs to
        control the minimum and maximum period interval to be searched when
        generating this periodogram.

    starfeatures : str or None
        If not None, this should be the filename of the
        `starfeatures-<objectid>.pkl` created by
        :py:func:`astrobase.lcproc.lcsfeatures.get_starfeatures` for this
        object. This is used to get the neighbor's light curve and phase it with
        this object's period to see if this object is blended.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    verbose : bool
        If True, will indicate progress while working.

    raiseonfail : bool
        If True, will raise an Exception if something goes wrong.

    Returns
    -------

    str
        Returns a filename for the output pickle containing all of the periodic
        features for the input object's LC.


try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(fileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if timecols is None:<EOL><INDENT>timecols = dtimecols<EOL><DEDENT>if magcols is None:<EOL><INDENT>magcols = dmagcols<EOL><DEDENT>if errcols is None:<EOL><INDENT>errcols = derrcols<EOL><DEDENT>outdict = {}<EOL>for t, m, e in zip(timecols, magcols, errcols):<EOL><INDENT>tasks = [(x, t, m, e, externalparams, lcformat, lcformatdir,<EOL>epdsmooth_sigclip, epdsmooth_windowsize,<EOL>epdsmooth_func, epdsmooth_extraparams) for<EOL>x in lclist]<EOL>pool = mp.Pool(nworkers, maxtasksperchild=maxworkertasks)<EOL>results = pool.map(parallel_epd_worker, tasks)<EOL>pool.close()<EOL>pool.join()<EOL>outdict[m] = results<EOL><DEDENT>return outdict<EOL>

This applies EPD in parallel to all LCs in the input list.

    Parameters
    ----------

    lclist : list of str
        This is the list of light curve files to run EPD on.

    externalparams : dict or None
        This is a dict that indicates which keys in the lcdict obtained from the
        lcfile correspond to the required external parameters. As with timecol,
        magcol, and errcol, these can be simple keys (e.g. 'rjd') or compound
        keys ('magaperture1.mags'). The dict should look something like::

          {'fsv':'<lcdict key>' array: S values for each observation,
           'fdv':'<lcdict key>' array: D values for each observation,
           'fkv':'<lcdict key>' array: K values for each observation,
           'xcc':'<lcdict key>' array: x coords for each observation,
           'ycc':'<lcdict key>' array: y coords for each observation,
           'bgv':'<lcdict key>' array: sky background for each observation,
           'bge':'<lcdict key>' array: sky background err for each observation,
           'iha':'<lcdict key>' array: hour angle for each observation,
           'izd':'<lcdict key>' array: zenith distance for each observation}

        Alternatively, if these exact keys are already present in the lcdict,
        indicate this by setting externalparams to None.

    timecols,magcols,errcols : lists of str
        The keys in the lcdict produced by your light curve reader function that
        correspond to the times, mags/fluxes, and associated measurement errors
        that will be used as inputs to the EPD process. If these are None, the
        default values for `timecols`, `magcols`, and `errcols` for your light
        curve format will be used here.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curve files.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    epdsmooth_sigclip : float or int or sequence of two floats/ints or None
        This specifies how to sigma-clip the input LC before fitting the EPD
        function to it.

        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    epdsmooth_windowsize : int
        This is the number of LC points to smooth over to generate a smoothed
        light curve that will be used to fit the EPD function.

    epdsmooth_func : Python function
        This sets the smoothing filter function to use. A Savitsky-Golay filter
        is used to smooth the light curve by default. The functions that can be
        used with this kwarg are listed in `varbase.trends`. If you want to use
        your own function, it MUST have the following signature::

                def smoothfunc(mags_array, window_size, **extraparams)

        and return a numpy array of the same size as `mags_array` with the
        smoothed time-series. Any extra params can be provided using the
        `extraparams` dict.

    epdsmooth_extraparams : dict
        This is a dict of any extra filter params to supply to the smoothing
        function.

    nworkers : int
        The number of parallel workers to launch when processing the LCs.

    maxworkertasks : int
        The maximum number of tasks a parallel worker will complete before it is
        replaced with a new one (sometimes helps with memory-leaks).

    Returns
    -------

    dict
        Returns a dict organized by all the keys in the input `magcols` list,
        containing lists of EPD pickle light curves for that `magcol`.

    Notes
    -----

    - S -> measure of PSF sharpness (~1/sigma^2 sosmaller S = wider PSF)
    - D -> measure of PSF ellipticity in xy direction
    - K -> measure of PSF ellipticity in cross direction

    S, D, K are related to the PSF's variance and covariance, see eqn 30-33 in
    A. Pal's thesis: https://arxiv.org/abs/0906.3486


pfpickle, outdir, lcbasedir, kwargs = task<EOL>try:<EOL><INDENT>return runcp(pfpickle, outdir, lcbasedir, **kwargs)<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' % (pfpickle, e))<EOL>return None<EOL><DEDENT>

This is the worker for running checkplots.

Parameters
----------

task : tuple
    This is of the form: (pfpickle, outdir, lcbasedir, kwargs).

Returns
-------

list of str
    The list of checkplot pickles returned by the `runcp` function.


<EOL>if sys.platform == '<STR_LIT>':<EOL><INDENT>import requests<EOL>requests.get('<STR_LIT>')<EOL><DEDENT>if not os.path.exists(outdir):<EOL><INDENT>os.mkdir(outdir)<EOL><DEDENT>if (liststartindex is not None) and (maxobjects is None):<EOL><INDENT>pfpicklelist = pfpicklelist[liststartindex:]<EOL>if lcfnamelist is not None:<EOL><INDENT>lcfnamelist = lcfnamelist[liststartindex:]<EOL><DEDENT><DEDENT>elif (liststartindex is None) and (maxobjects is not None):<EOL><INDENT>pfpicklelist = pfpicklelist[:maxobjects]<EOL>if lcfnamelist is not None:<EOL><INDENT>lcfnamelist = lcfnamelist[:maxobjects]<EOL><DEDENT><DEDENT>elif (liststartindex is not None) and (maxobjects is not None):<EOL><INDENT>pfpicklelist = (<EOL>pfpicklelist[liststartindex:liststartindex+maxobjects]<EOL>)<EOL>if lcfnamelist is not None:<EOL><INDENT>lcfnamelist = lcfnamelist[liststartindex:liststartindex+maxobjects]<EOL><DEDENT><DEDENT>if lcfnamelist is None:<EOL><INDENT>lcfnamelist = [None]*len(pfpicklelist)<EOL><DEDENT>tasklist = [(x, outdir, lcbasedir,<EOL>{'<STR_LIT>':lcformat,<EOL>'<STR_LIT>':lcformatdir,<EOL>'<STR_LIT>':y,<EOL>'<STR_LIT>':timecols,<EOL>'<STR_LIT>':magcols,<EOL>'<STR_LIT>':errcols,<EOL>'<STR_LIT>':lclistpkl,<EOL>'<STR_LIT>':gaia_max_timeout,<EOL>'<STR_LIT>':gaia_mirror,<EOL>'<STR_LIT>':nbrradiusarcsec,<EOL>'<STR_LIT>':maxnumneighbors,<EOL>'<STR_LIT>':makeneighborlcs,<EOL>'<STR_LIT>':xmatchinfo,<EOL>'<STR_LIT>':xmatchradiusarcsec,<EOL>'<STR_LIT>':sigclip,<EOL>'<STR_LIT>':minobservations,<EOL>'<STR_LIT>':skipdone,<EOL>'<STR_LIT>':cprenorm,<EOL>'<STR_LIT>':fast_mode,<EOL>'<STR_LIT>':done_callback,<EOL>'<STR_LIT>':done_callback_args,<EOL>'<STR_LIT>':done_callback_kwargs}) for<EOL>x,y in zip(pfpicklelist, lcfnamelist)]<EOL>resultfutures = []<EOL>results = []<EOL>with ProcessPoolExecutor(max_workers=nworkers) as executor:<EOL><INDENT>resultfutures = executor.map(runcp_worker, tasklist)<EOL><DEDENT>results = [x for x in resultfutures]<EOL>executor.shutdown()<EOL>return results<EOL>

This drives the parallel execution of `runcp` for a list of periodfinding
    result pickles.

    Parameters
    ----------

    pfpicklelist : list of str or list of Nones
        This is the list of the filenames of the period-finding result pickles
        to process. To make checkplots using the light curves directly, set this
        to a list of Nones with the same length as the list of light curve files
        that you provide in `lcfnamelist`.

    outdir : str
        The directory the checkplot pickles will be written to.

    lcbasedir : str
        The base directory that this function will look in to find the light
        curves pointed to by the period-finding result files. If you're using
        `lcfnamelist` to provide a list of light curve filenames directly, this
        arg is ignored.

    lcfnamelist : list of str or None
        If this is provided, it must be a list of the input light curve
        filenames to process. These can either be associated with each input
        period-finder result pickle, or can be provided standalone to make
        checkplots without phased LC plots in them. In the second case, you must
        set `pfpicklelist` to a list of Nones that matches the length of
        `lcfnamelist`.

    cprenorm : bool
        Set this to True if the light curves should be renormalized by
        `checkplot.checkplot_pickle`. This is set to False by default because we
        do our own normalization in this function using the light curve's
        registered normalization function and pass the normalized times, mags,
        errs to the `checkplot.checkplot_pickle` function.

    lclistpkl : str or dict
        This is either the filename of a pickle or the actual dict produced by
        lcproc.make_lclist. This is used to gather neighbor information.

    nbrradiusarcsec : float
        The radius in arcseconds to use for a search conducted around the
        coordinates of this object to look for any potential confusion and
        blending of variability amplitude caused by their proximity.

    maxnumneighbors : int
        The maximum number of neighbors that will have their light curves and
        magnitudes noted in this checkplot as potential blends with the target
        object.

    makeneighborlcs : bool
        If True, will make light curve and phased light curve plots for all
        neighbors found in the object collection for each input object.

    fast_mode : bool or float
        This runs the external catalog operations in a "fast" mode, with short
        timeouts and not trying to hit external catalogs that take a long time
        to respond.

        If this is set to True, the default settings for the external requests
        will then become::

                skyview_lookup = False
                skyview_timeout = 10.0
                skyview_retry_failed = False
                dust_timeout = 10.0
                gaia_submit_timeout = 7.0
                gaia_max_timeout = 10.0
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

        If this is a float, will run in "fast" mode with the provided timeout
        value in seconds and the following settings::

                skyview_lookup = True
                skyview_timeout = fast_mode
                skyview_retry_failed = False
                dust_timeout = fast_mode
                gaia_submit_timeout = 0.66*fast_mode
                gaia_max_timeout = fast_mode
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

    gaia_max_timeout : float
        Sets the timeout in seconds to use when waiting for the GAIA service to
        respond to our request for the object's information. Note that if
        `fast_mode` is set, this is ignored.

    gaia_mirror : str or None
        This sets the GAIA mirror to use. This is a key in the
        `services.gaia.GAIA_URLS` dict which defines the URLs to hit for each
        mirror.

    xmatchinfo : str or dict
        This is either the xmatch dict produced by the function
        `load_xmatch_external_catalogs` above, or the path to the xmatch info
        pickle file produced by that function.

    xmatchradiusarcsec : float
        This is the cross-matching radius to use in arcseconds.

    minobservations : int
        The minimum of observations the input object's mag/flux time-series must
        have for this function to plot its light curve and phased light
        curve. If the object has less than this number, no light curves will be
        plotted, but the checkplotdict will still contain all of the other
        information.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols : list of str or None
        The timecol keys to use from the lcdict in generating this checkplot.

    magcols : list of str or None
        The magcol keys to use from the lcdict in generating this checkplot.

    errcols : list of str or None
        The errcol keys to use from the lcdict in generating this checkplot.

    skipdone : bool
        This indicates if this function will skip creating checkplots that
        already exist corresponding to the current `objectid` and `magcol`. If
        `skipdone` is set to True, this will be done.

    done_callback : Python function or None
        This is used to provide a function to execute after the checkplot
        pickles are generated. This is useful if you want to stream the results
        of checkplot making to some other process, e.g. directly running an
        ingestion into an LCC-Server collection. The function will always get
        the list of the generated checkplot pickles as its first arg, and all of
        the kwargs for runcp in the kwargs dict. Additional args and kwargs can
        be provided by giving a list in the `done_callbacks_args` kwarg and a
        dict in the `done_callbacks_kwargs` kwarg.

        NOTE: the function you pass in here should be pickleable by normal
        Python if you want to use it with the parallel_cp and parallel_cp_lcdir
        functions below.

    done_callback_args : tuple or None
        If not None, contains any args to pass into the `done_callback`
        function.

    done_callback_kwargs : dict or None
        If not None, contains any kwargs to pass into the `done_callback`
        function.

    liststartindex : int
        The index of the `pfpicklelist` (and `lcfnamelist` if provided) to start
        working at.

    maxobjects : int
        The maximum number of objects to process in this run. Use this with
        `liststartindex` to effectively distribute working on a large list of
        input period-finding result pickles (and light curves if `lcfnamelist`
        is also provided) over several sessions or machines.

    nworkers : int
        The number of parallel workers that will work on the checkplot
        generation process.

    Returns
    -------

    dict
        This returns a dict with keys = input period-finding pickles and vals =
        list of the corresponding checkplot pickles produced.


This drives the parallel execution of `runcp` for a directory of
    periodfinding pickles.

    Parameters
    ----------

    pfpickledir : str
        This is the directory containing all of the period-finding pickles to
        process.

    outdir : str
        The directory the checkplot pickles will be written to.

    lcbasedir : str
        The base directory that this function will look in to find the light
        curves pointed to by the period-finding result files. If you're using
        `lcfnamelist` to provide a list of light curve filenames directly, this
        arg is ignored.

    pkpickleglob : str
        This is a UNIX file glob to select period-finding result pickles in the
        specified `pfpickledir`.

    lclistpkl : str or dict
        This is either the filename of a pickle or the actual dict produced by
        lcproc.make_lclist. This is used to gather neighbor information.

    cprenorm : bool
        Set this to True if the light curves should be renormalized by
        `checkplot.checkplot_pickle`. This is set to False by default because we
        do our own normalization in this function using the light curve's
        registered normalization function and pass the normalized times, mags,
        errs to the `checkplot.checkplot_pickle` function.

    nbrradiusarcsec : float
        The radius in arcseconds to use for a search conducted around the
        coordinates of this object to look for any potential confusion and
        blending of variability amplitude caused by their proximity.

    maxnumneighbors : int
        The maximum number of neighbors that will have their light curves and
        magnitudes noted in this checkplot as potential blends with the target
        object.

    makeneighborlcs : bool
        If True, will make light curve and phased light curve plots for all
        neighbors found in the object collection for each input object.

    fast_mode : bool or float
        This runs the external catalog operations in a "fast" mode, with short
        timeouts and not trying to hit external catalogs that take a long time
        to respond.

        If this is set to True, the default settings for the external requests
        will then become::

                skyview_lookup = False
                skyview_timeout = 10.0
                skyview_retry_failed = False
                dust_timeout = 10.0
                gaia_submit_timeout = 7.0
                gaia_max_timeout = 10.0
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

        If this is a float, will run in "fast" mode with the provided timeout
        value in seconds and the following settings::

                skyview_lookup = True
                skyview_timeout = fast_mode
                skyview_retry_failed = False
                dust_timeout = fast_mode
                gaia_submit_timeout = 0.66*fast_mode
                gaia_max_timeout = fast_mode
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

    gaia_max_timeout : float
        Sets the timeout in seconds to use when waiting for the GAIA service to
        respond to our request for the object's information. Note that if
        `fast_mode` is set, this is ignored.

    gaia_mirror : str or None
        This sets the GAIA mirror to use. This is a key in the
        `services.gaia.GAIA_URLS` dict which defines the URLs to hit for each
        mirror.

    xmatchinfo : str or dict
        This is either the xmatch dict produced by the function
        `load_xmatch_external_catalogs` above, or the path to the xmatch info
        pickle file produced by that function.

    xmatchradiusarcsec : float
        This is the cross-matching radius to use in arcseconds.

    minobservations : int
        The minimum of observations the input object's mag/flux time-series must
        have for this function to plot its light curve and phased light
        curve. If the object has less than this number, no light curves will be
        plotted, but the checkplotdict will still contain all of the other
        information.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols : list of str or None
        The timecol keys to use from the lcdict in generating this checkplot.

    magcols : list of str or None
        The magcol keys to use from the lcdict in generating this checkplot.

    errcols : list of str or None
        The errcol keys to use from the lcdict in generating this checkplot.

    skipdone : bool
        This indicates if this function will skip creating checkplots that
        already exist corresponding to the current `objectid` and `magcol`. If
        `skipdone` is set to True, this will be done.

    done_callback : Python function or None
        This is used to provide a function to execute after the checkplot
        pickles are generated. This is useful if you want to stream the results
        of checkplot making to some other process, e.g. directly running an
        ingestion into an LCC-Server collection. The function will always get
        the list of the generated checkplot pickles as its first arg, and all of
        the kwargs for runcp in the kwargs dict. Additional args and kwargs can
        be provided by giving a list in the `done_callbacks_args` kwarg and a
        dict in the `done_callbacks_kwargs` kwarg.

        NOTE: the function you pass in here should be pickleable by normal
        Python if you want to use it with the parallel_cp and parallel_cp_lcdir
        functions below.

    done_callback_args : tuple or None
        If not None, contains any args to pass into the `done_callback`
        function.

    done_callback_kwargs : dict or None
        If not None, contains any kwargs to pass into the `done_callback`
        function.

    maxobjects : int
        The maximum number of objects to process in this run.

    nworkers : int
        The number of parallel workers that will work on the checkplot
        generation process.

    Returns
    -------

    dict
        This returns a dict with keys = input period-finding pickles and vals =
        list of the corresponding checkplot pickles produced.


if isinstance(use_lcformat_dir, str):<EOL><INDENT>lcformat_jsonpath = os.path.join(<EOL>use_lcformat_dir,<EOL>'<STR_LIT>' % formatkey<EOL>)<EOL>if not os.path.exists(lcformat_jsonpath):<EOL><INDENT>lcformat_jsonpath = os.path.join(<EOL>os.path.expanduser('<STR_LIT>'),<EOL>'<STR_LIT>' % formatkey<EOL>)<EOL>if not os.path.exists(lcformat_jsonpath):<EOL><INDENT>install_path = os.path.dirname(__file__)<EOL>install_path = os.path.abspath(<EOL>os.path.join(install_path, '<STR_LIT:..>', '<STR_LIT:data>','<STR_LIT>')<EOL>)<EOL>lcformat_jsonpath = os.path.join(<EOL>install_path,<EOL>'<STR_LIT>' % formatkey<EOL>)<EOL>if not os.path.exists(lcformat_jsonpath):<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% formatkey)<EOL>return None<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>lcformat_jsonpath = os.path.join(<EOL>os.path.expanduser('<STR_LIT>'),<EOL>'<STR_LIT>' % formatkey<EOL>)<EOL>if not os.path.exists(lcformat_jsonpath):<EOL><INDENT>install_path = os.path.dirname(__file__)<EOL>install_path = os.path.abspath(<EOL>os.path.join(install_path, '<STR_LIT:..>', '<STR_LIT:data>','<STR_LIT>')<EOL>)<EOL>lcformat_jsonpath = os.path.join(<EOL>install_path,<EOL>'<STR_LIT>' % formatkey<EOL>)<EOL>if not os.path.exists(lcformat_jsonpath):<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% formatkey)<EOL>return None<EOL><DEDENT><DEDENT><DEDENT>with open(lcformat_jsonpath) as infd:<EOL><INDENT>lcformatdict = json.load(infd)<EOL><DEDENT>readerfunc_module = lcformatdict['<STR_LIT>']<EOL>readerfunc = lcformatdict['<STR_LIT>']<EOL>readerfunc_kwargs = lcformatdict['<STR_LIT>']<EOL>normfunc_module = lcformatdict['<STR_LIT>']<EOL>normfunc = lcformatdict['<STR_LIT>']<EOL>normfunc_kwargs = lcformatdict['<STR_LIT>']<EOL>fileglob = lcformatdict['<STR_LIT>']<EOL>timecols = lcformatdict['<STR_LIT>']<EOL>magcols = lcformatdict['<STR_LIT>']<EOL>errcols = lcformatdict['<STR_LIT>']<EOL>magsarefluxes = lcformatdict['<STR_LIT>']<EOL>readermodule = _check_extmodule(readerfunc_module, formatkey)<EOL>if not readermodule:<EOL><INDENT>LOGERROR("<STR_LIT>"<EOL>"<STR_LIT>" %<EOL>(readerfunc_module, formatkey))<EOL>return None<EOL><DEDENT>try:<EOL><INDENT>readerfunc_in = getattr(readermodule, readerfunc)<EOL><DEDENT>except AttributeError:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% (formatkey, readerfunc_module, readerfunc))<EOL>raise<EOL><DEDENT>if normfunc_module:<EOL><INDENT>normmodule = _check_extmodule(normfunc_module, formatkey)<EOL>if not normmodule:<EOL><INDENT>LOGERROR("<STR_LIT>"<EOL>"<STR_LIT>" %<EOL>(normfunc_module, formatkey))<EOL>return None<EOL><DEDENT><DEDENT>else:<EOL><INDENT>normmodule = None<EOL><DEDENT>if normfunc_module and normfunc:<EOL><INDENT>try:<EOL><INDENT>normfunc_in = getattr(normmodule, normfunc)<EOL><DEDENT>except AttributeError:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% (formatkey, normfunc_module, normfunc))<EOL>raise<EOL><DEDENT><DEDENT>else:<EOL><INDENT>normfunc_in = None<EOL><DEDENT>if isinstance(readerfunc_kwargs, dict):<EOL><INDENT>readerfunc_in = partial(readerfunc_in, **readerfunc_kwargs)<EOL><DEDENT>if normfunc_in is not None:<EOL><INDENT>if isinstance(normfunc_kwargs, dict):<EOL><INDENT>normfunc_in = partial(normfunc_in, **normfunc_kwargs)<EOL><DEDENT><DEDENT>returntuple = (<EOL>fileglob,<EOL>readerfunc_in,<EOL>timecols,<EOL>magcols,<EOL>errcols,<EOL>magsarefluxes,<EOL>normfunc_in,<EOL>)<EOL>return returntuple<EOL>

This loads an LC format description from a previously-saved JSON file.

    Parameters
    ----------

    formatkey : str
        The key used to refer to the LC format. This is part of the JSON file's
        name, e.g. the format key 'hat-csv' maps to the format JSON file:
        '<astrobase install path>/data/lcformats/hat-csv.json'.

    use_lcformat_dir : str or None
        If provided, must be the path to a directory that contains the
        corresponding lcformat JSON file for `formatkey`. If this is None, this
        function will look for lcformat JSON files corresponding to the given
        `formatkey`:

        - first, in the directory specified in this kwarg,
        - if not found there, in the home directory: ~/.astrobase/lcformat-jsons
        - if not found there, in: <astrobase install path>/data/lcformats

    Returns
    -------

    tuple
        A tuple of the following form is returned::

            (fileglob       : the file glob of the associated LC files,
             readerfunc_in  : the imported Python function for reading LCs,
             timecols       : list of time col keys to get from the lcdict,
             magcols        : list of mag col keys to get from the lcdict ,
             errcols        : list of err col keys to get from the lcdict,
             magsarefluxes  : True if the measurements are fluxes not mags,
             normfunc_in    : the imported Python function for normalizing LCs)

        All `astrobase.lcproc` functions can then use this tuple to dynamically
        import your LC reader and normalization functions to work with your LC
        format transparently.


<EOL>if not os.path.exists(outdir):<EOL><INDENT>os.makedirs(outdir)<EOL><DEDENT>if (liststartindex is not None) and (listmaxobjects is None):<EOL><INDENT>lclist = lclist[liststartindex:]<EOL><DEDENT>elif (liststartindex is None) and (listmaxobjects is not None):<EOL><INDENT>lclist = lclist[:listmaxobjects]<EOL><DEDENT>elif (liststartindex is not None) and (listmaxobjects is not None):<EOL><INDENT>lclist = lclist[liststartindex:liststartindex+listmaxobjects]<EOL><DEDENT>tasklist = [(x, outdir, timecols, magcols, errcols, lcformat, lcformatdir,<EOL>pfmethods, pfkwargs, getblssnr, sigclip, nperiodworkers,<EOL>minobservations,<EOL>excludeprocessed)<EOL>for x in lclist]<EOL>with ProcessPoolExecutor(max_workers=ncontrolworkers) as executor:<EOL><INDENT>resultfutures = executor.map(_runpf_worker, tasklist)<EOL><DEDENT>results = [x for x in resultfutures]<EOL>return results<EOL>

This drives the overall parallel period processing for a list of LCs.

    As a rough benchmark, 25000 HATNet light curves with up to 50000 points per
    LC take about 26 days in total for an invocation of this function using
    GLS+PDM+BLS, 10 periodworkers, and 4 controlworkers (so all 40 'cores') on a
    2 x Xeon E5-2660v3 machine.

    Parameters
    ----------

    lclist : list of str
        The list of light curve file to process.

    outdir : str
        The output directory where the period-finding result pickles will go.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    pfmethods : list of str
        This is a list of period finding methods to run. Each element is a
        string matching the keys of the `PFMETHODS` dict above. By default, this
        runs GLS, PDM, AoVMH, and the spectral window Lomb-Scargle periodogram.

    pfkwargs : list of dicts
        This is used to provide any special kwargs as dicts to each
        period-finding method function specified in `pfmethods`.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    getblssnr : bool
        If this is True and BLS is one of the methods specified in `pfmethods`,
        will also calculate the stats for each best period in the BLS results:
        transit depth, duration, ingress duration, refit period and epoch, and
        the SNR of the transit.

    nperiodworkers : int
        The number of parallel period-finding workers to launch per object task.

    ncontrolworkers : int
        The number of controlling processes to launch. This effectively sets how
        many objects from `lclist` will be processed in parallel.

    liststartindex : int or None
        This sets the index from where to start in `lclist`.

    listmaxobjects : int or None
        This sets the maximum number of objects in `lclist` to run
        period-finding for in this invocation. Together with `liststartindex`,
        `listmaxobjects` can be used to distribute processing over several
        independent machines if the number of light curves is very large.

    minobservations : int
        The minimum number of finite LC points required to process a light
        curve.

    excludeprocessed : bool
        If this is True, light curves that have existing period-finding result
        pickles in `outdir` will not be processed.

        FIXME: currently, this uses a dumb method of excluding already-processed
        files. A smarter way to do this is to (i) generate a SHA512 cachekey
        based on a repr of `{'lcfile', 'timecols', 'magcols', 'errcols',
        'lcformat', 'pfmethods', 'sigclip', 'getblssnr', 'pfkwargs'}`, (ii) make
        sure all list kwargs in the dict are sorted, (iii) check if the output
        file has the same cachekey in its filename (last 8 chars of cachekey
        should work), so the result was processed in exactly the same way as
        specifed in the input to this function, and can therefore be
        ignored. Will implement this later.

    Returns
    -------

    list of str
        A list of the period-finding pickles created for all of input LCs
        processed.


cpf, cpkwargs = task<EOL>try:<EOL><INDENT>newcpf = update_checkplot_objectinfo(cpf, **cpkwargs)<EOL>return newcpf<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' % cpf)<EOL>return None<EOL><DEDENT>

This is a parallel worker for `parallel_update_cp_objectinfo`.

    Parameters
    ----------

    task : tuple
        - task[0] = checkplot pickle file
        - task[1] = kwargs

    Returns
    -------

    str
        The name of the checkplot file that was updated. None if the update
        fails for some reason.


<EOL>if sys.platform == '<STR_LIT>':<EOL><INDENT>import requests<EOL>requests.get('<STR_LIT>')<EOL><DEDENT>if (liststartindex is not None) and (maxobjects is None):<EOL><INDENT>cplist = cplist[liststartindex:]<EOL><DEDENT>elif (liststartindex is None) and (maxobjects is not None):<EOL><INDENT>cplist = cplist[:maxobjects]<EOL><DEDENT>elif (liststartindex is not None) and (maxobjects is not None):<EOL><INDENT>cplist = (<EOL>cplist[liststartindex:liststartindex+maxobjects]<EOL>)<EOL><DEDENT>tasks = [(x, {'<STR_LIT>':fast_mode,<EOL>'<STR_LIT>':findercmap,<EOL>'<STR_LIT>':finderconvolve,<EOL>'<STR_LIT>':deredden_object,<EOL>'<STR_LIT>':custom_bandpasses,<EOL>'<STR_LIT>':gaia_submit_timeout,<EOL>'<STR_LIT>':gaia_submit_tries,<EOL>'<STR_LIT>':gaia_max_timeout,<EOL>'<STR_LIT>':gaia_mirror,<EOL>'<STR_LIT>':complete_query_later,<EOL>'<STR_LIT>':lclistpkl,<EOL>'<STR_LIT>':nbrradiusarcsec,<EOL>'<STR_LIT>':maxnumneighbors,<EOL>'<STR_LIT>':plotdpi,<EOL>'<STR_LIT>':findercachedir,<EOL>'<STR_LIT>':verbose}) for x in cplist]<EOL>resultfutures = []<EOL>results = []<EOL>with ProcessPoolExecutor(max_workers=nworkers) as executor:<EOL><INDENT>resultfutures = executor.map(cp_objectinfo_worker, tasks)<EOL><DEDENT>results = [x for x in resultfutures]<EOL>executor.shutdown()<EOL>return results<EOL>

This updates objectinfo for a list of checkplots.

Useful in cases where a previous round of GAIA/finderchart/external catalog
acquisition failed. This will preserve the following keys in the checkplots
if they exist:

comments
varinfo
objectinfo.objecttags

Parameters
----------

cplist : list of str
    A list of checkplot pickle file names to update.

liststartindex : int
    The index of the input list to start working at.

maxobjects : int
    The maximum number of objects to process in this run. Use this with
    `liststartindex` to effectively distribute working on a large list of
    input checkplot pickles over several sessions or machines.

nworkers : int
    The number of parallel workers that will work on the checkplot
    update process.

fast_mode : bool or float
    This runs the external catalog operations in a "fast" mode, with short
    timeouts and not trying to hit external catalogs that take a long time
    to respond. See the docstring for
    `checkplot.pkl_utils._pkl_finder_objectinfo` for details on how this
    works. If this is True, will run in "fast" mode with default timeouts (5
    seconds in most cases). If this is a float, will run in "fast" mode with
    the provided timeout value in seconds.

findercmap : str or matplotlib.cm.Colormap object

findercmap : str or matplotlib.cm.ColorMap object
    The Colormap object to use for the finder chart image.

finderconvolve : astropy.convolution.Kernel object or None
    If not None, the Kernel object to use for convolving the finder image.

deredden_objects : bool
    If this is True, will use the 2MASS DUST service to get extinction
    coefficients in various bands, and then try to deredden the magnitudes
    and colors of the object already present in the checkplot's objectinfo
    dict.

custom_bandpasses : dict
    This is a dict used to provide custom bandpass definitions for any
    magnitude measurements in the objectinfo dict that are not automatically
    recognized by the `varclass.starfeatures.color_features` function. See
    its docstring for details on the required format.

gaia_submit_timeout : float
    Sets the timeout in seconds to use when submitting a request to look up
    the object's information to the GAIA service. Note that if `fast_mode`
    is set, this is ignored.

gaia_submit_tries : int
    Sets the maximum number of times the GAIA services will be contacted to
    obtain this object's information. If `fast_mode` is set, this is
    ignored, and the services will be contacted only once (meaning that a
    failure to respond will be silently ignored and no GAIA data will be
    added to the checkplot's objectinfo dict).

gaia_max_timeout : float
    Sets the timeout in seconds to use when waiting for the GAIA service to
    respond to our request for the object's information. Note that if
    `fast_mode` is set, this is ignored.

gaia_mirror : str
    This sets the GAIA mirror to use. This is a key in the
    `services.gaia.GAIA_URLS` dict which defines the URLs to hit for each
    mirror.

complete_query_later : bool
    If this is True, saves the state of GAIA queries that are not yet
    complete when `gaia_max_timeout` is reached while waiting for the GAIA
    service to respond to our request. A later call for GAIA info on the
    same object will attempt to pick up the results from the existing query
    if it's completed. If `fast_mode` is True, this is ignored.

lclistpkl : dict or str
    If this is provided, must be a dict resulting from reading a catalog
    produced by the `lcproc.catalogs.make_lclist` function or a str path
    pointing to the pickle file produced by that function. This catalog is
    used to find neighbors of the current object in the current light curve
    collection. Looking at neighbors of the object within the radius
    specified by `nbrradiusarcsec` is useful for light curves produced by
    instruments that have a large pixel scale, so are susceptible to
    blending of variability and potential confusion of neighbor variability
    with that of the actual object being looked at. If this is None, no
    neighbor lookups will be performed.

nbrradiusarcsec : float
    The radius in arcseconds to use for a search conducted around the
    coordinates of this object to look for any potential confusion and
    blending of variability amplitude caused by their proximity.

maxnumneighbors : int
    The maximum number of neighbors that will have their light curves and
    magnitudes noted in this checkplot as potential blends with the target
    object.

plotdpi : int
    The resolution in DPI of the plots to generate in this function
    (e.g. the finder chart, etc.)

findercachedir : str
    The path to the astrobase cache directory for finder chart downloads
    from the NASA SkyView service.

verbose : bool
    If True, will indicate progress and warn about potential problems.

Returns
-------

list of str
    Paths to the updated checkplot pickle file.


<EOL>with open(cmdpkl,'<STR_LIT:rb>') as infd:<EOL><INDENT>cmd = pickle.load(infd)<EOL><DEDENT>for cpf in cplist:<EOL><INDENT>add_cmd_to_checkplot(cpf, cmd,<EOL>require_cmd_magcolor=require_cmd_magcolor,<EOL>save_cmd_pngs=save_cmd_pngs)<EOL><DEDENT>

This adds CMDs for each object in cplist.

    Parameters
    ----------

    cplist : list of str
        This is the input list of checkplot pickles to add the CMDs to.

    cmdpkl : str
        This is the filename of the CMD pickle created previously.

    require_cmd_magcolor : bool
        If this is True, a CMD plot will not be made if the color and mag keys
        required by the CMD are not present or are nan in each checkplot's
        objectinfo dict.

    save_cmd_pngs : bool
        If this is True, then will save the CMD plots that were generated and
        added back to the checkplotdict as PNGs to the same directory as
        `cpx`.

    Returns
    -------

    Nothing.


<EOL>if isinstance(cpx, str) and os.path.exists(cpx):<EOL><INDENT>cpdict = _read_checkplot_picklefile(cpx)<EOL><DEDENT>elif isinstance(cpx, dict):<EOL><INDENT>cpdict = cpx<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>')<EOL>return None<EOL><DEDENT>if isinstance(cmdpkl, str) and os.path.exists(cmdpkl):<EOL><INDENT>with open(cmdpkl, '<STR_LIT:rb>') as infd:<EOL><INDENT>cmd = pickle.load(infd)<EOL><DEDENT><DEDENT>elif isinstance(cmdpkl, dict):<EOL><INDENT>cmd = cmdpkl<EOL><DEDENT>cpdict['<STR_LIT>'] = {}<EOL>cplist_mags = cmd['<STR_LIT>']<EOL>cplist_colors = cmd['<STR_LIT>']<EOL>for c1, c2, ym, ind in zip(cmd['<STR_LIT>'],<EOL>cmd['<STR_LIT>'],<EOL>cmd['<STR_LIT>'],<EOL>range(len(cmd['<STR_LIT>']))):<EOL><INDENT>if (c1 in cpdict['<STR_LIT>'] and<EOL>cpdict['<STR_LIT>'][c1] is not None):<EOL><INDENT>c1mag = cpdict['<STR_LIT>'][c1]<EOL><DEDENT>else:<EOL><INDENT>c1mag = np.nan<EOL><DEDENT>if (c2 in cpdict['<STR_LIT>'] and<EOL>cpdict['<STR_LIT>'][c2] is not None):<EOL><INDENT>c2mag = cpdict['<STR_LIT>'][c2]<EOL><DEDENT>else:<EOL><INDENT>c2mag = np.nan<EOL><DEDENT>if (ym in cpdict['<STR_LIT>'] and<EOL>cpdict['<STR_LIT>'][ym] is not None):<EOL><INDENT>ymmag = cpdict['<STR_LIT>'][ym]<EOL><DEDENT>else:<EOL><INDENT>ymmag = np.nan<EOL><DEDENT>if (require_cmd_magcolor and<EOL>not (np.isfinite(c1mag) and<EOL>np.isfinite(c2mag) and<EOL>np.isfinite(ymmag))):<EOL><INDENT>LOGWARNING("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" %<EOL>(c1, c2, ym, cpdict['<STR_LIT>']))<EOL>continue<EOL><DEDENT>try:<EOL><INDENT>thiscmd_title = r'<STR_LIT>' % (CMD_LABELS[c1],<EOL>CMD_LABELS[c2],<EOL>CMD_LABELS[ym])<EOL>fig = plt.figure(figsize=(<NUM_LIT:10>,<NUM_LIT:8>))<EOL>plt.plot(cplist_colors[:,ind],<EOL>cplist_mags[:,ind],<EOL>rasterized=True,<EOL>marker='<STR_LIT:o>',<EOL>linestyle='<STR_LIT:none>',<EOL>mew=<NUM_LIT:0>,<EOL>ms=<NUM_LIT:3>)<EOL>plt.plot([c1mag - c2mag], [ymmag],<EOL>ms=<NUM_LIT:20>,<EOL>color='<STR_LIT>',<EOL>marker='<STR_LIT:*>',<EOL>mew=<NUM_LIT:0>)<EOL>plt.xlabel(r'<STR_LIT>' % (CMD_LABELS[c1], CMD_LABELS[c2]))<EOL>plt.ylabel(r'<STR_LIT>' % CMD_LABELS[ym])<EOL>plt.title('<STR_LIT>' % (cpdict['<STR_LIT>'], thiscmd_title))<EOL>plt.gca().invert_yaxis()<EOL>cmdpng = StrIO()<EOL>plt.savefig(cmdpng, bbox_inches='<STR_LIT>',<EOL>pad_inches=<NUM_LIT:0.0>, format='<STR_LIT>')<EOL>cmdpng.seek(<NUM_LIT:0>)<EOL>cmdb64 = base64.b64encode(cmdpng.read())<EOL>cmdpng.close()<EOL>plt.close('<STR_LIT:all>')<EOL>plt.gcf().clear()<EOL>cpdict['<STR_LIT>']['<STR_LIT>' % (c1,c2,ym)] = cmdb64<EOL>if save_cmd_pngs:<EOL><INDENT>if isinstance(cpx, str):<EOL><INDENT>outpng = os.path.join(os.path.dirname(cpx),<EOL>'<STR_LIT>' %<EOL>(cpdict['<STR_LIT>'],<EOL>c1,c2,ym))<EOL><DEDENT>else:<EOL><INDENT>outpng = '<STR_LIT>' % (cpdict['<STR_LIT>'],<EOL>c1,c2,ym)<EOL><DEDENT>_base64_to_file(cmdb64, outpng)<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' %<EOL>(c1, c2, ym, cmdpkl))<EOL>continue<EOL><DEDENT><DEDENT>if isinstance(cpx, str):<EOL><INDENT>cpf = _write_checkplot_picklefile(cpdict, outfile=cpx, protocol=<NUM_LIT:4>)<EOL>return cpf<EOL><DEDENT>elif isinstance(cpx, dict):<EOL><INDENT>return cpdict<EOL><DEDENT>

This adds CMD figures to a checkplot dict or pickle.

    Looks up the CMDs in `cmdpkl`, adds the object from `cpx` as a gold(-ish)
    star in the plot, and then saves the figure to a base64 encoded PNG, which
    can then be read and used by the `checkplotserver`.

    Parameters
    ----------

    cpx : str or dict
        This is the input checkplot pickle or dict to add the CMD to.

    cmdpkl : str or dict
        The CMD pickle generated by the `colormagdiagram_cplist` or
        `colormagdiagram_cpdir` functions above, or the dict produced by reading
        this pickle in.

    require_cmd_magcolor : bool
        If this is True, a CMD plot will not be made if the color and mag keys
        required by the CMD are not present or are nan in this checkplot's
        objectinfo dict.

    save_cmd_png : bool
        If this is True, then will save the CMD plots that were generated and
        added back to the checkplotdict as PNGs to the same directory as
        `cpx`. If `cpx` is a dict, will save them to the current working
        directory.

    Returns
    -------

    str or dict
        If `cpx` was a str filename of checkplot pickle, this will return that
        filename to indicate that the CMD was added to the file. If `cpx` was a
        checkplotdict, this will return the checkplotdict with a new key called
        'colormagdiagram' containing the base64 encoded PNG binary streams of
        all CMDs generated.


raise KeyboardInterrupt<EOL>

This raises a KeyboardInterrupt when a SIGKILL comes in.

    This is a handle for use with the Python `signal.signal` function.


if not sqs_client:<EOL><INDENT>sqs_client = boto3.client('<STR_LIT>')<EOL><DEDENT>if not s3_client:<EOL><INDENT>s3_client = boto3.client('<STR_LIT>')<EOL><DEDENT>lclist_pklf = lclist_pkl_s3url.split('<STR_LIT:/>')[-<NUM_LIT:1>]<EOL>if not os.path.exists(lclist_pklf):<EOL><INDENT>lclist_pklf = awsutils.s3_get_url(<EOL>lclist_pkl_s3url,<EOL>client=s3_client<EOL>)<EOL><DEDENT>with open(lclist_pklf,'<STR_LIT:rb>') as infd:<EOL><INDENT>lclistpkl = pickle.load(infd)<EOL><DEDENT>signal.signal(signal.SIGINT, kill_handler)<EOL>signal.signal(signal.SIGTERM, kill_handler)<EOL>shutdown_last_time = time.monotonic()<EOL>diskspace_last_time = time.monotonic()<EOL>while True:<EOL><INDENT>curr_time = time.monotonic()<EOL>if (curr_time - shutdown_last_time) > shutdown_check_timer_seconds:<EOL><INDENT>shutdown_check = shutdown_check_handler()<EOL>if shutdown_check:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>shutdown_last_time = time.monotonic()<EOL><DEDENT>if (curr_time - diskspace_last_time) > cache_clean_timer_seconds:<EOL><INDENT>cache_clean_handler()<EOL>diskspace_last_time = time.monotonic()<EOL><DEDENT>try:<EOL><INDENT>work = awsutils.sqs_get_item(in_queue_url,<EOL>client=sqs_client,<EOL>raiseonfail=True)<EOL>if work is not None and len(work) > <NUM_LIT:0>:<EOL><INDENT>recv = work[<NUM_LIT:0>]<EOL>action = recv['<STR_LIT>']['<STR_LIT:action>']<EOL>if action != '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>target = recv['<STR_LIT>']['<STR_LIT:target>']<EOL>args = recv['<STR_LIT>']['<STR_LIT:args>']<EOL>kwargs = recv['<STR_LIT>']['<STR_LIT>']<EOL>outbucket = recv['<STR_LIT>']['<STR_LIT>']<EOL>if '<STR_LIT>' in recv['<STR_LIT>']:<EOL><INDENT>out_queue_url = recv['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>out_queue_url = None<EOL><DEDENT>receipt = recv['<STR_LIT>']<EOL>try:<EOL><INDENT>lc_filename = awsutils.s3_get_url(<EOL>target,<EOL>altexts=lc_altexts,<EOL>client=s3_client,<EOL>)<EOL>if len(args) > <NUM_LIT:0> and args[<NUM_LIT:0>] is not None:<EOL><INDENT>pf_pickle = awsutils.s3_get_url(<EOL>args[<NUM_LIT:0>],<EOL>client=s3_client<EOL>)<EOL><DEDENT>else:<EOL><INDENT>pf_pickle = None<EOL><DEDENT>cpfs = runcp(<EOL>pf_pickle,<EOL>workdir,<EOL>workdir,<EOL>lcfname=lc_filename,<EOL>lclistpkl=lclistpkl,<EOL>makeneighborlcs=False,<EOL>**kwargs<EOL>)<EOL>if cpfs and all(os.path.exists(x) for x in cpfs):<EOL><INDENT>LOGINFO('<STR_LIT>' %<EOL>(lc_filename, pf_pickle, cpfs))<EOL>resp = s3_client.list_objects_v2(<EOL>Bucket=outbucket,<EOL>MaxKeys=<NUM_LIT:1>,<EOL>Prefix=cpfs[<NUM_LIT:0>]<EOL>)<EOL>outbucket_list = resp.get('<STR_LIT>',[])<EOL>if outbucket_list and len(outbucket_list) > <NUM_LIT:0>:<EOL><INDENT>LOGWARNING(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% target<EOL>)<EOL>awsutils.sqs_delete_item(in_queue_url, receipt)<EOL>continue<EOL><DEDENT>for cpf in cpfs:<EOL><INDENT>put_url = awsutils.s3_put_file(cpf,<EOL>outbucket,<EOL>client=s3_client)<EOL>if put_url is not None:<EOL><INDENT>LOGINFO('<STR_LIT>' % put_url)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT:target>': target,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':lclist_pklf,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>os.remove(cpf)<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>' % cpf)<EOL><DEDENT><DEDENT>awsutils.sqs_delete_item(in_queue_url,<EOL>receipt)<EOL>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>' %<EOL>(lc_filename, pf_pickle))<EOL>with open('<STR_LIT>' %<EOL>lc_filename, '<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(<EOL>{'<STR_LIT>':in_queue_url,<EOL>'<STR_LIT:target>':target,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':lclist_pklf,<EOL>'<STR_LIT>':kwargs,<EOL>'<STR_LIT>':outbucket,<EOL>'<STR_LIT>':out_queue_url},<EOL>outfd, pickle.HIGHEST_PROTOCOL<EOL>)<EOL><DEDENT>put_url = awsutils.s3_put_file(<EOL>'<STR_LIT>' % lc_filename,<EOL>outbucket,<EOL>client=s3_client<EOL>)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':lclist_pklf,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>awsutils.sqs_delete_item(in_queue_url,<EOL>receipt,<EOL>raiseonfail=True)<EOL>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT><DEDENT>except ClientError as e:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>')<EOL>if '<STR_LIT>' in locals():<EOL><INDENT>with open('<STR_LIT>' %<EOL>lc_filename,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(<EOL>{'<STR_LIT>':in_queue_url,<EOL>'<STR_LIT:target>':target,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':lclist_pklf,<EOL>'<STR_LIT>':kwargs,<EOL>'<STR_LIT>':outbucket,<EOL>'<STR_LIT>':out_queue_url},<EOL>outfd, pickle.HIGHEST_PROTOCOL<EOL>)<EOL><DEDENT>put_url = awsutils.s3_put_file(<EOL>'<STR_LIT>' % lc_filename,<EOL>outbucket,<EOL>client=s3_client<EOL>)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':lclist_pklf,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT>awsutils.sqs_delete_item(in_queue_url,<EOL>receipt,<EOL>raiseonfail=True)<EOL><DEDENT><DEDENT><DEDENT>except KeyboardInterrupt:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>except ClientError as e:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>')<EOL>if '<STR_LIT>' in locals():<EOL><INDENT>with open('<STR_LIT>' %<EOL>lc_filename,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(<EOL>{'<STR_LIT>':in_queue_url,<EOL>'<STR_LIT:target>':target,<EOL>'<STR_LIT>':lclist_pklf,<EOL>'<STR_LIT>':kwargs,<EOL>'<STR_LIT>':outbucket,<EOL>'<STR_LIT>':out_queue_url},<EOL>outfd, pickle.HIGHEST_PROTOCOL<EOL>)<EOL><DEDENT>put_url = awsutils.s3_put_file(<EOL>'<STR_LIT>' % lc_filename,<EOL>outbucket,<EOL>client=s3_client<EOL>)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT>':lclist_pklf,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT>awsutils.sqs_delete_item(in_queue_url, receipt, raiseonfail=True)<EOL><DEDENT><DEDENT>

This runs checkplot pickle making in a loop until interrupted.

    Consumes work task items from an input queue set up by `runcp_producer_loop`
    above. For the moment, we don't generate neighbor light curves since this
    would require a lot more S3 calls.

    Parameters
    ----------

    in_queue_url : str
        The SQS URL of the input queue to listen to for work assignment
        messages. The task orders will include the input and output S3 bucket
        names, as well as the URL of the output queue to where this function
        will report its work-complete or work-failed status.

    workdir : str
        The directory on the local machine where this worker loop will download
        the input light curves and associated period-finder results (if any),
        process them, and produce its output checkplot pickles. These will then
        be uploaded to the specified S3 output bucket and then deleted from the
        workdir when the upload is confirmed to make it safely to S3.

    lclist_pkl : str
        S3 URL of a catalog pickle generated by `lcproc.catalogs.make_lclist`
        that contains objectids and coordinates, as well as a kdtree for all of
        the objects in the current light curve collection being processed. This
        is used to look up neighbors for each object being processed.

    lc_altexts : sequence of str
        If not None, this is a sequence of alternate extensions to try for the
        input light curve file other than the one provided in the input task
        order. For example, to get anything that's an .sqlite where .sqlite.gz
        is expected, use altexts=[''] to strip the .gz.

    wait_time_seconds : int
        The amount of time to wait in the input SQS queue for an input task
        order. If this timeout expires and no task has been received, this
        function goes back to the top of the work loop.

    cache_clean_timer_seconds : float
        The amount of time in seconds to wait before periodically removing old
        files (such as finder chart FITS, external service result pickles) from
        the astrobase cache directory. These accumulate as the work items are
        processed, and take up significant space, so must be removed
        periodically.

    shutdown_check_timer_seconds : float
        The amount of time to wait before checking for a pending EC2 shutdown
        message for the instance this worker loop is operating on. If a shutdown
        is noticed, the worker loop is cancelled in preparation for instance
        shutdown.

    sqs_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its SQS operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    s3_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its S3 operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    Returns
    -------

    Nothing.


if not sqs_client:<EOL><INDENT>sqs_client = boto3.client('<STR_LIT>')<EOL><DEDENT>if not s3_client:<EOL><INDENT>s3_client = boto3.client('<STR_LIT>')<EOL><DEDENT>if isinstance(lightcurve_list, str) and os.path.exists(lightcurve_list):<EOL><INDENT>with open(lightcurve_list, '<STR_LIT:r>') as infd:<EOL><INDENT>lclist = infd.readlines()<EOL><DEDENT>lclist = [x.replace('<STR_LIT:\n>','<STR_LIT>') for x in lclist if len(x) > <NUM_LIT:0>]<EOL>if process_list_slice is not None:<EOL><INDENT>lclist = lclist[process_list_slice[<NUM_LIT:0>]:process_list_slice[<NUM_LIT:1>]]<EOL><DEDENT>lclist = [x[<NUM_LIT:1>:] for x in lclist if x.startswith('<STR_LIT:/>')]<EOL>lclist = ['<STR_LIT>' % (input_bucket, x) for x in lclist]<EOL><DEDENT>elif isinstance(lightcurve_list, list):<EOL><INDENT>lclist = lightcurve_list<EOL><DEDENT>try:<EOL><INDENT>inq = sqs_client.get_queue_url(QueueName=input_queue)<EOL>inq_url = inq['<STR_LIT>']<EOL>LOGINFO('<STR_LIT>')<EOL><DEDENT>except ClientError as e:<EOL><INDENT>inq = awsutils.sqs_create_queue(input_queue, client=sqs_client)<EOL>inq_url = inq['<STR_LIT:url>']<EOL><DEDENT>try:<EOL><INDENT>outq = sqs_client.get_queue_url(QueueName=result_queue)<EOL>outq_url = outq['<STR_LIT>']<EOL>LOGINFO('<STR_LIT>')<EOL><DEDENT>except ClientError as e:<EOL><INDENT>outq = awsutils.sqs_create_queue(result_queue, client=sqs_client)<EOL>outq_url = outq['<STR_LIT:url>']<EOL><DEDENT>LOGINFO('<STR_LIT>' % inq_url)<EOL>LOGINFO('<STR_LIT>' % outq_url)<EOL>LOGINFO('<STR_LIT>')<EOL>time.sleep(<NUM_LIT>)<EOL>if pfresult_list is None:<EOL><INDENT>pfresult_list = [None for x in lclist]<EOL><DEDENT>for lc, pf in zip(lclist, pfresult_list):<EOL><INDENT>this_item = {<EOL>'<STR_LIT:target>': lc,<EOL>'<STR_LIT:action>': '<STR_LIT>',<EOL>'<STR_LIT:args>': (pf,),<EOL>'<STR_LIT>':runcp_kwargs if runcp_kwargs is not None else {},<EOL>'<STR_LIT>': result_bucket,<EOL>'<STR_LIT>': outq_url<EOL>}<EOL>resp = awsutils.sqs_put_item(inq_url, this_item, client=sqs_client)<EOL>if resp:<EOL><INDENT>LOGINFO('<STR_LIT>' % (lc,inq_url))<EOL><DEDENT><DEDENT>done_objects = {}<EOL>LOGINFO('<STR_LIT>')<EOL>signal.signal(signal.SIGINT, kill_handler)<EOL>signal.signal(signal.SIGTERM, kill_handler)<EOL>while len(list(done_objects.keys())) < len(lclist):<EOL><INDENT>try:<EOL><INDENT>result = awsutils.sqs_get_item(outq_url, client=sqs_client)<EOL>if result is not None and len(result) > <NUM_LIT:0>:<EOL><INDENT>recv = result[<NUM_LIT:0>]<EOL>try:<EOL><INDENT>processed_object = recv['<STR_LIT>']['<STR_LIT:target>']<EOL><DEDENT>except KeyError:<EOL><INDENT>LOGWARNING('<STR_LIT>' % recv)<EOL>processed_object = '<STR_LIT>'<EOL><DEDENT>cpf = recv['<STR_LIT>']['<STR_LIT>']<EOL>receipt = recv['<STR_LIT>']<EOL>if processed_object in lclist:<EOL><INDENT>if processed_object not in done_objects:<EOL><INDENT>done_objects[processed_object] = [cpf]<EOL><DEDENT>else:<EOL><INDENT>done_objects[processed_object].append(cpf)<EOL><DEDENT>LOGINFO('<STR_LIT>' % (processed_object, cpf))<EOL>if download_when_done:<EOL><INDENT>getobj = awsutils.awsutils.s3_get_url(<EOL>cpf,<EOL>client=s3_client<EOL>)<EOL>LOGINFO('<STR_LIT>' % (cpf, getobj))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>awsutils.sqs_delete_item(outq_url, receipt)<EOL><DEDENT><DEDENT>except KeyboardInterrupt as e:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT><DEDENT>LOGINFO('<STR_LIT>')<EOL>time.sleep(<NUM_LIT:1.0>)<EOL>if purge_queues_when_done:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>sqs_client.purge_queue(QueueUrl=inq_url)<EOL>sqs_client.purge_queue(QueueUrl=outq_url)<EOL>time.sleep(<NUM_LIT>)<EOL><DEDENT>if delete_queues_when_done:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>awsutils.sqs_delete_queue(inq_url)<EOL>awsutils.sqs_delete_queue(outq_url)<EOL><DEDENT>work_state = {<EOL>'<STR_LIT>': done_objects,<EOL>'<STR_LIT>': list(set(lclist) - set(done_objects.keys())),<EOL>'<STR_LIT:args>':((os.path.abspath(lightcurve_list) if<EOL>isinstance(lightcurve_list, str) else lightcurve_list),<EOL>input_queue,<EOL>input_bucket,<EOL>result_queue,<EOL>result_bucket),<EOL>'<STR_LIT>':{'<STR_LIT>':pfresult_list,<EOL>'<STR_LIT>':runcp_kwargs,<EOL>'<STR_LIT>':process_list_slice,<EOL>'<STR_LIT>':download_when_done,<EOL>'<STR_LIT>':purge_queues_when_done,<EOL>'<STR_LIT>':save_state_when_done,<EOL>'<STR_LIT>':delete_queues_when_done}<EOL>}<EOL>if save_state_when_done:<EOL><INDENT>with open('<STR_LIT>','<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(work_state, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT><DEDENT>return work_state<EOL>

This sends checkplot making tasks to the input queue and monitors the
    result queue for task completion.

    Parameters
    ----------

    lightcurve_list : str or list of str
        This is either a string pointing to a file containing a list of light
        curves filenames to process or the list itself. The names must
        correspond to the full filenames of files stored on S3, including all
        prefixes, but not include the 's3://<bucket name>/' bit (these will be
        added automatically).

    input_queue : str
        This is the name of the SQS queue which will receive processing tasks
        generated by this function. The queue URL will automatically be obtained
        from AWS.

    input_bucket : str
        The name of the S3 bucket containing the light curve files to process.

    result_queue : str
        This is the name of the SQS queue that this function will listen to for
        messages from the workers as they complete processing on their input
        elements. This function will attempt to match input sent to the
        `input_queue` with results coming into the `result_queue` so it knows
        how many objects have been successfully processed. If this function
        receives task results that aren't in its own input queue, it will
        acknowledge them so they complete successfully, but not download them
        automatically. This handles leftover tasks completing from a previous
        run of this function.

    result_bucket : str
        The name of the S3 bucket which will receive the results from the
        workers.

    pfresult_list : list of str or None
        This is a list of periodfinder result pickle S3 URLs associated with
        each light curve. If provided, this will be used to add in phased light
        curve plots to each checkplot pickle. If this is None, the worker loop
        will produce checkplot pickles that only contain object information,
        neighbor information, and unphased light curves.

    runcp_kwargs : dict
        This is a dict used to pass any extra keyword arguments to the
        `lcproc.checkplotgen.runcp` function that will be run by the worker
        loop.

    process_list_slice : list
        This is used to index into the input light curve list so a subset of the
        full list can be processed in this specific run of this function.

        Use None for a slice index elem to emulate single slice spec behavior:

        process_list_slice = [10, None]  -> lightcurve_list[10:]
        process_list_slice = [None, 500] -> lightcurve_list[:500]

    purge_queues_when_done : bool
        If this is True, and this function exits (either when all done, or when
        it is interrupted with a Ctrl+C), all outstanding elements in the
        input/output queues that have not yet been acknowledged by workers or by
        this function will be purged. This effectively cancels all outstanding
        work.

    delete_queues_when_done : bool
        If this is True, and this function exits (either when all done, or when
        it is interrupted with a Ctrl+C'), all outstanding work items will be
        purged from the input/queues and the queues themselves will be deleted.

    download_when_done : bool
        If this is True, the generated checkplot pickle for each input work item
        will be downloaded immediately to the current working directory when the
        worker functions report they're done with it.

    save_state_when_done : bool
        If this is True, will save the current state of the work item queue and
        the work items acknowledged as completed to a pickle in the current
        working directory. Call the `runcp_producer_loop_savedstate` function
        below to resume processing from this saved state later.

    s3_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its S3 download operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    sqs_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its SQS operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    Returns
    -------

    dict or str
        Returns the current work state as a dict or str path to the generated
        work state pickle depending on if `save_state_when_done` is True.


if not sqs_client:<EOL><INDENT>sqs_client = boto3.client('<STR_LIT>')<EOL><DEDENT>if not s3_client:<EOL><INDENT>s3_client = boto3.client('<STR_LIT>')<EOL><DEDENT>signal.signal(signal.SIGINT, kill_handler)<EOL>signal.signal(signal.SIGTERM, kill_handler)<EOL>shutdown_last_time = time.monotonic()<EOL>while True:<EOL><INDENT>curr_time = time.monotonic()<EOL>if (curr_time - shutdown_last_time) > shutdown_check_timer_seconds:<EOL><INDENT>shutdown_check = shutdown_check_handler()<EOL>if shutdown_check:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>shutdown_last_time = time.monotonic()<EOL><DEDENT>try:<EOL><INDENT>work = awsutils.sqs_get_item(in_queue_url,<EOL>client=sqs_client,<EOL>raiseonfail=True)<EOL>if work is not None and len(work) > <NUM_LIT:0>:<EOL><INDENT>recv = work[<NUM_LIT:0>]<EOL>action = recv['<STR_LIT>']['<STR_LIT:action>']<EOL>if action != '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>target = recv['<STR_LIT>']['<STR_LIT:target>']<EOL>args = recv['<STR_LIT>']['<STR_LIT:args>']<EOL>kwargs = recv['<STR_LIT>']['<STR_LIT>']<EOL>outbucket = recv['<STR_LIT>']['<STR_LIT>']<EOL>if '<STR_LIT>' in recv['<STR_LIT>']:<EOL><INDENT>out_queue_url = recv['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>out_queue_url = None<EOL><DEDENT>receipt = recv['<STR_LIT>']<EOL>try:<EOL><INDENT>lc_filename = awsutils.s3_get_url(<EOL>target,<EOL>altexts=lc_altexts,<EOL>client=s3_client<EOL>)<EOL>runpf_args = (lc_filename, args[<NUM_LIT:0>])<EOL>pfresult = runpf(<EOL>*runpf_args,<EOL>**kwargs<EOL>)<EOL>if pfresult and os.path.exists(pfresult):<EOL><INDENT>LOGINFO('<STR_LIT>' %<EOL>(lc_filename, pfresult))<EOL>resp = s3_client.list_objects_v2(<EOL>Bucket=outbucket,<EOL>MaxKeys=<NUM_LIT:1>,<EOL>Prefix=pfresult<EOL>)<EOL>outbucket_list = resp.get('<STR_LIT>',[])<EOL>if outbucket_list and len(outbucket_list) > <NUM_LIT:0>:<EOL><INDENT>LOGWARNING(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% target<EOL>)<EOL>awsutils.sqs_delete_item(in_queue_url, receipt)<EOL>continue<EOL><DEDENT>put_url = awsutils.s3_put_file(pfresult,<EOL>outbucket,<EOL>client=s3_client)<EOL>if put_url is not None:<EOL><INDENT>LOGINFO('<STR_LIT>' % put_url)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT:target>': target,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>os.remove(pfresult)<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>' % pfresult)<EOL>os.remove(pfresult)<EOL><DEDENT>awsutils.sqs_delete_item(in_queue_url, receipt)<EOL>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>' %<EOL>(lc_filename,))<EOL>with open('<STR_LIT>' %<EOL>lc_filename, '<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(<EOL>{'<STR_LIT>':in_queue_url,<EOL>'<STR_LIT:target>':target,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':kwargs,<EOL>'<STR_LIT>':outbucket,<EOL>'<STR_LIT>':out_queue_url},<EOL>outfd, pickle.HIGHEST_PROTOCOL<EOL>)<EOL><DEDENT>put_url = awsutils.s3_put_file(<EOL>'<STR_LIT>' % lc_filename,<EOL>outbucket,<EOL>client=s3_client<EOL>)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>awsutils.sqs_delete_item(in_queue_url,<EOL>receipt,<EOL>raiseonfail=True)<EOL>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT><DEDENT>except ClientError as e:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>')<EOL>if '<STR_LIT>' in locals():<EOL><INDENT>with open('<STR_LIT>' %<EOL>lc_filename,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(<EOL>{'<STR_LIT>':in_queue_url,<EOL>'<STR_LIT:target>':target,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':kwargs,<EOL>'<STR_LIT>':outbucket,<EOL>'<STR_LIT>':out_queue_url},<EOL>outfd, pickle.HIGHEST_PROTOCOL<EOL>)<EOL><DEDENT>put_url = awsutils.s3_put_file(<EOL>'<STR_LIT>' % lc_filename,<EOL>outbucket,<EOL>client=s3_client<EOL>)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT>awsutils.sqs_delete_item(in_queue_url,<EOL>receipt,<EOL>raiseonfail=True)<EOL><DEDENT><DEDENT><DEDENT>except KeyboardInterrupt:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>except ClientError as e:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>')<EOL>if '<STR_LIT>' in locals():<EOL><INDENT>with open('<STR_LIT>' %<EOL>lc_filename,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(<EOL>{'<STR_LIT>':in_queue_url,<EOL>'<STR_LIT:target>':target,<EOL>'<STR_LIT>':kwargs,<EOL>'<STR_LIT>':outbucket,<EOL>'<STR_LIT>':out_queue_url},<EOL>outfd, pickle.HIGHEST_PROTOCOL<EOL>)<EOL><DEDENT>put_url = awsutils.s3_put_file(<EOL>'<STR_LIT>' % lc_filename,<EOL>outbucket,<EOL>client=s3_client<EOL>)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT>awsutils.sqs_delete_item(in_queue_url, receipt, raiseonfail=True)<EOL><DEDENT><DEDENT>

This runs period-finding in a loop until interrupted.

    Consumes work task items from an input queue set up by `runpf_producer_loop`
    above.

    Parameters
    ----------

    in_queue_url : str
        The SQS URL of the input queue to listen to for work assignment
        messages. The task orders will include the input and output S3 bucket
        names, as well as the URL of the output queue to where this function
        will report its work-complete or work-failed status.

    workdir : str
        The directory on the local machine where this worker loop will download
        the input light curves, process them, and produce its output
        periodfinding result pickles. These will then be uploaded to the
        specified S3 output bucket, and then deleted from the local disk.

    lc_altexts : sequence of str
        If not None, this is a sequence of alternate extensions to try for the
        input light curve file other than the one provided in the input task
        order. For example, to get anything that's an .sqlite where .sqlite.gz
        is expected, use altexts=[''] to strip the .gz.

    wait_time_seconds : int
        The amount of time to wait in the input SQS queue for an input task
        order. If this timeout expires and no task has been received, this
        function goes back to the top of the work loop.

    shutdown_check_timer_seconds : float
        The amount of time to wait before checking for a pending EC2 shutdown
        message for the instance this worker loop is operating on. If a shutdown
        is noticed, the worker loop is cancelled in preparation for instance
        shutdown.

    sqs_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its SQS operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    s3_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its S3 operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    Returns
    -------

    Nothing.


try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(fileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>lclist = sorted(glob.glob(os.path.join(lcdir, fileglob)))<EOL>return parallel_timebin(lclist,<EOL>binsizesec,<EOL>maxobjects=maxobjects,<EOL>outdir=outdir,<EOL>lcformat=lcformat,<EOL>timecols=timecols,<EOL>magcols=magcols,<EOL>errcols=errcols,<EOL>minbinelems=minbinelems,<EOL>nworkers=nworkers,<EOL>maxworkertasks=maxworkertasks)<EOL>

This time bins all the light curves in the specified directory.

Parameters
----------

lcdir : list of str
    Directory containing the input LCs to process.

binsizesec : float
    The time bin size to use in seconds.

maxobjects : int or None
    If provided, LC processing will stop at `lclist[maxobjects]`.

outdir : str or None
    The directory where output LCs will be written. If None, will write to
    the same directory as the input LCs.

lcformat : str
    This is the `formatkey` associated with your light curve format, which
    you previously passed in to the `lcproc.register_lcformat`
    function. This will be used to look up how to find and read the light
    curve file.

lcformatdir : str or None
    If this is provided, gives the path to a directory when you've stored
    your lcformat description JSONs, other than the usual directories lcproc
    knows to search for them in. Use this along with `lcformat` to specify
    an LC format JSON file that's not currently registered with lcproc.

timecols,magcols,errcols : lists of str
    The keys in the lcdict produced by your light curve reader function that
    correspond to the times, mags/fluxes, and associated measurement errors
    that will be used as inputs to the binning process. If these are None,
    the default values for `timecols`, `magcols`, and `errcols` for your
    light curve format will be used here.

minbinelems : int
    The minimum number of time-bin elements required to accept a time-bin as
    valid for the output binned light curve.

nworkers : int
    Number of parallel workers to launch.

maxworkertasks : int
    The maximum number of tasks a parallel worker will complete before being
    replaced to guard against memory leaks.

Returns
-------

dict
    The returned dict contains keys = input LCs, vals = output LCs.


if maxobjects:<EOL><INDENT>lclist = lclist[:maxobjects]<EOL><DEDENT>tasks = [(x, outdir, timecols, magcols, errcols,<EOL>mindet, lcformat, lcformatdir)<EOL>for x in lclist]<EOL>for task in tqdm(tasks):<EOL><INDENT>result = _varfeatures_worker(task)<EOL><DEDENT>return result<EOL>

This runs variability feature extraction for a list of LCs.

    Parameters
    ----------

    lclist : list of str
        The list of light curve file names to process.

    outdir : str
        The directory where the output varfeatures pickle files will be written.

    maxobjects : int
        The number of LCs to process from `lclist`.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    mindet : int
        The minimum number of LC points required to generate variability
        features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    Returns
    -------

    list of str
        List of the generated variability features pickles for the input LCs,
        with results for each magcol in the input `magcol` or light curve
        format's default `magcol` list.


try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if isinstance(templateinfo,str) and os.path.exists(templateinfo):<EOL><INDENT>with open(templateinfo,'<STR_LIT:rb>') as infd:<EOL><INDENT>templateinfo = pickle.load(infd)<EOL><DEDENT><DEDENT>lcdict = readerfunc(lcfile)<EOL>if ((isinstance(lcdict, (tuple, list))) and<EOL>isinstance(lcdict[<NUM_LIT:0>], dict)):<EOL><INDENT>lcdict = lcdict[<NUM_LIT:0>]<EOL><DEDENT>objectid = lcdict['<STR_LIT>']<EOL>tmagseries = templateinfo[magcol][<EOL>'<STR_LIT>'<EOL>][::]<EOL>if objectid in templateinfo[magcol]['<STR_LIT>']:<EOL><INDENT>LOGWARNING('<STR_LIT>' %<EOL>objectid)<EOL>templateind = templateinfo[magcol]['<STR_LIT>'] == objectid<EOL>tmagseries = tmagseries[~templateind,:]<EOL><DEDENT>object_matches = coordutils.conesearch_kdtree(<EOL>templateinfo[magcol]['<STR_LIT>'],<EOL>lcdict['<STR_LIT>']['<STR_LIT>'], lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>mintemplatedist_arcmin/<NUM_LIT><EOL>)<EOL>if len(object_matches) > <NUM_LIT:0>:<EOL><INDENT>LOGWARNING(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" %<EOL>(objectid, mintemplatedist_arcmin, len(object_matches))<EOL>)<EOL>removalind = np.full(<EOL>templateinfo[magcol]['<STR_LIT>'].size,<EOL>False, dtype=np.bool<EOL>)<EOL>removalind[np.array(object_matches)] = True<EOL>tmagseries = tmagseries[~removalind,:]<EOL><DEDENT>normal_matrix = np.dot(tmagseries, tmagseries.T)<EOL>normal_matrix_inverse = spla.pinv2(normal_matrix)<EOL>timebase = templateinfo[magcol]['<STR_LIT>']<EOL>reformed_targetlc = _reform_templatelc_for_tfa((<EOL>lcfile,<EOL>lcformat,<EOL>lcformatdir,<EOL>timecol,<EOL>magcol,<EOL>errcol,<EOL>timebase,<EOL>interp,<EOL>sigclip<EOL>))<EOL>scalar_products = np.dot(tmagseries, reformed_targetlc['<STR_LIT>'])<EOL>corrections = np.dot(normal_matrix_inverse, scalar_products)<EOL>corrected_magseries = (<EOL>reformed_targetlc['<STR_LIT>'] -<EOL>np.dot(tmagseries.T, corrections)<EOL>)<EOL>outdict = {<EOL>'<STR_LIT>':timebase,<EOL>'<STR_LIT>':corrected_magseries,<EOL>'<STR_LIT>':reformed_targetlc['<STR_LIT>'],<EOL>'<STR_LIT>':np.median(corrected_magseries),<EOL>'<STR_LIT>': np.median(np.abs(corrected_magseries -<EOL>np.median(corrected_magseries))),<EOL>'<STR_LIT>':{'<STR_LIT>':tmagseries,<EOL>'<STR_LIT>':normal_matrix,<EOL>'<STR_LIT>':normal_matrix_inverse,<EOL>'<STR_LIT>':scalar_products,<EOL>'<STR_LIT>':corrections,<EOL>'<STR_LIT>':reformed_targetlc},<EOL>}<EOL>lcdict['<STR_LIT>'] = outdict<EOL>outfile = os.path.join(<EOL>os.path.dirname(lcfile),<EOL>'<STR_LIT>' % (<EOL>squeeze(objectid).replace('<STR_LIT:U+0020>','<STR_LIT:->'),<EOL>magcol<EOL>)<EOL>)<EOL>with open(outfile,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(lcdict, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>return outfile<EOL>

This applies the TFA correction to an LC given TFA template information.

    Parameters
    ----------

    lcfile : str
        This is the light curve file to apply the TFA correction to.

    timecol,magcol,errcol : str
        These are the column keys in the lcdict for the LC file to apply the TFA
        correction to.

    templateinfo : dict or str
        This is either the dict produced by `tfa_templates_lclist` or the pickle
        produced by the same function.

    mintemplatedist_arcmin : float
        This sets the minimum distance required from the target object for
        objects in the TFA template ensemble. Objects closer than this distance
        will be removed from the ensemble.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    interp : str
        This is passed to scipy.interpolate.interp1d as the kind of
        interpolation to use when reforming this light curve to the timebase of
        the TFA templates.

    sigclip : float or sequence of two floats or None
        This is the sigma clip to apply to this light curve before running TFA
        on it.

    Returns
    -------

    str
        This returns the filename of the light curve file generated after TFA
        applications. This is a pickle (that can be read by `lcproc.read_pklc`)
        in the same directory as `lcfile`. The `magcol` will be encoded in the
        filename, so each `magcol` in `lcfile` gets its own output file.