cplist = glob.glob(os.path.join(cpdir, cpfileglob))<EOL>return colormagdiagram_cplist(cplist,<EOL>outpkl,<EOL>color_mag1=color_mag1,<EOL>color_mag2=color_mag2,<EOL>yaxis_mag=yaxis_mag)<EOL>

This makes CMDs for all checkplot pickles in the provided directory.

    Can make an arbitrary number of CMDs given lists of x-axis colors and y-axis
    mags to use.

    Parameters
    ----------

    cpdir : list of str
        This is the directory to get the list of input checkplot pickles from.

    outpkl : str
        The filename of the output pickle that will contain the color-mag
        information for all objects in the checkplots specified in `cplist`.

    cpfileglob : str
        The UNIX fileglob to use to search for checkplot pickle files.

    color_mag1 : list of str
        This a list of the keys in each checkplot's `objectinfo` dict that will
        be used as color_1 in the equation::

                x-axis color = color_mag1 - color_mag2

    color_mag2 : list of str
        This a list of the keys in each checkplot's `objectinfo` dict that will
        be used as color_2 in the equation::

                x-axis color = color_mag1 - color_mag2

    yaxis_mag : list of str
        This is a list of the keys in each checkplot's `objectinfo` dict that
        will be used as the (absolute) magnitude y-axis of the color-mag
        diagrams.

    Returns
    -------

    str
        The path to the generated CMD pickle file for the collection of objects
        in the input checkplot directory.

    Notes
    -----

    This can make many CMDs in one go. For example, the default kwargs for
    `color_mag`, `color_mag2`, and `yaxis_mag` result in two CMDs generated and
    written to the output pickle file:

    - CMD1 -> gaiamag - kmag on the x-axis vs gaia_absmag on the y-axis
    - CMD2 -> sdssg - kmag on the x-axis vs rpmj (J reduced PM) on the y-axis


<EOL>if isinstance(cpx, str) and os.path.exists(cpx):<EOL><INDENT>cpdict = _read_checkplot_picklefile(cpx)<EOL><DEDENT>elif isinstance(cpx, dict):<EOL><INDENT>cpdict = cpx<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>')<EOL>return None<EOL><DEDENT>if isinstance(cmdpkl, str) and os.path.exists(cmdpkl):<EOL><INDENT>with open(cmdpkl, '<STR_LIT:rb>') as infd:<EOL><INDENT>cmd = pickle.load(infd)<EOL><DEDENT><DEDENT>elif isinstance(cmdpkl, dict):<EOL><INDENT>cmd = cmdpkl<EOL><DEDENT>cpdict['<STR_LIT>'] = {}<EOL>cplist_mags = cmd['<STR_LIT>']<EOL>cplist_colors = cmd['<STR_LIT>']<EOL>for c1, c2, ym, ind in zip(cmd['<STR_LIT>'],<EOL>cmd['<STR_LIT>'],<EOL>cmd['<STR_LIT>'],<EOL>range(len(cmd['<STR_LIT>']))):<EOL><INDENT>if (c1 in cpdict['<STR_LIT>'] and<EOL>cpdict['<STR_LIT>'][c1] is not None):<EOL><INDENT>c1mag = cpdict['<STR_LIT>'][c1]<EOL><DEDENT>else:<EOL><INDENT>c1mag = np.nan<EOL><DEDENT>if (c2 in cpdict['<STR_LIT>'] and<EOL>cpdict['<STR_LIT>'][c2] is not None):<EOL><INDENT>c2mag = cpdict['<STR_LIT>'][c2]<EOL><DEDENT>else:<EOL><INDENT>c2mag = np.nan<EOL><DEDENT>if (ym in cpdict['<STR_LIT>'] and<EOL>cpdict['<STR_LIT>'][ym] is not None):<EOL><INDENT>ymmag = cpdict['<STR_LIT>'][ym]<EOL><DEDENT>else:<EOL><INDENT>ymmag = np.nan<EOL><DEDENT>if (require_cmd_magcolor and<EOL>not (np.isfinite(c1mag) and<EOL>np.isfinite(c2mag) and<EOL>np.isfinite(ymmag))):<EOL><INDENT>LOGWARNING("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" %<EOL>(c1, c2, ym, cpdict['<STR_LIT>']))<EOL>continue<EOL><DEDENT>try:<EOL><INDENT>thiscmd_title = r'<STR_LIT>' % (CMD_LABELS[c1],<EOL>CMD_LABELS[c2],<EOL>CMD_LABELS[ym])<EOL>fig = plt.figure(figsize=(<NUM_LIT:10>,<NUM_LIT:8>))<EOL>plt.plot(cplist_colors[:,ind],<EOL>cplist_mags[:,ind],<EOL>rasterized=True,<EOL>marker='<STR_LIT:o>',<EOL>linestyle='<STR_LIT:none>',<EOL>mew=<NUM_LIT:0>,<EOL>ms=<NUM_LIT:3>)<EOL>plt.plot([c1mag - c2mag], [ymmag],<EOL>ms=<NUM_LIT:20>,<EOL>color='<STR_LIT>',<EOL>marker='<STR_LIT:*>',<EOL>mew=<NUM_LIT:0>)<EOL>plt.xlabel(r'<STR_LIT>' % (CMD_LABELS[c1], CMD_LABELS[c2]))<EOL>plt.ylabel(r'<STR_LIT>' % CMD_LABELS[ym])<EOL>plt.title('<STR_LIT>' % (cpdict['<STR_LIT>'], thiscmd_title))<EOL>plt.gca().invert_yaxis()<EOL>cmdpng = StrIO()<EOL>plt.savefig(cmdpng, bbox_inches='<STR_LIT>',<EOL>pad_inches=<NUM_LIT:0.0>, format='<STR_LIT>')<EOL>cmdpng.seek(<NUM_LIT:0>)<EOL>cmdb64 = base64.b64encode(cmdpng.read())<EOL>cmdpng.close()<EOL>plt.close('<STR_LIT:all>')<EOL>plt.gcf().clear()<EOL>cpdict['<STR_LIT>']['<STR_LIT>' % (c1,c2,ym)] = cmdb64<EOL>if save_cmd_pngs:<EOL><INDENT>if isinstance(cpx, str):<EOL><INDENT>outpng = os.path.join(os.path.dirname(cpx),<EOL>'<STR_LIT>' %<EOL>(cpdict['<STR_LIT>'],<EOL>c1,c2,ym))<EOL><DEDENT>else:<EOL><INDENT>outpng = '<STR_LIT>' % (cpdict['<STR_LIT>'],<EOL>c1,c2,ym)<EOL><DEDENT>_base64_to_file(cmdb64, outpng)<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' %<EOL>(c1, c2, ym, cmdpkl))<EOL>continue<EOL><DEDENT><DEDENT>if isinstance(cpx, str):<EOL><INDENT>cpf = _write_checkplot_picklefile(cpdict, outfile=cpx, protocol=<NUM_LIT:4>)<EOL>return cpf<EOL><DEDENT>elif isinstance(cpx, dict):<EOL><INDENT>return cpdict<EOL><DEDENT>

This adds CMD figures to a checkplot dict or pickle.

    Looks up the CMDs in `cmdpkl`, adds the object from `cpx` as a gold(-ish)
    star in the plot, and then saves the figure to a base64 encoded PNG, which
    can then be read and used by the `checkplotserver`.

    Parameters
    ----------

    cpx : str or dict
        This is the input checkplot pickle or dict to add the CMD to.

    cmdpkl : str or dict
        The CMD pickle generated by the `colormagdiagram_cplist` or
        `colormagdiagram_cpdir` functions above, or the dict produced by reading
        this pickle in.

    require_cmd_magcolor : bool
        If this is True, a CMD plot will not be made if the color and mag keys
        required by the CMD are not present or are nan in this checkplot's
        objectinfo dict.

    save_cmd_png : bool
        If this is True, then will save the CMD plots that were generated and
        added back to the checkplotdict as PNGs to the same directory as
        `cpx`. If `cpx` is a dict, will save them to the current working
        directory.

    Returns
    -------

    str or dict
        If `cpx` was a str filename of checkplot pickle, this will return that
        filename to indicate that the CMD was added to the file. If `cpx` was a
        checkplotdict, this will return the checkplotdict with a new key called
        'colormagdiagram' containing the base64 encoded PNG binary streams of
        all CMDs generated.


<EOL>with open(cmdpkl,'<STR_LIT:rb>') as infd:<EOL><INDENT>cmd = pickle.load(infd)<EOL><DEDENT>for cpf in cplist:<EOL><INDENT>add_cmd_to_checkplot(cpf, cmd,<EOL>require_cmd_magcolor=require_cmd_magcolor,<EOL>save_cmd_pngs=save_cmd_pngs)<EOL><DEDENT>

This adds CMDs for each object in cplist.

    Parameters
    ----------

    cplist : list of str
        This is the input list of checkplot pickles to add the CMDs to.

    cmdpkl : str
        This is the filename of the CMD pickle created previously.

    require_cmd_magcolor : bool
        If this is True, a CMD plot will not be made if the color and mag keys
        required by the CMD are not present or are nan in each checkplot's
        objectinfo dict.

    save_cmd_pngs : bool
        If this is True, then will save the CMD plots that were generated and
        added back to the checkplotdict as PNGs to the same directory as
        `cpx`.

    Returns
    -------

    Nothing.


cplist = glob.glob(os.path.join(cpdir, cpfileglob))<EOL>return add_cmds_cplist(cplist,<EOL>cmdpkl,<EOL>require_cmd_magcolor=require_cmd_magcolor,<EOL>save_cmd_pngs=save_cmd_pngs)<EOL>

This adds CMDs for each object in cpdir.

    Parameters
    ----------

    cpdir : list of str
        This is the directory to search for checkplot pickles.

    cmdpkl : str
        This is the filename of the CMD pickle created previously.

    cpfileglob : str
        The UNIX fileglob to use when searching for checkplot pickles to operate
        on.

    require_cmd_magcolor : bool
        If this is True, a CMD plot will not be made if the color and mag keys
        required by the CMD are not present or are nan in each checkplot's
        objectinfo dict.

    save_cmd_pngs : bool
        If this is True, then will save the CMD plots that were generated and
        added back to the checkplotdict as PNGs to the same directory as
        `cpx`.

    Returns
    -------

    Nothing.


cpf, cpkwargs = task<EOL>try:<EOL><INDENT>newcpf = update_checkplot_objectinfo(cpf, **cpkwargs)<EOL>return newcpf<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' % cpf)<EOL>return None<EOL><DEDENT>

This is a parallel worker for `parallel_update_cp_objectinfo`.

    Parameters
    ----------

    task : tuple
        - task[0] = checkplot pickle file
        - task[1] = kwargs

    Returns
    -------

    str
        The name of the checkplot file that was updated. None if the update
        fails for some reason.


<EOL>if sys.platform == '<STR_LIT>':<EOL><INDENT>import requests<EOL>requests.get('<STR_LIT>')<EOL><DEDENT>if (liststartindex is not None) and (maxobjects is None):<EOL><INDENT>cplist = cplist[liststartindex:]<EOL><DEDENT>elif (liststartindex is None) and (maxobjects is not None):<EOL><INDENT>cplist = cplist[:maxobjects]<EOL><DEDENT>elif (liststartindex is not None) and (maxobjects is not None):<EOL><INDENT>cplist = (<EOL>cplist[liststartindex:liststartindex+maxobjects]<EOL>)<EOL><DEDENT>tasks = [(x, {'<STR_LIT>':fast_mode,<EOL>'<STR_LIT>':findercmap,<EOL>'<STR_LIT>':finderconvolve,<EOL>'<STR_LIT>':deredden_object,<EOL>'<STR_LIT>':custom_bandpasses,<EOL>'<STR_LIT>':gaia_submit_timeout,<EOL>'<STR_LIT>':gaia_submit_tries,<EOL>'<STR_LIT>':gaia_max_timeout,<EOL>'<STR_LIT>':gaia_mirror,<EOL>'<STR_LIT>':complete_query_later,<EOL>'<STR_LIT>':lclistpkl,<EOL>'<STR_LIT>':nbrradiusarcsec,<EOL>'<STR_LIT>':maxnumneighbors,<EOL>'<STR_LIT>':plotdpi,<EOL>'<STR_LIT>':findercachedir,<EOL>'<STR_LIT>':verbose}) for x in cplist]<EOL>resultfutures = []<EOL>results = []<EOL>with ProcessPoolExecutor(max_workers=nworkers) as executor:<EOL><INDENT>resultfutures = executor.map(cp_objectinfo_worker, tasks)<EOL><DEDENT>results = [x for x in resultfutures]<EOL>executor.shutdown()<EOL>return results<EOL>

This updates objectinfo for a list of checkplots.

Useful in cases where a previous round of GAIA/finderchart/external catalog
acquisition failed. This will preserve the following keys in the checkplots
if they exist:

comments
varinfo
objectinfo.objecttags

Parameters
----------

cplist : list of str
    A list of checkplot pickle file names to update.

liststartindex : int
    The index of the input list to start working at.

maxobjects : int
    The maximum number of objects to process in this run. Use this with
    `liststartindex` to effectively distribute working on a large list of
    input checkplot pickles over several sessions or machines.

nworkers : int
    The number of parallel workers that will work on the checkplot
    update process.

fast_mode : bool or float
    This runs the external catalog operations in a "fast" mode, with short
    timeouts and not trying to hit external catalogs that take a long time
    to respond. See the docstring for
    `checkplot.pkl_utils._pkl_finder_objectinfo` for details on how this
    works. If this is True, will run in "fast" mode with default timeouts (5
    seconds in most cases). If this is a float, will run in "fast" mode with
    the provided timeout value in seconds.

findercmap : str or matplotlib.cm.Colormap object

findercmap : str or matplotlib.cm.ColorMap object
    The Colormap object to use for the finder chart image.

finderconvolve : astropy.convolution.Kernel object or None
    If not None, the Kernel object to use for convolving the finder image.

deredden_objects : bool
    If this is True, will use the 2MASS DUST service to get extinction
    coefficients in various bands, and then try to deredden the magnitudes
    and colors of the object already present in the checkplot's objectinfo
    dict.

custom_bandpasses : dict
    This is a dict used to provide custom bandpass definitions for any
    magnitude measurements in the objectinfo dict that are not automatically
    recognized by the `varclass.starfeatures.color_features` function. See
    its docstring for details on the required format.

gaia_submit_timeout : float
    Sets the timeout in seconds to use when submitting a request to look up
    the object's information to the GAIA service. Note that if `fast_mode`
    is set, this is ignored.

gaia_submit_tries : int
    Sets the maximum number of times the GAIA services will be contacted to
    obtain this object's information. If `fast_mode` is set, this is
    ignored, and the services will be contacted only once (meaning that a
    failure to respond will be silently ignored and no GAIA data will be
    added to the checkplot's objectinfo dict).

gaia_max_timeout : float
    Sets the timeout in seconds to use when waiting for the GAIA service to
    respond to our request for the object's information. Note that if
    `fast_mode` is set, this is ignored.

gaia_mirror : str
    This sets the GAIA mirror to use. This is a key in the
    `services.gaia.GAIA_URLS` dict which defines the URLs to hit for each
    mirror.

complete_query_later : bool
    If this is True, saves the state of GAIA queries that are not yet
    complete when `gaia_max_timeout` is reached while waiting for the GAIA
    service to respond to our request. A later call for GAIA info on the
    same object will attempt to pick up the results from the existing query
    if it's completed. If `fast_mode` is True, this is ignored.

lclistpkl : dict or str
    If this is provided, must be a dict resulting from reading a catalog
    produced by the `lcproc.catalogs.make_lclist` function or a str path
    pointing to the pickle file produced by that function. This catalog is
    used to find neighbors of the current object in the current light curve
    collection. Looking at neighbors of the object within the radius
    specified by `nbrradiusarcsec` is useful for light curves produced by
    instruments that have a large pixel scale, so are susceptible to
    blending of variability and potential confusion of neighbor variability
    with that of the actual object being looked at. If this is None, no
    neighbor lookups will be performed.

nbrradiusarcsec : float
    The radius in arcseconds to use for a search conducted around the
    coordinates of this object to look for any potential confusion and
    blending of variability amplitude caused by their proximity.

maxnumneighbors : int
    The maximum number of neighbors that will have their light curves and
    magnitudes noted in this checkplot as potential blends with the target
    object.

plotdpi : int
    The resolution in DPI of the plots to generate in this function
    (e.g. the finder chart, etc.)

findercachedir : str
    The path to the astrobase cache directory for finder chart downloads
    from the NASA SkyView service.

verbose : bool
    If True, will indicate progress and warn about potential problems.

Returns
-------

list of str
    Paths to the updated checkplot pickle file.


This updates the objectinfo for a directory of checkplot pickles.

    Useful in cases where a previous round of GAIA/finderchart/external catalog
    acquisition failed. This will preserve the following keys in the checkplots
    if they exist:

    comments
    varinfo
    objectinfo.objecttags

    Parameters
    ----------

    cpdir : str
        The directory to look for checkplot pickles in.

    cpglob : str
        The UNIX fileglob to use when searching for checkplot pickle files.

    liststartindex : int
        The index of the input list to start working at.

    maxobjects : int
        The maximum number of objects to process in this run. Use this with
        `liststartindex` to effectively distribute working on a large list of
        input checkplot pickles over several sessions or machines.

    nworkers : int
        The number of parallel workers that will work on the checkplot
        update process.

    fast_mode : bool or float
        This runs the external catalog operations in a "fast" mode, with short
        timeouts and not trying to hit external catalogs that take a long time
        to respond. See the docstring for
        `checkplot.pkl_utils._pkl_finder_objectinfo` for details on how this
        works. If this is True, will run in "fast" mode with default timeouts (5
        seconds in most cases). If this is a float, will run in "fast" mode with
        the provided timeout value in seconds.

    findercmap : str or matplotlib.cm.Colormap object

    findercmap : str or matplotlib.cm.ColorMap object
        The Colormap object to use for the finder chart image.

    finderconvolve : astropy.convolution.Kernel object or None
        If not None, the Kernel object to use for convolving the finder image.

    deredden_objects : bool
        If this is True, will use the 2MASS DUST service to get extinction
        coefficients in various bands, and then try to deredden the magnitudes
        and colors of the object already present in the checkplot's objectinfo
        dict.

    custom_bandpasses : dict
        This is a dict used to provide custom bandpass definitions for any
        magnitude measurements in the objectinfo dict that are not automatically
        recognized by the `varclass.starfeatures.color_features` function. See
        its docstring for details on the required format.

    gaia_submit_timeout : float
        Sets the timeout in seconds to use when submitting a request to look up
        the object's information to the GAIA service. Note that if `fast_mode`
        is set, this is ignored.

    gaia_submit_tries : int
        Sets the maximum number of times the GAIA services will be contacted to
        obtain this object's information. If `fast_mode` is set, this is
        ignored, and the services will be contacted only once (meaning that a
        failure to respond will be silently ignored and no GAIA data will be
        added to the checkplot's objectinfo dict).

    gaia_max_timeout : float
        Sets the timeout in seconds to use when waiting for the GAIA service to
        respond to our request for the object's information. Note that if
        `fast_mode` is set, this is ignored.

    gaia_mirror : str
        This sets the GAIA mirror to use. This is a key in the
        `services.gaia.GAIA_URLS` dict which defines the URLs to hit for each
        mirror.

    complete_query_later : bool
        If this is True, saves the state of GAIA queries that are not yet
        complete when `gaia_max_timeout` is reached while waiting for the GAIA
        service to respond to our request. A later call for GAIA info on the
        same object will attempt to pick up the results from the existing query
        if it's completed. If `fast_mode` is True, this is ignored.

    lclistpkl : dict or str
        If this is provided, must be a dict resulting from reading a catalog
        produced by the `lcproc.catalogs.make_lclist` function or a str path
        pointing to the pickle file produced by that function. This catalog is
        used to find neighbors of the current object in the current light curve
        collection. Looking at neighbors of the object within the radius
        specified by `nbrradiusarcsec` is useful for light curves produced by
        instruments that have a large pixel scale, so are susceptible to
        blending of variability and potential confusion of neighbor variability
        with that of the actual object being looked at. If this is None, no
        neighbor lookups will be performed.

    nbrradiusarcsec : float
        The radius in arcseconds to use for a search conducted around the
        coordinates of this object to look for any potential confusion and
        blending of variability amplitude caused by their proximity.

    maxnumneighbors : int
        The maximum number of neighbors that will have their light curves and
        magnitudes noted in this checkplot as potential blends with the target
        object.

    plotdpi : int
        The resolution in DPI of the plots to generate in this function
        (e.g. the finder chart, etc.)

    findercachedir : str
        The path to the astrobase cache directory for finder chart downloads
        from the NASA SkyView service.

    verbose : bool
        If True, will indicate progress and warn about potential problems.

    Returns
    -------

    list of str
        Paths to the updated checkplot pickle file.


raise KeyboardInterrupt<EOL>

This raises a KeyboardInterrupt when a SIGKILL comes in.

    This is a handle for use with the Python `signal.signal` function.


<EOL>cmd = (<EOL>"<STR_LIT>"<EOL>)<EOL>mmin = '<STR_LIT>' % (min_age_hours*<NUM_LIT>)<EOL>cmd = cmd.format(mmin=mmin)<EOL>try:<EOL><INDENT>proc = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE)<EOL>ndeleted = len(proc.stdout.decode().split('<STR_LIT:\n>'))<EOL>LOGWARNING('<STR_LIT>' %<EOL>(ndeleted, min_age_hours))<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>')<EOL><DEDENT>

This periodically cleans up the ~/.astrobase cache to save us from
    disk-space doom.

    Parameters
    ----------

    min_age_hours : int
        Files older than this number of hours from the current time will be
        deleted.

    Returns
    -------

    Nothing.


url = '<STR_LIT>'<EOL>try:<EOL><INDENT>resp = requests.get(url, timeout=<NUM_LIT:1.0>)<EOL>resp.raise_for_status()<EOL>stopinfo = resp.json()<EOL>if '<STR_LIT:action>' in stopinfo and stopinfo['<STR_LIT:action>'] in ('<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>'):<EOL><INDENT>stoptime = stopinfo['<STR_LIT:time>']<EOL>LOGWARNING('<STR_LIT>' % (stopinfo['<STR_LIT:action>'],<EOL>stoptime))<EOL>resp.close()<EOL>return True<EOL><DEDENT>else:<EOL><INDENT>resp.close()<EOL>return False<EOL><DEDENT><DEDENT>except HTTPError as e:<EOL><INDENT>resp.close()<EOL>return False<EOL><DEDENT>except Exception as e:<EOL><INDENT>resp.close()<EOL>return False<EOL><DEDENT>

This checks the AWS instance data URL to see if there's a pending
    shutdown for the instance.

    This is useful for AWS spot instances. If there is a pending shutdown posted
    to the instance data URL, we'll use the result of this function break out of
    the processing loop and shut everything down ASAP before the instance dies.

    Returns
    -------

    bool
        - True if the instance is going to die soon.
        - False if the instance is still safe.


if not sqs_client:<EOL><INDENT>sqs_client = boto3.client('<STR_LIT>')<EOL><DEDENT>if not s3_client:<EOL><INDENT>s3_client = boto3.client('<STR_LIT>')<EOL><DEDENT>if isinstance(lightcurve_list, str) and os.path.exists(lightcurve_list):<EOL><INDENT>with open(lightcurve_list, '<STR_LIT:r>') as infd:<EOL><INDENT>lclist = infd.readlines()<EOL><DEDENT>lclist = [x.replace('<STR_LIT:\n>','<STR_LIT>') for x in lclist if len(x) > <NUM_LIT:0>]<EOL>if process_list_slice is not None:<EOL><INDENT>lclist = lclist[process_list_slice[<NUM_LIT:0>]:process_list_slice[<NUM_LIT:1>]]<EOL><DEDENT>lclist = [x[<NUM_LIT:1>:] for x in lclist if x.startswith('<STR_LIT:/>')]<EOL>lclist = ['<STR_LIT>' % (input_bucket, x) for x in lclist]<EOL><DEDENT>elif isinstance(lightcurve_list, list):<EOL><INDENT>lclist = lightcurve_list<EOL><DEDENT>try:<EOL><INDENT>inq = sqs_client.get_queue_url(QueueName=input_queue)<EOL>inq_url = inq['<STR_LIT>']<EOL>LOGINFO('<STR_LIT>')<EOL><DEDENT>except ClientError as e:<EOL><INDENT>inq = awsutils.sqs_create_queue(input_queue, client=sqs_client)<EOL>inq_url = inq['<STR_LIT:url>']<EOL><DEDENT>try:<EOL><INDENT>outq = sqs_client.get_queue_url(QueueName=result_queue)<EOL>outq_url = outq['<STR_LIT>']<EOL>LOGINFO('<STR_LIT>')<EOL><DEDENT>except ClientError as e:<EOL><INDENT>outq = awsutils.sqs_create_queue(result_queue, client=sqs_client)<EOL>outq_url = outq['<STR_LIT:url>']<EOL><DEDENT>LOGINFO('<STR_LIT>' % inq_url)<EOL>LOGINFO('<STR_LIT>' % outq_url)<EOL>LOGINFO('<STR_LIT>')<EOL>time.sleep(<NUM_LIT>)<EOL>if pfresult_list is None:<EOL><INDENT>pfresult_list = [None for x in lclist]<EOL><DEDENT>for lc, pf in zip(lclist, pfresult_list):<EOL><INDENT>this_item = {<EOL>'<STR_LIT:target>': lc,<EOL>'<STR_LIT:action>': '<STR_LIT>',<EOL>'<STR_LIT:args>': (pf,),<EOL>'<STR_LIT>':runcp_kwargs if runcp_kwargs is not None else {},<EOL>'<STR_LIT>': result_bucket,<EOL>'<STR_LIT>': outq_url<EOL>}<EOL>resp = awsutils.sqs_put_item(inq_url, this_item, client=sqs_client)<EOL>if resp:<EOL><INDENT>LOGINFO('<STR_LIT>' % (lc,inq_url))<EOL><DEDENT><DEDENT>done_objects = {}<EOL>LOGINFO('<STR_LIT>')<EOL>signal.signal(signal.SIGINT, kill_handler)<EOL>signal.signal(signal.SIGTERM, kill_handler)<EOL>while len(list(done_objects.keys())) < len(lclist):<EOL><INDENT>try:<EOL><INDENT>result = awsutils.sqs_get_item(outq_url, client=sqs_client)<EOL>if result is not None and len(result) > <NUM_LIT:0>:<EOL><INDENT>recv = result[<NUM_LIT:0>]<EOL>try:<EOL><INDENT>processed_object = recv['<STR_LIT>']['<STR_LIT:target>']<EOL><DEDENT>except KeyError:<EOL><INDENT>LOGWARNING('<STR_LIT>' % recv)<EOL>processed_object = '<STR_LIT>'<EOL><DEDENT>cpf = recv['<STR_LIT>']['<STR_LIT>']<EOL>receipt = recv['<STR_LIT>']<EOL>if processed_object in lclist:<EOL><INDENT>if processed_object not in done_objects:<EOL><INDENT>done_objects[processed_object] = [cpf]<EOL><DEDENT>else:<EOL><INDENT>done_objects[processed_object].append(cpf)<EOL><DEDENT>LOGINFO('<STR_LIT>' % (processed_object, cpf))<EOL>if download_when_done:<EOL><INDENT>getobj = awsutils.awsutils.s3_get_url(<EOL>cpf,<EOL>client=s3_client<EOL>)<EOL>LOGINFO('<STR_LIT>' % (cpf, getobj))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>awsutils.sqs_delete_item(outq_url, receipt)<EOL><DEDENT><DEDENT>except KeyboardInterrupt as e:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT><DEDENT>LOGINFO('<STR_LIT>')<EOL>time.sleep(<NUM_LIT:1.0>)<EOL>if purge_queues_when_done:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>sqs_client.purge_queue(QueueUrl=inq_url)<EOL>sqs_client.purge_queue(QueueUrl=outq_url)<EOL>time.sleep(<NUM_LIT>)<EOL><DEDENT>if delete_queues_when_done:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>awsutils.sqs_delete_queue(inq_url)<EOL>awsutils.sqs_delete_queue(outq_url)<EOL><DEDENT>work_state = {<EOL>'<STR_LIT>': done_objects,<EOL>'<STR_LIT>': list(set(lclist) - set(done_objects.keys())),<EOL>'<STR_LIT:args>':((os.path.abspath(lightcurve_list) if<EOL>isinstance(lightcurve_list, str) else lightcurve_list),<EOL>input_queue,<EOL>input_bucket,<EOL>result_queue,<EOL>result_bucket),<EOL>'<STR_LIT>':{'<STR_LIT>':pfresult_list,<EOL>'<STR_LIT>':runcp_kwargs,<EOL>'<STR_LIT>':process_list_slice,<EOL>'<STR_LIT>':download_when_done,<EOL>'<STR_LIT>':purge_queues_when_done,<EOL>'<STR_LIT>':save_state_when_done,<EOL>'<STR_LIT>':delete_queues_when_done}<EOL>}<EOL>if save_state_when_done:<EOL><INDENT>with open('<STR_LIT>','<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(work_state, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT><DEDENT>return work_state<EOL>

This sends checkplot making tasks to the input queue and monitors the
    result queue for task completion.

    Parameters
    ----------

    lightcurve_list : str or list of str
        This is either a string pointing to a file containing a list of light
        curves filenames to process or the list itself. The names must
        correspond to the full filenames of files stored on S3, including all
        prefixes, but not include the 's3://<bucket name>/' bit (these will be
        added automatically).

    input_queue : str
        This is the name of the SQS queue which will receive processing tasks
        generated by this function. The queue URL will automatically be obtained
        from AWS.

    input_bucket : str
        The name of the S3 bucket containing the light curve files to process.

    result_queue : str
        This is the name of the SQS queue that this function will listen to for
        messages from the workers as they complete processing on their input
        elements. This function will attempt to match input sent to the
        `input_queue` with results coming into the `result_queue` so it knows
        how many objects have been successfully processed. If this function
        receives task results that aren't in its own input queue, it will
        acknowledge them so they complete successfully, but not download them
        automatically. This handles leftover tasks completing from a previous
        run of this function.

    result_bucket : str
        The name of the S3 bucket which will receive the results from the
        workers.

    pfresult_list : list of str or None
        This is a list of periodfinder result pickle S3 URLs associated with
        each light curve. If provided, this will be used to add in phased light
        curve plots to each checkplot pickle. If this is None, the worker loop
        will produce checkplot pickles that only contain object information,
        neighbor information, and unphased light curves.

    runcp_kwargs : dict
        This is a dict used to pass any extra keyword arguments to the
        `lcproc.checkplotgen.runcp` function that will be run by the worker
        loop.

    process_list_slice : list
        This is used to index into the input light curve list so a subset of the
        full list can be processed in this specific run of this function.

        Use None for a slice index elem to emulate single slice spec behavior:

        process_list_slice = [10, None]  -> lightcurve_list[10:]
        process_list_slice = [None, 500] -> lightcurve_list[:500]

    purge_queues_when_done : bool
        If this is True, and this function exits (either when all done, or when
        it is interrupted with a Ctrl+C), all outstanding elements in the
        input/output queues that have not yet been acknowledged by workers or by
        this function will be purged. This effectively cancels all outstanding
        work.

    delete_queues_when_done : bool
        If this is True, and this function exits (either when all done, or when
        it is interrupted with a Ctrl+C'), all outstanding work items will be
        purged from the input/queues and the queues themselves will be deleted.

    download_when_done : bool
        If this is True, the generated checkplot pickle for each input work item
        will be downloaded immediately to the current working directory when the
        worker functions report they're done with it.

    save_state_when_done : bool
        If this is True, will save the current state of the work item queue and
        the work items acknowledged as completed to a pickle in the current
        working directory. Call the `runcp_producer_loop_savedstate` function
        below to resume processing from this saved state later.

    s3_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its S3 download operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    sqs_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its SQS operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    Returns
    -------

    dict or str
        Returns the current work state as a dict or str path to the generated
        work state pickle depending on if `save_state_when_done` is True.


This wraps the function above to allow for loading previous state from a
    file.

    Parameters
    ----------

    use_saved_state : str or None
        This is the path to the saved state pickle file produced by a previous
        run of `runcp_producer_loop`. Will get all of the arguments to run
        another instance of the loop from that pickle file. If this is None, you
        MUST provide all of the appropriate arguments to that function.

    lightcurve_list : str or list of str or None
        This is either a string pointing to a file containing a list of light
        curves filenames to process or the list itself. The names must
        correspond to the full filenames of files stored on S3, including all
        prefixes, but not include the 's3://<bucket name>/' bit (these will be
        added automatically).

    input_queue : str or None
        This is the name of the SQS queue which will receive processing tasks
        generated by this function. The queue URL will automatically be obtained
        from AWS.

    input_bucket : str or None
        The name of the S3 bucket containing the light curve files to process.

    result_queue : str or None
        This is the name of the SQS queue that this function will listen to for
        messages from the workers as they complete processing on their input
        elements. This function will attempt to match input sent to the
        `input_queue` with results coming into the `result_queue` so it knows
        how many objects have been successfully processed. If this function
        receives task results that aren't in its own input queue, it will
        acknowledge them so they complete successfully, but not download them
        automatically. This handles leftover tasks completing from a previous
        run of this function.

    result_bucket : str or None
        The name of the S3 bucket which will receive the results from the
        workers.

    pfresult_list : list of str or None
        This is a list of periodfinder result pickle S3 URLs associated with
        each light curve. If provided, this will be used to add in phased light
        curve plots to each checkplot pickle. If this is None, the worker loop
        will produce checkplot pickles that only contain object information,
        neighbor information, and unphased light curves.

    runcp_kwargs : dict or None
        This is a dict used to pass any extra keyword arguments to the
        `lcproc.checkplotgen.runcp` function that will be run by the worker
        loop.

    process_list_slice : list or None
        This is used to index into the input light curve list so a subset of the
        full list can be processed in this specific run of this function.

        Use None for a slice index elem to emulate single slice spec behavior:

        process_list_slice = [10, None]  -> lightcurve_list[10:]
        process_list_slice = [None, 500] -> lightcurve_list[:500]

    purge_queues_when_done : bool or None
        If this is True, and this function exits (either when all done, or when
        it is interrupted with a Ctrl+C), all outstanding elements in the
        input/output queues that have not yet been acknowledged by workers or by
        this function will be purged. This effectively cancels all outstanding
        work.

    delete_queues_when_done : bool or None
        If this is True, and this function exits (either when all done, or when
        it is interrupted with a Ctrl+C'), all outstanding work items will be
        purged from the input/queues and the queues themselves will be deleted.

    download_when_done : bool or None
        If this is True, the generated checkplot pickle for each input work item
        will be downloaded immediately to the current working directory when the
        worker functions report they're done with it.

    save_state_when_done : bool or None
        If this is True, will save the current state of the work item queue and
        the work items acknowledged as completed to a pickle in the current
        working directory. Call the `runcp_producer_loop_savedstate` function
        below to resume processing from this saved state later.

    s3_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its S3 download operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    sqs_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its SQS operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    Returns
    -------

    dict or str
        Returns the current work state as a dict or str path to the generated
        work state pickle depending on if `save_state_when_done` is True.


if not sqs_client:<EOL><INDENT>sqs_client = boto3.client('<STR_LIT>')<EOL><DEDENT>if not s3_client:<EOL><INDENT>s3_client = boto3.client('<STR_LIT>')<EOL><DEDENT>lclist_pklf = lclist_pkl_s3url.split('<STR_LIT:/>')[-<NUM_LIT:1>]<EOL>if not os.path.exists(lclist_pklf):<EOL><INDENT>lclist_pklf = awsutils.s3_get_url(<EOL>lclist_pkl_s3url,<EOL>client=s3_client<EOL>)<EOL><DEDENT>with open(lclist_pklf,'<STR_LIT:rb>') as infd:<EOL><INDENT>lclistpkl = pickle.load(infd)<EOL><DEDENT>signal.signal(signal.SIGINT, kill_handler)<EOL>signal.signal(signal.SIGTERM, kill_handler)<EOL>shutdown_last_time = time.monotonic()<EOL>diskspace_last_time = time.monotonic()<EOL>while True:<EOL><INDENT>curr_time = time.monotonic()<EOL>if (curr_time - shutdown_last_time) > shutdown_check_timer_seconds:<EOL><INDENT>shutdown_check = shutdown_check_handler()<EOL>if shutdown_check:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>shutdown_last_time = time.monotonic()<EOL><DEDENT>if (curr_time - diskspace_last_time) > cache_clean_timer_seconds:<EOL><INDENT>cache_clean_handler()<EOL>diskspace_last_time = time.monotonic()<EOL><DEDENT>try:<EOL><INDENT>work = awsutils.sqs_get_item(in_queue_url,<EOL>client=sqs_client,<EOL>raiseonfail=True)<EOL>if work is not None and len(work) > <NUM_LIT:0>:<EOL><INDENT>recv = work[<NUM_LIT:0>]<EOL>action = recv['<STR_LIT>']['<STR_LIT:action>']<EOL>if action != '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>target = recv['<STR_LIT>']['<STR_LIT:target>']<EOL>args = recv['<STR_LIT>']['<STR_LIT:args>']<EOL>kwargs = recv['<STR_LIT>']['<STR_LIT>']<EOL>outbucket = recv['<STR_LIT>']['<STR_LIT>']<EOL>if '<STR_LIT>' in recv['<STR_LIT>']:<EOL><INDENT>out_queue_url = recv['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>out_queue_url = None<EOL><DEDENT>receipt = recv['<STR_LIT>']<EOL>try:<EOL><INDENT>lc_filename = awsutils.s3_get_url(<EOL>target,<EOL>altexts=lc_altexts,<EOL>client=s3_client,<EOL>)<EOL>if len(args) > <NUM_LIT:0> and args[<NUM_LIT:0>] is not None:<EOL><INDENT>pf_pickle = awsutils.s3_get_url(<EOL>args[<NUM_LIT:0>],<EOL>client=s3_client<EOL>)<EOL><DEDENT>else:<EOL><INDENT>pf_pickle = None<EOL><DEDENT>cpfs = runcp(<EOL>pf_pickle,<EOL>workdir,<EOL>workdir,<EOL>lcfname=lc_filename,<EOL>lclistpkl=lclistpkl,<EOL>makeneighborlcs=False,<EOL>**kwargs<EOL>)<EOL>if cpfs and all(os.path.exists(x) for x in cpfs):<EOL><INDENT>LOGINFO('<STR_LIT>' %<EOL>(lc_filename, pf_pickle, cpfs))<EOL>resp = s3_client.list_objects_v2(<EOL>Bucket=outbucket,<EOL>MaxKeys=<NUM_LIT:1>,<EOL>Prefix=cpfs[<NUM_LIT:0>]<EOL>)<EOL>outbucket_list = resp.get('<STR_LIT>',[])<EOL>if outbucket_list and len(outbucket_list) > <NUM_LIT:0>:<EOL><INDENT>LOGWARNING(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% target<EOL>)<EOL>awsutils.sqs_delete_item(in_queue_url, receipt)<EOL>continue<EOL><DEDENT>for cpf in cpfs:<EOL><INDENT>put_url = awsutils.s3_put_file(cpf,<EOL>outbucket,<EOL>client=s3_client)<EOL>if put_url is not None:<EOL><INDENT>LOGINFO('<STR_LIT>' % put_url)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT:target>': target,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':lclist_pklf,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>os.remove(cpf)<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>' % cpf)<EOL><DEDENT><DEDENT>awsutils.sqs_delete_item(in_queue_url,<EOL>receipt)<EOL>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>' %<EOL>(lc_filename, pf_pickle))<EOL>with open('<STR_LIT>' %<EOL>lc_filename, '<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(<EOL>{'<STR_LIT>':in_queue_url,<EOL>'<STR_LIT:target>':target,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':lclist_pklf,<EOL>'<STR_LIT>':kwargs,<EOL>'<STR_LIT>':outbucket,<EOL>'<STR_LIT>':out_queue_url},<EOL>outfd, pickle.HIGHEST_PROTOCOL<EOL>)<EOL><DEDENT>put_url = awsutils.s3_put_file(<EOL>'<STR_LIT>' % lc_filename,<EOL>outbucket,<EOL>client=s3_client<EOL>)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':lclist_pklf,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>awsutils.sqs_delete_item(in_queue_url,<EOL>receipt,<EOL>raiseonfail=True)<EOL>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT><DEDENT>except ClientError as e:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>')<EOL>if '<STR_LIT>' in locals():<EOL><INDENT>with open('<STR_LIT>' %<EOL>lc_filename,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(<EOL>{'<STR_LIT>':in_queue_url,<EOL>'<STR_LIT:target>':target,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':lclist_pklf,<EOL>'<STR_LIT>':kwargs,<EOL>'<STR_LIT>':outbucket,<EOL>'<STR_LIT>':out_queue_url},<EOL>outfd, pickle.HIGHEST_PROTOCOL<EOL>)<EOL><DEDENT>put_url = awsutils.s3_put_file(<EOL>'<STR_LIT>' % lc_filename,<EOL>outbucket,<EOL>client=s3_client<EOL>)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':lclist_pklf,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT>awsutils.sqs_delete_item(in_queue_url,<EOL>receipt,<EOL>raiseonfail=True)<EOL><DEDENT><DEDENT><DEDENT>except KeyboardInterrupt:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>except ClientError as e:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>')<EOL>if '<STR_LIT>' in locals():<EOL><INDENT>with open('<STR_LIT>' %<EOL>lc_filename,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(<EOL>{'<STR_LIT>':in_queue_url,<EOL>'<STR_LIT:target>':target,<EOL>'<STR_LIT>':lclist_pklf,<EOL>'<STR_LIT>':kwargs,<EOL>'<STR_LIT>':outbucket,<EOL>'<STR_LIT>':out_queue_url},<EOL>outfd, pickle.HIGHEST_PROTOCOL<EOL>)<EOL><DEDENT>put_url = awsutils.s3_put_file(<EOL>'<STR_LIT>' % lc_filename,<EOL>outbucket,<EOL>client=s3_client<EOL>)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT>':lclist_pklf,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT>awsutils.sqs_delete_item(in_queue_url, receipt, raiseonfail=True)<EOL><DEDENT><DEDENT>

This runs checkplot pickle making in a loop until interrupted.

    Consumes work task items from an input queue set up by `runcp_producer_loop`
    above. For the moment, we don't generate neighbor light curves since this
    would require a lot more S3 calls.

    Parameters
    ----------

    in_queue_url : str
        The SQS URL of the input queue to listen to for work assignment
        messages. The task orders will include the input and output S3 bucket
        names, as well as the URL of the output queue to where this function
        will report its work-complete or work-failed status.

    workdir : str
        The directory on the local machine where this worker loop will download
        the input light curves and associated period-finder results (if any),
        process them, and produce its output checkplot pickles. These will then
        be uploaded to the specified S3 output bucket and then deleted from the
        workdir when the upload is confirmed to make it safely to S3.

    lclist_pkl : str
        S3 URL of a catalog pickle generated by `lcproc.catalogs.make_lclist`
        that contains objectids and coordinates, as well as a kdtree for all of
        the objects in the current light curve collection being processed. This
        is used to look up neighbors for each object being processed.

    lc_altexts : sequence of str
        If not None, this is a sequence of alternate extensions to try for the
        input light curve file other than the one provided in the input task
        order. For example, to get anything that's an .sqlite where .sqlite.gz
        is expected, use altexts=[''] to strip the .gz.

    wait_time_seconds : int
        The amount of time to wait in the input SQS queue for an input task
        order. If this timeout expires and no task has been received, this
        function goes back to the top of the work loop.

    cache_clean_timer_seconds : float
        The amount of time in seconds to wait before periodically removing old
        files (such as finder chart FITS, external service result pickles) from
        the astrobase cache directory. These accumulate as the work items are
        processed, and take up significant space, so must be removed
        periodically.

    shutdown_check_timer_seconds : float
        The amount of time to wait before checking for a pending EC2 shutdown
        message for the instance this worker loop is operating on. If a shutdown
        is noticed, the worker loop is cancelled in preparation for instance
        shutdown.

    sqs_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its SQS operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    s3_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its S3 operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    Returns
    -------

    Nothing.


if not sqs_client:<EOL><INDENT>sqs_client = boto3.client('<STR_LIT>')<EOL><DEDENT>if not s3_client:<EOL><INDENT>s3_client = boto3.client('<STR_LIT>')<EOL><DEDENT>if isinstance(lightcurve_list, str) and os.path.exists(lightcurve_list):<EOL><INDENT>with open(lightcurve_list, '<STR_LIT:r>') as infd:<EOL><INDENT>lclist = infd.readlines()<EOL><DEDENT>lclist = [x.replace('<STR_LIT:\n>','<STR_LIT>') for x in lclist if len(x) > <NUM_LIT:0>]<EOL>if process_list_slice is not None:<EOL><INDENT>lclist = lclist[process_list_slice[<NUM_LIT:0>]:process_list_slice[<NUM_LIT:1>]]<EOL><DEDENT>lclist = [x[<NUM_LIT:1>:] for x in lclist if x.startswith('<STR_LIT:/>')]<EOL>lclist = ['<STR_LIT>' % (input_bucket, x) for x in lclist]<EOL><DEDENT>elif isinstance(lightcurve_list, list):<EOL><INDENT>lclist = lightcurve_list<EOL><DEDENT>try:<EOL><INDENT>inq = sqs_client.get_queue_url(QueueName=input_queue)<EOL>inq_url = inq['<STR_LIT>']<EOL>LOGINFO('<STR_LIT>')<EOL><DEDENT>except ClientError as e:<EOL><INDENT>inq = awsutils.sqs_create_queue(input_queue, client=sqs_client)<EOL>inq_url = inq['<STR_LIT:url>']<EOL><DEDENT>try:<EOL><INDENT>outq = sqs_client.get_queue_url(QueueName=result_queue)<EOL>outq_url = outq['<STR_LIT>']<EOL>LOGINFO('<STR_LIT>')<EOL><DEDENT>except ClientError as e:<EOL><INDENT>outq = awsutils.sqs_create_queue(result_queue, client=sqs_client)<EOL>outq_url = outq['<STR_LIT:url>']<EOL><DEDENT>LOGINFO('<STR_LIT>' % inq_url)<EOL>LOGINFO('<STR_LIT>' % outq_url)<EOL>LOGINFO('<STR_LIT>')<EOL>time.sleep(<NUM_LIT>)<EOL>all_runpf_kwargs = {'<STR_LIT>':pfmethods,<EOL>'<STR_LIT>':pfkwargs}<EOL>if isinstance(extra_runpf_kwargs, dict):<EOL><INDENT>all_runpf_kwargs.update(extra_runpf_kwargs)<EOL><DEDENT>for lc in lclist:<EOL><INDENT>this_item = {<EOL>'<STR_LIT:target>': lc,<EOL>'<STR_LIT:action>': '<STR_LIT>',<EOL>'<STR_LIT:args>': ('<STR_LIT:.>',),<EOL>'<STR_LIT>':all_runpf_kwargs,<EOL>'<STR_LIT>': result_bucket,<EOL>'<STR_LIT>': outq_url<EOL>}<EOL>resp = awsutils.sqs_put_item(inq_url, this_item, client=sqs_client)<EOL>if resp:<EOL><INDENT>LOGINFO('<STR_LIT>' % (lc, inq_url))<EOL><DEDENT><DEDENT>done_objects = {}<EOL>LOGINFO('<STR_LIT>')<EOL>signal.signal(signal.SIGINT, kill_handler)<EOL>signal.signal(signal.SIGTERM, kill_handler)<EOL>while len(list(done_objects.keys())) < len(lclist):<EOL><INDENT>try:<EOL><INDENT>result = awsutils.sqs_get_item(outq_url, client=sqs_client)<EOL>if result is not None and len(result) > <NUM_LIT:0>:<EOL><INDENT>recv = result[<NUM_LIT:0>]<EOL>try:<EOL><INDENT>processed_object = recv['<STR_LIT>']['<STR_LIT:target>']<EOL><DEDENT>except KeyError:<EOL><INDENT>LOGWARNING('<STR_LIT>' % recv)<EOL>processed_object = '<STR_LIT>'<EOL><DEDENT>pfresult = recv['<STR_LIT>']['<STR_LIT>']<EOL>receipt = recv['<STR_LIT>']<EOL>if processed_object in lclist:<EOL><INDENT>if processed_object not in done_objects:<EOL><INDENT>done_objects[processed_object] = [pfresult]<EOL><DEDENT>else:<EOL><INDENT>done_objects[processed_object].append(pfresult)<EOL><DEDENT>LOGINFO('<STR_LIT>' % (processed_object, pfresult))<EOL>if download_when_done:<EOL><INDENT>getobj = awsutils.s3_get_url(<EOL>pfresult,<EOL>client=s3_client<EOL>)<EOL>LOGINFO('<STR_LIT>' % (pfresult, getobj))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>awsutils.sqs_delete_item(outq_url, receipt)<EOL><DEDENT><DEDENT>except KeyboardInterrupt as e:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT><DEDENT>LOGINFO('<STR_LIT>')<EOL>time.sleep(<NUM_LIT:1.0>)<EOL>if purge_queues_when_done:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>sqs_client.purge_queue(QueueUrl=inq_url)<EOL>sqs_client.purge_queue(QueueUrl=outq_url)<EOL>time.sleep(<NUM_LIT>)<EOL><DEDENT>if delete_queues_when_done:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>awsutils.sqs_delete_queue(inq_url)<EOL>awsutils.sqs_delete_queue(outq_url)<EOL><DEDENT>work_state = {<EOL>'<STR_LIT>': done_objects,<EOL>'<STR_LIT>': list(set(lclist) - set(done_objects.keys())),<EOL>'<STR_LIT:args>':((os.path.abspath(lightcurve_list) if<EOL>isinstance(lightcurve_list, str) else lightcurve_list),<EOL>input_queue,<EOL>input_bucket,<EOL>result_queue,<EOL>result_bucket),<EOL>'<STR_LIT>':{'<STR_LIT>':pfmethods,<EOL>'<STR_LIT>':pfkwargs,<EOL>'<STR_LIT>':extra_runpf_kwargs,<EOL>'<STR_LIT>':process_list_slice,<EOL>'<STR_LIT>':purge_queues_when_done,<EOL>'<STR_LIT>':delete_queues_when_done,<EOL>'<STR_LIT>':download_when_done,<EOL>'<STR_LIT>':save_state_when_done}<EOL>}<EOL>if save_state_when_done:<EOL><INDENT>with open('<STR_LIT>','<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(work_state, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT><DEDENT>return work_state<EOL>

This queues up work for period-finders using SQS.

    Parameters
    ----------

    lightcurve_list : str or list of str
        This is either a string pointing to a file containing a list of light
        curves filenames to process or the list itself. The names must
        correspond to the full filenames of files stored on S3, including all
        prefixes, but not include the 's3://<bucket name>/' bit (these will be
        added automatically).

    input_queue : str
        This is the name of the SQS queue which will receive processing tasks
        generated by this function. The queue URL will automatically be obtained
        from AWS.

    input_bucket : str
        The name of the S3 bucket containing the light curve files to process.

    result_queue : str
        This is the name of the SQS queue that this function will listen to for
        messages from the workers as they complete processing on their input
        elements. This function will attempt to match input sent to the
        `input_queue` with results coming into the `result_queue` so it knows
        how many objects have been successfully processed. If this function
        receives task results that aren't in its own input queue, it will
        acknowledge them so they complete successfully, but not download them
        automatically. This handles leftover tasks completing from a previous
        run of this function.

    result_bucket : str
        The name of the S3 bucket which will receive the results from the
        workers.

    pfmethods : sequence of str
        This is a list of period-finder method short names as listed in the
        `lcproc.periodfinding.PFMETHODS` dict. This is used to tell the worker
        loop which period-finders to run on the input light curve.

    pfkwargs : sequence of dicts
        This contains optional kwargs as dicts to be supplied to all of the
        period-finder functions listed in `pfmethods`. This should be the same
        length as that sequence.

    extra_runpf_kwargs : dict
        This is a dict of kwargs to be supplied to `runpf` driver function
        itself.

    process_list_slice : list
        This is used to index into the input light curve list so a subset of the
        full list can be processed in this specific run of this function.

        Use None for a slice index elem to emulate single slice spec behavior:

        process_list_slice = [10, None]  -> lightcurve_list[10:]
        process_list_slice = [None, 500] -> lightcurve_list[:500]

    purge_queues_when_done : bool
        If this is True, and this function exits (either when all done, or when
        it is interrupted with a Ctrl+C), all outstanding elements in the
        input/output queues that have not yet been acknowledged by workers or by
        this function will be purged. This effectively cancels all outstanding
        work.

    delete_queues_when_done : bool
        If this is True, and this function exits (either when all done, or when
        it is interrupted with a Ctrl+C'), all outstanding work items will be
        purged from the input/queues and the queues themselves will be deleted.

    download_when_done : bool
        If this is True, the generated periodfinding result pickle for each
        input work item will be downloaded immediately to the current working
        directory when the worker functions report they're done with it.

    save_state_when_done : bool
        If this is True, will save the current state of the work item queue and
        the work items acknowledged as completed to a pickle in the current
        working directory. Call the `runcp_producer_loop_savedstate` function
        below to resume processing from this saved state later.

    s3_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its S3 download operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    sqs_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its SQS operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    Returns
    -------

    dict or str
        Returns the current work state as a dict or str path to the generated
        work state pickle depending on if `save_state_when_done` is True.


if not sqs_client:<EOL><INDENT>sqs_client = boto3.client('<STR_LIT>')<EOL><DEDENT>if not s3_client:<EOL><INDENT>s3_client = boto3.client('<STR_LIT>')<EOL><DEDENT>signal.signal(signal.SIGINT, kill_handler)<EOL>signal.signal(signal.SIGTERM, kill_handler)<EOL>shutdown_last_time = time.monotonic()<EOL>while True:<EOL><INDENT>curr_time = time.monotonic()<EOL>if (curr_time - shutdown_last_time) > shutdown_check_timer_seconds:<EOL><INDENT>shutdown_check = shutdown_check_handler()<EOL>if shutdown_check:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>shutdown_last_time = time.monotonic()<EOL><DEDENT>try:<EOL><INDENT>work = awsutils.sqs_get_item(in_queue_url,<EOL>client=sqs_client,<EOL>raiseonfail=True)<EOL>if work is not None and len(work) > <NUM_LIT:0>:<EOL><INDENT>recv = work[<NUM_LIT:0>]<EOL>action = recv['<STR_LIT>']['<STR_LIT:action>']<EOL>if action != '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>target = recv['<STR_LIT>']['<STR_LIT:target>']<EOL>args = recv['<STR_LIT>']['<STR_LIT:args>']<EOL>kwargs = recv['<STR_LIT>']['<STR_LIT>']<EOL>outbucket = recv['<STR_LIT>']['<STR_LIT>']<EOL>if '<STR_LIT>' in recv['<STR_LIT>']:<EOL><INDENT>out_queue_url = recv['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>out_queue_url = None<EOL><DEDENT>receipt = recv['<STR_LIT>']<EOL>try:<EOL><INDENT>lc_filename = awsutils.s3_get_url(<EOL>target,<EOL>altexts=lc_altexts,<EOL>client=s3_client<EOL>)<EOL>runpf_args = (lc_filename, args[<NUM_LIT:0>])<EOL>pfresult = runpf(<EOL>*runpf_args,<EOL>**kwargs<EOL>)<EOL>if pfresult and os.path.exists(pfresult):<EOL><INDENT>LOGINFO('<STR_LIT>' %<EOL>(lc_filename, pfresult))<EOL>resp = s3_client.list_objects_v2(<EOL>Bucket=outbucket,<EOL>MaxKeys=<NUM_LIT:1>,<EOL>Prefix=pfresult<EOL>)<EOL>outbucket_list = resp.get('<STR_LIT>',[])<EOL>if outbucket_list and len(outbucket_list) > <NUM_LIT:0>:<EOL><INDENT>LOGWARNING(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% target<EOL>)<EOL>awsutils.sqs_delete_item(in_queue_url, receipt)<EOL>continue<EOL><DEDENT>put_url = awsutils.s3_put_file(pfresult,<EOL>outbucket,<EOL>client=s3_client)<EOL>if put_url is not None:<EOL><INDENT>LOGINFO('<STR_LIT>' % put_url)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT:target>': target,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>os.remove(pfresult)<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>' % pfresult)<EOL>os.remove(pfresult)<EOL><DEDENT>awsutils.sqs_delete_item(in_queue_url, receipt)<EOL>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>' %<EOL>(lc_filename,))<EOL>with open('<STR_LIT>' %<EOL>lc_filename, '<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(<EOL>{'<STR_LIT>':in_queue_url,<EOL>'<STR_LIT:target>':target,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':kwargs,<EOL>'<STR_LIT>':outbucket,<EOL>'<STR_LIT>':out_queue_url},<EOL>outfd, pickle.HIGHEST_PROTOCOL<EOL>)<EOL><DEDENT>put_url = awsutils.s3_put_file(<EOL>'<STR_LIT>' % lc_filename,<EOL>outbucket,<EOL>client=s3_client<EOL>)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>awsutils.sqs_delete_item(in_queue_url,<EOL>receipt,<EOL>raiseonfail=True)<EOL>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT><DEDENT>except ClientError as e:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>')<EOL>if '<STR_LIT>' in locals():<EOL><INDENT>with open('<STR_LIT>' %<EOL>lc_filename,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(<EOL>{'<STR_LIT>':in_queue_url,<EOL>'<STR_LIT:target>':target,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':kwargs,<EOL>'<STR_LIT>':outbucket,<EOL>'<STR_LIT>':out_queue_url},<EOL>outfd, pickle.HIGHEST_PROTOCOL<EOL>)<EOL><DEDENT>put_url = awsutils.s3_put_file(<EOL>'<STR_LIT>' % lc_filename,<EOL>outbucket,<EOL>client=s3_client<EOL>)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT>':lc_filename,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT>awsutils.sqs_delete_item(in_queue_url,<EOL>receipt,<EOL>raiseonfail=True)<EOL><DEDENT><DEDENT><DEDENT>except KeyboardInterrupt:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>except ClientError as e:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>break<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>')<EOL>if '<STR_LIT>' in locals():<EOL><INDENT>with open('<STR_LIT>' %<EOL>lc_filename,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(<EOL>{'<STR_LIT>':in_queue_url,<EOL>'<STR_LIT:target>':target,<EOL>'<STR_LIT>':kwargs,<EOL>'<STR_LIT>':outbucket,<EOL>'<STR_LIT>':out_queue_url},<EOL>outfd, pickle.HIGHEST_PROTOCOL<EOL>)<EOL><DEDENT>put_url = awsutils.s3_put_file(<EOL>'<STR_LIT>' % lc_filename,<EOL>outbucket,<EOL>client=s3_client<EOL>)<EOL>if out_queue_url is not None:<EOL><INDENT>awsutils.sqs_put_item(<EOL>out_queue_url,<EOL>{'<STR_LIT>':put_url,<EOL>'<STR_LIT>':kwargs},<EOL>raiseonfail=True<EOL>)<EOL><DEDENT>if ( (lc_filename is not None) and<EOL>(os.path.exists(lc_filename)) ):<EOL><INDENT>os.remove(lc_filename)<EOL><DEDENT><DEDENT>awsutils.sqs_delete_item(in_queue_url, receipt, raiseonfail=True)<EOL><DEDENT><DEDENT>

This runs period-finding in a loop until interrupted.

    Consumes work task items from an input queue set up by `runpf_producer_loop`
    above.

    Parameters
    ----------

    in_queue_url : str
        The SQS URL of the input queue to listen to for work assignment
        messages. The task orders will include the input and output S3 bucket
        names, as well as the URL of the output queue to where this function
        will report its work-complete or work-failed status.

    workdir : str
        The directory on the local machine where this worker loop will download
        the input light curves, process them, and produce its output
        periodfinding result pickles. These will then be uploaded to the
        specified S3 output bucket, and then deleted from the local disk.

    lc_altexts : sequence of str
        If not None, this is a sequence of alternate extensions to try for the
        input light curve file other than the one provided in the input task
        order. For example, to get anything that's an .sqlite where .sqlite.gz
        is expected, use altexts=[''] to strip the .gz.

    wait_time_seconds : int
        The amount of time to wait in the input SQS queue for an input task
        order. If this timeout expires and no task has been received, this
        function goes back to the top of the work loop.

    shutdown_check_timer_seconds : float
        The amount of time to wait before checking for a pending EC2 shutdown
        message for the instance this worker loop is operating on. If a shutdown
        is noticed, the worker loop is cancelled in preparation for instance
        shutdown.

    sqs_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its SQS operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    s3_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its S3 operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    Returns
    -------

    Nothing.


try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if timecols is None:<EOL><INDENT>timecols = dtimecols<EOL><DEDENT>if magcols is None:<EOL><INDENT>magcols = dmagcols<EOL><DEDENT>if errcols is None:<EOL><INDENT>errcols = derrcols<EOL><DEDENT>lcdict = readerfunc(lcfile)<EOL>if ( (isinstance(lcdict, (list, tuple))) and<EOL>(isinstance(lcdict[<NUM_LIT:0>], dict)) ):<EOL><INDENT>lcdict = lcdict[<NUM_LIT:0>]<EOL><DEDENT>if '<STR_LIT>' in lcdict:<EOL><INDENT>LOGERROR('<STR_LIT>')<EOL>return None<EOL><DEDENT>lcdict['<STR_LIT>'] = {}<EOL>for tcol, mcol, ecol in zip(timecols, magcols, errcols):<EOL><INDENT>if '<STR_LIT:.>' in tcol:<EOL><INDENT>tcolget = tcol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>tcolget = [tcol]<EOL><DEDENT>times = _dict_get(lcdict, tcolget)<EOL>if '<STR_LIT:.>' in mcol:<EOL><INDENT>mcolget = mcol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>mcolget = [mcol]<EOL><DEDENT>mags = _dict_get(lcdict, mcolget)<EOL>if '<STR_LIT:.>' in ecol:<EOL><INDENT>ecolget = ecol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>ecolget = [ecol]<EOL><DEDENT>errs = _dict_get(lcdict, ecolget)<EOL>if normfunc is None:<EOL><INDENT>ntimes, nmags = normalize_magseries(<EOL>times, mags,<EOL>magsarefluxes=magsarefluxes<EOL>)<EOL>times, mags, errs = ntimes, nmags, errs<EOL><DEDENT>binned = time_bin_magseries_with_errs(times,<EOL>mags,<EOL>errs,<EOL>binsize=binsizesec,<EOL>minbinelems=minbinelems)<EOL>lcdict['<STR_LIT>'][mcol] = {'<STR_LIT>':binned['<STR_LIT>'],<EOL>'<STR_LIT>':binned['<STR_LIT>'],<EOL>'<STR_LIT>':binned['<STR_LIT>'],<EOL>'<STR_LIT>':binned['<STR_LIT>'],<EOL>'<STR_LIT>':binned['<STR_LIT>'],<EOL>'<STR_LIT>':binsizesec}<EOL><DEDENT>if outdir is None:<EOL><INDENT>outdir = os.path.dirname(lcfile)<EOL><DEDENT>outfile = os.path.join(outdir, '<STR_LIT>' %<EOL>(squeeze(lcdict['<STR_LIT>']).replace('<STR_LIT:U+0020>','<STR_LIT:->'),<EOL>binsizesec, lcformat))<EOL>with open(outfile, '<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(lcdict, outfd, protocol=pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>return outfile<EOL>

This bins the given light curve file in time using the specified bin size.

    Parameters
    ----------

    lcfile : str
        The file name to process.

    binsizesec : float
        The time bin-size in seconds.

    outdir : str or None
        If this is a str, the output LC will be written to `outdir`. If this is
        None, the output LC will be written to the same directory as `lcfile`.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curve file.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols,magcols,errcols : lists of str
        The keys in the lcdict produced by your light curve reader function that
        correspond to the times, mags/fluxes, and associated measurement errors
        that will be used as inputs to the binning process. If these are None,
        the default values for `timecols`, `magcols`, and `errcols` for your
        light curve format will be used here.

    minbinelems : int
        The minimum number of time-bin elements required to accept a time-bin as
        valid for the output binned light curve.

    Returns
    -------

    str
        The name of the output pickle file with the binned LC.

        Writes the output binned light curve to a pickle that contains the
        lcdict with an added `lcdict['binned'][magcol]` key, which contains the
        binned times, mags/fluxes, and errs as
        `lcdict['binned'][magcol]['times']`, `lcdict['binned'][magcol]['mags']`,
        and `lcdict['epd'][magcol]['errs']` for each `magcol` provided in the
        input or default `magcols` value for this light curve format.


lcfile, binsizesec, kwargs = task<EOL>try:<EOL><INDENT>binnedlc = timebinlc(lcfile, binsizesec, **kwargs)<EOL>LOGINFO('<STR_LIT>' %<EOL>(lcfile, binsizesec, binnedlc))<EOL>return binnedlc<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' % (lcfile,<EOL>binsizesec))<EOL>return None<EOL><DEDENT>

This is a parallel worker for the function below.

Parameters
----------

task : tuple
    This is of the form::

        task[0] = lcfile
        task[1] = binsizesec
        task[3] = {'outdir','lcformat','lcformatdir',
                   'timecols','magcols','errcols','minbinelems'}

Returns
-------

str
    The output pickle file with the binned LC if successful. None otherwise.


if outdir and not os.path.exists(outdir):<EOL><INDENT>os.mkdir(outdir)<EOL><DEDENT>if maxobjects is not None:<EOL><INDENT>lclist = lclist[:maxobjects]<EOL><DEDENT>tasks = [(x, binsizesec, {'<STR_LIT>':outdir,<EOL>'<STR_LIT>':lcformat,<EOL>'<STR_LIT>':lcformatdir,<EOL>'<STR_LIT>':timecols,<EOL>'<STR_LIT>':magcols,<EOL>'<STR_LIT>':errcols,<EOL>'<STR_LIT>':minbinelems}) for x in lclist]<EOL>pool = mp.Pool(nworkers, maxtasksperchild=maxworkertasks)<EOL>results = pool.map(timebinlc_worker, tasks)<EOL>pool.close()<EOL>pool.join()<EOL>resdict = {os.path.basename(x):y for (x,y) in zip(lclist, results)}<EOL>return resdict<EOL>

This time-bins all the LCs in the list using the specified bin size.

    Parameters
    ----------

    lclist : list of str
        The input LCs to process.

    binsizesec : float
        The time bin size to use in seconds.

    maxobjects : int or None
        If provided, LC processing will stop at `lclist[maxobjects]`.

    outdir : str or None
        The directory where output LCs will be written. If None, will write to
        the same directory as the input LCs.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curve file.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols,magcols,errcols : lists of str
        The keys in the lcdict produced by your light curve reader function that
        correspond to the times, mags/fluxes, and associated measurement errors
        that will be used as inputs to the binning process. If these are None,
        the default values for `timecols`, `magcols`, and `errcols` for your
        light curve format will be used here.

    minbinelems : int
        The minimum number of time-bin elements required to accept a time-bin as
        valid for the output binned light curve.

    nworkers : int
        Number of parallel workers to launch.

    maxworkertasks : int
        The maximum number of tasks a parallel worker will complete before being
        replaced to guard against memory leaks.

    Returns
    -------

    dict
        The returned dict contains keys = input LCs, vals = output LCs.


try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(fileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>lclist = sorted(glob.glob(os.path.join(lcdir, fileglob)))<EOL>return parallel_timebin(lclist,<EOL>binsizesec,<EOL>maxobjects=maxobjects,<EOL>outdir=outdir,<EOL>lcformat=lcformat,<EOL>timecols=timecols,<EOL>magcols=magcols,<EOL>errcols=errcols,<EOL>minbinelems=minbinelems,<EOL>nworkers=nworkers,<EOL>maxworkertasks=maxworkertasks)<EOL>

This time bins all the light curves in the specified directory.

Parameters
----------

lcdir : list of str
    Directory containing the input LCs to process.

binsizesec : float
    The time bin size to use in seconds.

maxobjects : int or None
    If provided, LC processing will stop at `lclist[maxobjects]`.

outdir : str or None
    The directory where output LCs will be written. If None, will write to
    the same directory as the input LCs.

lcformat : str
    This is the `formatkey` associated with your light curve format, which
    you previously passed in to the `lcproc.register_lcformat`
    function. This will be used to look up how to find and read the light
    curve file.

lcformatdir : str or None
    If this is provided, gives the path to a directory when you've stored
    your lcformat description JSONs, other than the usual directories lcproc
    knows to search for them in. Use this along with `lcformat` to specify
    an LC format JSON file that's not currently registered with lcproc.

timecols,magcols,errcols : lists of str
    The keys in the lcdict produced by your light curve reader function that
    correspond to the times, mags/fluxes, and associated measurement errors
    that will be used as inputs to the binning process. If these are None,
    the default values for `timecols`, `magcols`, and `errcols` for your
    light curve format will be used here.

minbinelems : int
    The minimum number of time-bin elements required to accept a time-bin as
    valid for the output binned light curve.

nworkers : int
    Number of parallel workers to launch.

maxworkertasks : int
    The maximum number of tasks a parallel worker will complete before being
    replaced to guard against memory leaks.

Returns
-------

dict
    The returned dict contains keys = input LCs, vals = output LCs.


try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if timecols is None:<EOL><INDENT>timecols = dtimecols<EOL><DEDENT>if magcols is None:<EOL><INDENT>magcols = dmagcols<EOL><DEDENT>if errcols is None:<EOL><INDENT>errcols = derrcols<EOL><DEDENT>try:<EOL><INDENT>lcdict = readerfunc(lcfile)<EOL>if ( (isinstance(lcdict, (list, tuple))) and<EOL>(isinstance(lcdict[<NUM_LIT:0>], dict)) ):<EOL><INDENT>lcdict = lcdict[<NUM_LIT:0>]<EOL><DEDENT>resultdict = {'<STR_LIT>':lcdict['<STR_LIT>'],<EOL>'<STR_LIT:info>':lcdict['<STR_LIT>'],<EOL>'<STR_LIT>':os.path.basename(lcfile)}<EOL>if normfunc is not None:<EOL><INDENT>lcdict = normfunc(lcdict)<EOL><DEDENT>for tcol, mcol, ecol in zip(timecols, magcols, errcols):<EOL><INDENT>if '<STR_LIT:.>' in tcol:<EOL><INDENT>tcolget = tcol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>tcolget = [tcol]<EOL><DEDENT>times = _dict_get(lcdict, tcolget)<EOL>if '<STR_LIT:.>' in mcol:<EOL><INDENT>mcolget = mcol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>mcolget = [mcol]<EOL><DEDENT>mags = _dict_get(lcdict, mcolget)<EOL>if '<STR_LIT:.>' in ecol:<EOL><INDENT>ecolget = ecol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>ecolget = [ecol]<EOL><DEDENT>errs = _dict_get(lcdict, ecolget)<EOL>if normfunc is None:<EOL><INDENT>ntimes, nmags = normalize_magseries(<EOL>times, mags,<EOL>magsarefluxes=magsarefluxes<EOL>)<EOL>times, mags, errs = ntimes, nmags, errs<EOL><DEDENT>finind = np.isfinite(times) & np.isfinite(mags) & np.isfinite(errs)<EOL>if mags[finind].size < mindet:<EOL><INDENT>LOGINFO('<STR_LIT>' %<EOL>(mags[finind].size, mcol, os.path.basename(lcfile)))<EOL>resultdict[mcol] = None<EOL><DEDENT>else:<EOL><INDENT>lcfeatures = varfeatures.all_nonperiodic_features(<EOL>times, mags, errs<EOL>)<EOL>resultdict[mcol] = lcfeatures<EOL><DEDENT><DEDENT>try:<EOL><INDENT>magmads = np.zeros(len(magcols))<EOL>for mind, mcol in enumerate(magcols):<EOL><INDENT>if '<STR_LIT:.>' in mcol:<EOL><INDENT>mcolget = mcol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>mcolget = [mcol]<EOL><DEDENT>magmads[mind] = resultdict[mcol]['<STR_LIT>']<EOL><DEDENT>bestmagcolind = np.where(magmads == np.min(magmads))[<NUM_LIT:0>]<EOL>resultdict['<STR_LIT>'] = magcols[bestmagcolind]<EOL><DEDENT>except Exception as e:<EOL><INDENT>resultdict['<STR_LIT>'] = None<EOL><DEDENT>outfile = os.path.join(outdir,<EOL>'<STR_LIT>' %<EOL>squeeze(resultdict['<STR_LIT>']).replace('<STR_LIT:U+0020>','<STR_LIT:->'))<EOL>with open(outfile, '<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(resultdict, outfd, protocol=<NUM_LIT:4>)<EOL><DEDENT>return outfile<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' %<EOL>(os.path.basename(lcfile), e))<EOL>return None<EOL><DEDENT>

This runs :py:func:`astrobase.varclass.varfeatures.all_nonperiodic_features`
    on a single LC file.

    Parameters
    ----------

    lcfile : str
        The input light curve to process.

    outfile : str
        The filename of the output variable features pickle that will be
        generated.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    mindet : int
        The minimum number of LC points required to generate variability
        features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    Returns
    -------

    str
        The generated variability features pickle for the input LC, with results
        for each magcol in the input `magcol` or light curve format's default
        `magcol` list.


try:<EOL><INDENT>(lcfile, outdir, timecols, magcols, errcols,<EOL>mindet, lcformat, lcformatdir) = task<EOL>return get_varfeatures(lcfile, outdir,<EOL>timecols=timecols,<EOL>magcols=magcols,<EOL>errcols=errcols,<EOL>mindet=mindet,<EOL>lcformat=lcformat,<EOL>lcformatdir=lcformatdir)<EOL><DEDENT>except Exception as e:<EOL><INDENT>return None<EOL><DEDENT>


if maxobjects:<EOL><INDENT>lclist = lclist[:maxobjects]<EOL><DEDENT>tasks = [(x, outdir, timecols, magcols, errcols,<EOL>mindet, lcformat, lcformatdir)<EOL>for x in lclist]<EOL>for task in tqdm(tasks):<EOL><INDENT>result = _varfeatures_worker(task)<EOL><DEDENT>return result<EOL>

This runs variability feature extraction for a list of LCs.

    Parameters
    ----------

    lclist : list of str
        The list of light curve file names to process.

    outdir : str
        The directory where the output varfeatures pickle files will be written.

    maxobjects : int
        The number of LCs to process from `lclist`.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    mindet : int
        The minimum number of LC points required to generate variability
        features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    Returns
    -------

    list of str
        List of the generated variability features pickles for the input LCs,
        with results for each magcol in the input `magcol` or light curve
        format's default `magcol` list.


<EOL>if not os.path.exists(outdir):<EOL><INDENT>os.makedirs(outdir)<EOL><DEDENT>if maxobjects:<EOL><INDENT>lclist = lclist[:maxobjects]<EOL><DEDENT>tasks = [(x, outdir, timecols, magcols, errcols, mindet,<EOL>lcformat, lcformatdir) for x in lclist]<EOL>with ProcessPoolExecutor(max_workers=nworkers) as executor:<EOL><INDENT>resultfutures = executor.map(varfeatures_worker, tasks)<EOL><DEDENT>results = [x for x in resultfutures]<EOL>resdict = {os.path.basename(x):y for (x,y) in zip(lclist, results)}<EOL>return resdict<EOL>

This runs variable feature extraction in parallel for all LCs in `lclist`.

    Parameters
    ----------

    lclist : list of str
        The list of light curve file names to process.

    outdir : str
        The directory where the output varfeatures pickle files will be written.

    maxobjects : int
        The number of LCs to process from `lclist`.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    mindet : int
        The minimum number of LC points required to generate variability
        features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    nworkers : int
        The number of parallel workers to launch.

    Returns
    -------

    dict
        A dict with key:val pairs of input LC file name : the generated
        variability features pickles for each of the input LCs, with results for
        each magcol in the input `magcol` or light curve format's default
        `magcol` list.


try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if not fileglob:<EOL><INDENT>fileglob = dfileglob<EOL><DEDENT>LOGINFO('<STR_LIT>' % (lcformat, lcdir))<EOL>if recursive is False:<EOL><INDENT>matching = glob.glob(os.path.join(lcdir, fileglob))<EOL><DEDENT>else:<EOL><INDENT>if sys.version_info[:<NUM_LIT:2>] > (<NUM_LIT:3>,<NUM_LIT:4>):<EOL><INDENT>matching = glob.glob(os.path.join(lcdir,<EOL>'<STR_LIT>',<EOL>fileglob),<EOL>recursive=True)<EOL><DEDENT>else:<EOL><INDENT>walker = os.walk(lcdir)<EOL>matching = []<EOL>for root, dirs, _files in walker:<EOL><INDENT>for sdir in dirs:<EOL><INDENT>searchpath = os.path.join(root,<EOL>sdir,<EOL>fileglob)<EOL>foundfiles = glob.glob(searchpath)<EOL>if foundfiles:<EOL><INDENT>matching.extend(foundfiles)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if matching and len(matching) > <NUM_LIT:0>:<EOL><INDENT>LOGINFO('<STR_LIT>' %<EOL>len(matching))<EOL>return parallel_varfeatures(matching,<EOL>outdir,<EOL>maxobjects=maxobjects,<EOL>timecols=timecols,<EOL>magcols=magcols,<EOL>errcols=errcols,<EOL>mindet=mindet,<EOL>lcformat=lcformat,<EOL>lcformatdir=lcformatdir,<EOL>nworkers=nworkers)<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>' % (lcformat,<EOL>lcdir))<EOL>return None<EOL><DEDENT>

This runs parallel variable feature extraction for a directory of LCs.

    Parameters
    ----------

    lcdir : str
        The directory of light curve files to process.

    outdir : str
        The directory where the output varfeatures pickle files will be written.

    fileglob : str or None
        The file glob to use when looking for light curve files in `lcdir`. If
        None, the default file glob associated for this LC format will be used.

    maxobjects : int
        The number of LCs to process from `lclist`.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    mindet : int
        The minimum number of LC points required to generate variability
        features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    nworkers : int
        The number of parallel workers to launch.

    Returns
    -------

    dict
        A dict with key:val pairs of input LC file name : the generated
        variability features pickles for each of the input LCs, with results for
        each magcol in the input `magcol` or light curve format's default
        `magcol` list.


try:<EOL><INDENT>(lcfile, lcformat, lcformatdir,<EOL>timecols, magcols, errcols,<EOL>custom_bandpasses) = task<EOL>try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if timecols is None:<EOL><INDENT>timecols = dtimecols<EOL><DEDENT>if magcols is None:<EOL><INDENT>magcols = dmagcols<EOL><DEDENT>if errcols is None:<EOL><INDENT>errcols = derrcols<EOL><DEDENT>lcdict = readerfunc(lcfile)<EOL>if ( (isinstance(lcdict, (list, tuple))) and<EOL>(isinstance(lcdict[<NUM_LIT:0>], dict)) ):<EOL><INDENT>lcdict = lcdict[<NUM_LIT:0>]<EOL><DEDENT>if '<STR_LIT>' in lcdict:<EOL><INDENT>objectid = lcdict['<STR_LIT>']<EOL><DEDENT>elif '<STR_LIT>' in lcdict and '<STR_LIT>' in lcdict['<STR_LIT>']:<EOL><INDENT>objectid = lcdict['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>elif '<STR_LIT>' in lcdict and '<STR_LIT>' in lcdict['<STR_LIT>']:<EOL><INDENT>objectid = lcdict['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>' % lcfile)<EOL>objectid = os.path.splitext(os.path.basename(lcfile))[<NUM_LIT:0>]<EOL><DEDENT>if '<STR_LIT>' in lcdict:<EOL><INDENT>colorfeat = starfeatures.color_features(<EOL>lcdict['<STR_LIT>'],<EOL>deredden=False,<EOL>custom_bandpasses=custom_bandpasses<EOL>)<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>lcfile)<EOL>return None<EOL><DEDENT>resultdict = {'<STR_LIT>':objectid,<EOL>'<STR_LIT>':lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>'<STR_LIT>':lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>'<STR_LIT>':colorfeat,<EOL>'<STR_LIT>':os.path.abspath(lcfile),<EOL>'<STR_LIT>':lcformat,<EOL>'<STR_LIT>':lcformatdir,<EOL>'<STR_LIT>':timecols,<EOL>'<STR_LIT>':magcols,<EOL>'<STR_LIT>':errcols}<EOL>for tcol, mcol, ecol in zip(timecols, magcols, errcols):<EOL><INDENT>try:<EOL><INDENT>if '<STR_LIT:.>' in tcol:<EOL><INDENT>tcolget = tcol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>tcolget = [tcol]<EOL><DEDENT>times = _dict_get(lcdict, tcolget)<EOL>if '<STR_LIT:.>' in mcol:<EOL><INDENT>mcolget = mcol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>mcolget = [mcol]<EOL><DEDENT>mags = _dict_get(lcdict, mcolget)<EOL>if '<STR_LIT:.>' in ecol:<EOL><INDENT>ecolget = ecol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>ecolget = [ecol]<EOL><DEDENT>errs = _dict_get(lcdict, ecolget)<EOL>if normfunc is None:<EOL><INDENT>ntimes, nmags = normalize_magseries(<EOL>times, mags,<EOL>magsarefluxes=magsarefluxes<EOL>)<EOL>times, mags, errs = ntimes, nmags, errs<EOL><DEDENT>varfeat = varfeatures.all_nonperiodic_features(<EOL>times, mags, errs<EOL>)<EOL>resultdict[mcol] = varfeat<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' %<EOL>(lcfile, mcol))<EOL>resultdict[mcol] = {'<STR_LIT>':<NUM_LIT:0>,<EOL>'<STR_LIT>':np.nan,<EOL>'<STR_LIT>':np.nan}<EOL><DEDENT><DEDENT>return resultdict<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' %<EOL>repr(task))<EOL>return None<EOL><DEDENT>

This is a parallel worker to gather LC stats.

task[0] = lcfile
task[1] = lcformat
task[2] = lcformatdir
task[3] = timecols
task[4] = magcols
task[5] = errcols
task[6] = custom_bandpasses


try:<EOL><INDENT>(lcfile, lcformat, lcformatdir,<EOL>tcol, mcol, ecol,<EOL>timebase, interpolate_type, sigclip) = task<EOL>try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>lcdict = readerfunc(lcfile)<EOL>if ( (isinstance(lcdict, (list, tuple))) and<EOL>(isinstance(lcdict[<NUM_LIT:0>], dict)) ):<EOL><INDENT>lcdict = lcdict[<NUM_LIT:0>]<EOL><DEDENT>outdict = {}<EOL>if '<STR_LIT:.>' in tcol:<EOL><INDENT>tcolget = tcol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>tcolget = [tcol]<EOL><DEDENT>times = _dict_get(lcdict, tcolget)<EOL>if '<STR_LIT:.>' in mcol:<EOL><INDENT>mcolget = mcol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>mcolget = [mcol]<EOL><DEDENT>mags = _dict_get(lcdict, mcolget)<EOL>if '<STR_LIT:.>' in ecol:<EOL><INDENT>ecolget = ecol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>ecolget = [ecol]<EOL><DEDENT>errs = _dict_get(lcdict, ecolget)<EOL>if normfunc is None:<EOL><INDENT>ntimes, nmags = normalize_magseries(<EOL>times, mags,<EOL>magsarefluxes=magsarefluxes<EOL>)<EOL><DEDENT>times, mags, errs = ntimes, nmags, errs<EOL>stimes, smags, serrs = sigclip_magseries(times,<EOL>mags,<EOL>errs,<EOL>sigclip=sigclip)<EOL>mags_interpolator = spi.interp1d(stimes, smags,<EOL>kind=interpolate_type,<EOL>fill_value='<STR_LIT>')<EOL>errs_interpolator = spi.interp1d(stimes, serrs,<EOL>kind=interpolate_type,<EOL>fill_value='<STR_LIT>')<EOL>interpolated_mags = mags_interpolator(timebase)<EOL>interpolated_errs = errs_interpolator(timebase)<EOL>magmedian = np.median(interpolated_mags)<EOL>renormed_mags = interpolated_mags - magmedian<EOL>outdict = {'<STR_LIT>':renormed_mags,<EOL>'<STR_LIT>':interpolated_errs,<EOL>'<STR_LIT>':interpolated_mags}<EOL>return outdict<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' % repr(task))<EOL>return None<EOL><DEDENT>

This is a parallel worker that reforms light curves for TFA.

task[0] = lcfile
task[1] = lcformat
task[2] = lcformatdir
task[3] = timecol
task[4] = magcol
task[5] = errcol
task[6] = timebase
task[7] = interpolate_type
task[8] = sigclip