try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if timecols is None:<EOL><INDENT>timecols = dtimecols<EOL><DEDENT>if magcols is None:<EOL><INDENT>magcols = dmagcols<EOL><DEDENT>if errcols is None:<EOL><INDENT>errcols = derrcols<EOL><DEDENT>LOGINFO('<STR_LIT>' %<EOL>len(lclist))<EOL>if lcinfo_pkl and os.path.exists(lcinfo_pkl):<EOL><INDENT>with open(lcinfo_pkl,'<STR_LIT:rb>') as infd:<EOL><INDENT>results = pickle.load(infd)<EOL><DEDENT><DEDENT>elif ((not outfile) and<EOL>os.path.exists('<STR_LIT>' % lcformat)):<EOL><INDENT>with open('<STR_LIT>' % lcformat, '<STR_LIT:rb>') as infd:<EOL><INDENT>results = pickle.load(infd)<EOL><DEDENT><DEDENT>elif (outfile and os.path.exists('<STR_LIT>' %<EOL>(lcformat, os.path.basename(outfile)))):<EOL><INDENT>with open(<EOL>'<STR_LIT>' %<EOL>(lcformat, os.path.basename(outfile)),<EOL>'<STR_LIT:rb>'<EOL>) as infd:<EOL><INDENT>results = pickle.load(infd)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>tasks = [(x, lcformat, lcformat,<EOL>timecols, magcols, errcols,<EOL>custom_bandpasses) for x in lclist]<EOL>pool = mp.Pool(nworkers, maxtasksperchild=maxworkertasks)<EOL>results = pool.map(_collect_tfa_stats, tasks)<EOL>pool.close()<EOL>pool.join()<EOL>if not outfile:<EOL><INDENT>with open('<STR_LIT>' % lcformat,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(results, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>with open(<EOL>'<STR_LIT>' %<EOL>(lcformat, os.path.basename(outfile)),<EOL>'<STR_LIT:wb>'<EOL>) as outfd:<EOL><INDENT>pickle.dump(results, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT><DEDENT><DEDENT>all_ras = np.array([res['<STR_LIT>'] for res in results])<EOL>all_decls = np.array([res['<STR_LIT>'] for res in results])<EOL>center_ra = np.nanmedian(all_ras)<EOL>center_decl = np.nanmedian(all_decls)<EOL>outdict = {<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':center_ra,<EOL>'<STR_LIT>':center_decl,<EOL>}<EOL>for tcol, mcol, ecol in zip(timecols, magcols, errcols):<EOL><INDENT>if '<STR_LIT:.>' in tcol:<EOL><INDENT>tcolget = tcol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>tcolget = [tcol]<EOL><DEDENT>(lcmag, lcmad, lceta,<EOL>lcndet, lcobj, lcfpaths,<EOL>lcra, lcdecl) = [], [], [], [], [], [], [], []<EOL>outdict['<STR_LIT>'].append(tcol)<EOL>outdict['<STR_LIT>'].append(mcol)<EOL>outdict['<STR_LIT>'].append(ecol)<EOL>outdict[mcol] = {'<STR_LIT>':{'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[]}}<EOL>LOGINFO('<STR_LIT>' %<EOL>mcol)<EOL>for result in results:<EOL><INDENT>try:<EOL><INDENT>thismag = result['<STR_LIT>'][mag_bandpass]<EOL>thismad = result[mcol]['<STR_LIT>']<EOL>thiseta = result[mcol]['<STR_LIT>']<EOL>thisndet = result[mcol]['<STR_LIT>']<EOL>thisobj = result['<STR_LIT>']<EOL>thislcf = result['<STR_LIT>']<EOL>thisra = result['<STR_LIT>']<EOL>thisdecl = result['<STR_LIT>']<EOL>outdict[mcol]['<STR_LIT>']['<STR_LIT>'].append(thismag)<EOL>outdict[mcol]['<STR_LIT>']['<STR_LIT>'].append(thismad)<EOL>outdict[mcol]['<STR_LIT>']['<STR_LIT>'].append(thiseta)<EOL>outdict[mcol]['<STR_LIT>']['<STR_LIT>'].append(thisndet)<EOL>outdict[mcol]['<STR_LIT>']['<STR_LIT>'].append(thisobj)<EOL>outdict[mcol]['<STR_LIT>']['<STR_LIT>'].append(thislcf)<EOL>outdict[mcol]['<STR_LIT>']['<STR_LIT>'].append(thisra)<EOL>outdict[mcol]['<STR_LIT>']['<STR_LIT>'].append(thisdecl)<EOL>if isinstance(mag_bright_limit, (list, tuple)):<EOL><INDENT>use_bright_maglim = mag_bright_limit[<EOL>magcols.index(mcol)<EOL>]<EOL><DEDENT>else:<EOL><INDENT>use_bright_maglim = mag_bright_limit<EOL><DEDENT>if isinstance(mag_faint_limit, (list, tuple)):<EOL><INDENT>use_faint_maglim = mag_faint_limit[<EOL>magcols.index(mcol)<EOL>]<EOL><DEDENT>else:<EOL><INDENT>use_faint_maglim = mag_faint_limit<EOL><DEDENT>if ((use_bright_maglim < thismag < use_faint_maglim) and<EOL>(<NUM_LIT>*thismad < max_rms)):<EOL><INDENT>lcmag.append(thismag)<EOL>lcmad.append(thismad)<EOL>lceta.append(thiseta)<EOL>lcndet.append(thisndet)<EOL>lcobj.append(thisobj)<EOL>lcfpaths.append(thislcf)<EOL>lcra.append(thisra)<EOL>lcdecl.append(thisdecl)<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>if len(lcobj) >= min_template_number:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(mcol, len(lcobj),<EOL>mag_bright_limit, mag_faint_limit, max_rms))<EOL>lcmag = np.array(lcmag)<EOL>lcmad = np.array(lcmad)<EOL>lceta = np.array(lceta)<EOL>lcndet = np.array(lcndet)<EOL>lcobj = np.array(lcobj)<EOL>lcfpaths = np.array(lcfpaths)<EOL>lcra = np.array(lcra)<EOL>lcdecl = np.array(lcdecl)<EOL>sortind = np.argsort(lcmag)<EOL>lcmag = lcmag[sortind]<EOL>lcmad = lcmad[sortind]<EOL>lceta = lceta[sortind]<EOL>lcndet = lcndet[sortind]<EOL>lcobj = lcobj[sortind]<EOL>lcfpaths = lcfpaths[sortind]<EOL>lcra = lcra[sortind]<EOL>lcdecl = lcdecl[sortind]<EOL>splfit_ind = np.diff(lcmag) > <NUM_LIT:0.0><EOL>splfit_ind = np.concatenate((np.array([True]), splfit_ind))<EOL>fit_lcmag = lcmag[splfit_ind]<EOL>fit_lcmad = lcmad[splfit_ind]<EOL>fit_lceta = lceta[splfit_ind]<EOL>magmadfit = np.poly1d(np.polyfit(<EOL>fit_lcmag,<EOL>fit_lcmad,<EOL><NUM_LIT:2><EOL>))<EOL>magmadind = lcmad/magmadfit(lcmag) < max_mult_above_magmad<EOL>magetafit = np.poly1d(np.polyfit(<EOL>fit_lcmag,<EOL>fit_lceta,<EOL><NUM_LIT:2><EOL>))<EOL>magetaind = magetafit(lcmag)/lceta < max_mult_above_mageta<EOL>median_ndet = np.median(lcndet)<EOL>ndetind = lcndet >= median_ndet<EOL>templateind = magmadind & magetaind & ndetind<EOL>if templateind.sum() >= min_template_number:<EOL><INDENT>LOGINFO('<STR_LIT>' %<EOL>(mcol, templateind.sum()))<EOL>templatemag = lcmag[templateind]<EOL>templatemad = lcmad[templateind]<EOL>templateeta = lceta[templateind]<EOL>templatendet = lcndet[templateind]<EOL>templateobj = lcobj[templateind]<EOL>templatelcf = lcfpaths[templateind]<EOL>templatera = lcra[templateind]<EOL>templatedecl = lcdecl[templateind]<EOL>target_number_templates = int(target_template_frac*len(results))<EOL>if target_number_templates > max_template_number:<EOL><INDENT>target_number_templates = max_template_number<EOL><DEDENT>LOGINFO('<STR_LIT>' %<EOL>(mcol, target_number_templates))<EOL>template_cxi, template_ceta = coordutils.xieta_from_radecl(<EOL>templatera,<EOL>templatedecl,<EOL>center_ra,<EOL>center_decl<EOL>)<EOL>cxi_bins = np.linspace(template_cxi.min(),<EOL>template_cxi.max(),<EOL>num=xieta_bins)<EOL>ceta_bins = np.linspace(template_ceta.min(),<EOL>template_ceta.max(),<EOL>num=xieta_bins)<EOL>digitized_cxi_inds = np.digitize(template_cxi, cxi_bins)<EOL>digitized_ceta_inds = np.digitize(template_ceta, ceta_bins)<EOL>targetind = npr.choice(xieta_bins,<EOL>target_number_templates,<EOL>replace=True)<EOL>selected_template_obj = []<EOL>selected_template_lcf = []<EOL>selected_template_ndet = []<EOL>selected_template_ra = []<EOL>selected_template_decl = []<EOL>selected_template_mag = []<EOL>selected_template_mad = []<EOL>selected_template_eta = []<EOL>for ind in targetind:<EOL><INDENT>pass<EOL><DEDENT>targetind = npr.choice(templateobj.size,<EOL>target_number_templates,<EOL>replace=False)<EOL>templatemag = templatemag[targetind]<EOL>templatemad = templatemad[targetind]<EOL>templateeta = templateeta[targetind]<EOL>templatendet = templatendet[targetind]<EOL>templateobj = templateobj[targetind]<EOL>templatelcf = templatelcf[targetind]<EOL>templatera = templatera[targetind]<EOL>templatedecl = templatedecl[targetind]<EOL>maxndetind = templatendet == templatendet.max()<EOL>timebaselcf = templatelcf[maxndetind][<NUM_LIT:0>]<EOL>timebasendet = templatendet[maxndetind][<NUM_LIT:0>]<EOL>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(mcol, timebaselcf, timebasendet))<EOL>if process_template_lcs:<EOL><INDENT>timebaselcdict = readerfunc(timebaselcf)<EOL>if ( (isinstance(timebaselcdict, (list, tuple))) and<EOL>(isinstance(timebaselcdict[<NUM_LIT:0>], dict)) ):<EOL><INDENT>timebaselcdict = timebaselcdict[<NUM_LIT:0>]<EOL><DEDENT>timebase = _dict_get(timebaselcdict, tcolget)<EOL><DEDENT>else:<EOL><INDENT>timebase = None<EOL><DEDENT>if target_number_templates > timebasendet:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(target_number_templates,<EOL>timebasendet,<EOL>max_target_frac_obs))<EOL>newmaxtemplates = int(max_target_frac_obs*timebasendet)<EOL>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(mcol, newmaxtemplates))<EOL>targetind = npr.choice(templateobj.size,<EOL>newmaxtemplates,<EOL>replace=False)<EOL>templatemag = templatemag[targetind]<EOL>templatemad = templatemad[targetind]<EOL>templateeta = templateeta[targetind]<EOL>templatendet = templatendet[targetind]<EOL>templateobj = templateobj[targetind]<EOL>templatelcf = templatelcf[targetind]<EOL>templatera = templatera[targetind]<EOL>templatedecl = templatedecl[targetind]<EOL>maxndetind = templatendet == templatendet.max()<EOL>timebaselcf = templatelcf[maxndetind][<NUM_LIT:0>]<EOL>timebasendet = templatendet[maxndetind][<NUM_LIT:0>]<EOL>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(mcol, timebaselcf, timebasendet))<EOL>if process_template_lcs:<EOL><INDENT>timebaselcdict = readerfunc(timebaselcf)<EOL>if ( (isinstance(timebaselcdict, (list, tuple))) and<EOL>(isinstance(timebaselcdict[<NUM_LIT:0>], dict)) ):<EOL><INDENT>timebaselcdict = timebaselcdict[<NUM_LIT:0>]<EOL><DEDENT>timebase = _dict_get(timebaselcdict, tcolget)<EOL><DEDENT>else:<EOL><INDENT>timebase = None<EOL><DEDENT><DEDENT>if process_template_lcs:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>' % mcol)<EOL>tasks = [(x, lcformat, lcformatdir,<EOL>tcol, mcol, ecol,<EOL>timebase, template_interpolate,<EOL>template_sigclip) for x<EOL>in templatelcf]<EOL>pool = mp.Pool(nworkers, maxtasksperchild=maxworkertasks)<EOL>reform_results = pool.map(_reform_templatelc_for_tfa, tasks)<EOL>pool.close()<EOL>pool.join()<EOL>template_magseries = np.array([x['<STR_LIT>']<EOL>for x in reform_results])<EOL>template_errseries = np.array([x['<STR_LIT>']<EOL>for x in reform_results])<EOL><DEDENT>else:<EOL><INDENT>template_magseries = None<EOL>template_errseries = None<EOL><DEDENT>outdict[mcol].update({<EOL>'<STR_LIT>':timebaselcf,<EOL>'<STR_LIT>':timebase,<EOL>'<STR_LIT>':{'<STR_LIT>':magmadfit,<EOL>'<STR_LIT>':magetafit},<EOL>'<STR_LIT>':templateobj,<EOL>'<STR_LIT>':templatera,<EOL>'<STR_LIT>':templatedecl,<EOL>'<STR_LIT>':templatemag,<EOL>'<STR_LIT>':templatemad,<EOL>'<STR_LIT>':templateeta,<EOL>'<STR_LIT>':templatendet,<EOL>'<STR_LIT>':template_magseries,<EOL>'<STR_LIT>':template_errseries<EOL>})<EOL>outdict[mcol]['<STR_LIT>'] = (<EOL>coordutils.make_kdtree(<EOL>templatera, templatedecl<EOL>)<EOL>)<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % mcol)<EOL>continue<EOL><DEDENT><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>' % (len(lcobj),mcol))<EOL>continue<EOL><DEDENT>plt.plot(lcmag, lcmad, marker='<STR_LIT:o>', linestyle='<STR_LIT:none>', ms=<NUM_LIT:1.0>)<EOL>modelmags = np.linspace(lcmag.min(), lcmag.max(), num=<NUM_LIT:1000>)<EOL>plt.plot(modelmags, outdict[mcol]['<STR_LIT>']['<STR_LIT>'](modelmags))<EOL>plt.yscale('<STR_LIT>')<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL>plt.savefig('<STR_LIT>' % mcol,<EOL>bbox_inches='<STR_LIT>')<EOL>plt.close('<STR_LIT:all>')<EOL>plt.plot(lcmag, lceta, marker='<STR_LIT:o>', linestyle='<STR_LIT:none>', ms=<NUM_LIT:1.0>)<EOL>modelmags = np.linspace(lcmag.min(), lcmag.max(), num=<NUM_LIT:1000>)<EOL>plt.plot(modelmags, outdict[mcol]['<STR_LIT>']['<STR_LIT>'](modelmags))<EOL>plt.yscale('<STR_LIT>')<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL>plt.savefig('<STR_LIT>' % mcol,<EOL>bbox_inches='<STR_LIT>')<EOL>plt.close('<STR_LIT:all>')<EOL><DEDENT>if outfile:<EOL><INDENT>if outfile.endswith('<STR_LIT>'):<EOL><INDENT>outfd = gzip.open(outfile,'<STR_LIT:wb>')<EOL><DEDENT>else:<EOL><INDENT>outfd = open(outfile,'<STR_LIT:wb>')<EOL><DEDENT>with outfd:<EOL><INDENT>pickle.dump(outdict, outfd, protocol=pickle.HIGHEST_PROTOCOL)<EOL><DEDENT><DEDENT>return outdict<EOL>

This selects template objects for TFA.

    Selection criteria for TFA template ensemble objects:

    - not variable: use a poly fit to the mag-MAD relation and eta-normal
      variability index to get nonvar objects

    - not more than 10% of the total number of objects in the field or
      `max_tfa_templates` at most

    - allow shuffling of the templates if the target ends up in them

    - nothing with less than the median number of observations in the field

    - sigma-clip the input time series observations

    - TODO: uniform sampling in tangent plane coordinates (we'll need ra and
      decl)

    This also determines the effective cadence that all TFA LCs will be binned
    to as the template LC with the largest number of non-nan observations will
    be used. All template LCs will be renormed to zero.

    Parameters
    ----------

    lclist : list of str
        This is a list of light curves to use as input to generate the template
        set.

    lcinfo_pkl : str or None
        If provided, is a file path to a pickle file created by this function on
        a previous run containing the LC information. This will be loaded
        directly instead of having to re-run LC info collection.

    outfile : str or None
        This is the pickle filename to which the TFA template list will be
        written to. If None, a default file name will be used for this.

    target_template_frac : float
        This is the fraction of total objects in lclist to use for the number of
        templates.

    max_target_frac_obs : float
        This sets the number of templates to generate if the number of
        observations for the light curves is smaller than the number of objects
        in the collection. The number of templates will be set to this fraction
        of the number of observations if this is the case.

    min_template_number : int
        This is the minimum number of templates to generate.

    max_template_number : int
        This is the maximum number of templates to generate. If
        `target_template_frac` times the number of objects is greater than
        `max_template_number`, only `max_template_number` templates will be
        used.

    max_rms : float
        This is the maximum light curve RMS for an object to consider it as a
        possible template ensemble member.

    max_mult_above_magmad : float
        This is the maximum multiplier above the mag-RMS fit to consider an
        object as variable and thus not part of the template ensemble.

    max_mult_above_mageta : float
        This is the maximum multiplier above the mag-eta (variable index) fit to
        consider an object as variable and thus not part of the template
        ensemble.

    mag_bandpass : str
        This sets the key in the light curve dict's objectinfo dict to use as
        the canonical magnitude for the object and apply any magnitude limits
        to.

    custom_bandpasses : dict or None
        This can be used to provide any custom band name keys to the star
        feature collection function.

    mag_bright_limit : float or list of floats
        This sets the brightest mag (in the `mag_bandpass` filter) for a
        potential member of the TFA template ensemble. If this is a single
        float, the value will be used for all magcols. If this is a list of
        floats with len = len(magcols), the specific bright limits will be used
        for each magcol individually.

    mag_faint_limit : float or list of floats
        This sets the faintest mag (in the `mag_bandpass` filter) for a
        potential member of the TFA template ensemble. If this is a single
        float, the value will be used for all magcols. If this is a list of
        floats with len = len(magcols), the specific faint limits will be used
        for each magcol individually.

    process_template_lcs : bool
        If True, will reform the template light curves to the chosen
        time-base. If False, will only select light curves for templates but not
        process them. This is useful for initial exploration of how the template
        LC are selected.

    template_sigclip : float or sequence of floats or None
        This sets the sigma-clip to be applied to the template light curves.

    template_interpolate : str
        This sets the kwarg to pass to `scipy.interpolate.interp1d` to set the
        kind of interpolation to use when reforming light curves to the TFA
        template timebase.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    nworkers : int
        The number of parallel workers to launch.

    maxworkertasks : int
        The maximum number of tasks to run per worker before it is replaced by a
        fresh one.

    Returns
    -------

    dict
        This function returns a dict that can be passed directly to
        `apply_tfa_magseries` below. It can optionally produce a pickle with the
        same dict, which can also be passed to that function.


try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if isinstance(templateinfo,str) and os.path.exists(templateinfo):<EOL><INDENT>with open(templateinfo,'<STR_LIT:rb>') as infd:<EOL><INDENT>templateinfo = pickle.load(infd)<EOL><DEDENT><DEDENT>lcdict = readerfunc(lcfile)<EOL>if ((isinstance(lcdict, (tuple, list))) and<EOL>isinstance(lcdict[<NUM_LIT:0>], dict)):<EOL><INDENT>lcdict = lcdict[<NUM_LIT:0>]<EOL><DEDENT>objectid = lcdict['<STR_LIT>']<EOL>tmagseries = templateinfo[magcol][<EOL>'<STR_LIT>'<EOL>][::]<EOL>if objectid in templateinfo[magcol]['<STR_LIT>']:<EOL><INDENT>LOGWARNING('<STR_LIT>' %<EOL>objectid)<EOL>templateind = templateinfo[magcol]['<STR_LIT>'] == objectid<EOL>tmagseries = tmagseries[~templateind,:]<EOL><DEDENT>object_matches = coordutils.conesearch_kdtree(<EOL>templateinfo[magcol]['<STR_LIT>'],<EOL>lcdict['<STR_LIT>']['<STR_LIT>'], lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>mintemplatedist_arcmin/<NUM_LIT><EOL>)<EOL>if len(object_matches) > <NUM_LIT:0>:<EOL><INDENT>LOGWARNING(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" %<EOL>(objectid, mintemplatedist_arcmin, len(object_matches))<EOL>)<EOL>removalind = np.full(<EOL>templateinfo[magcol]['<STR_LIT>'].size,<EOL>False, dtype=np.bool<EOL>)<EOL>removalind[np.array(object_matches)] = True<EOL>tmagseries = tmagseries[~removalind,:]<EOL><DEDENT>normal_matrix = np.dot(tmagseries, tmagseries.T)<EOL>normal_matrix_inverse = spla.pinv2(normal_matrix)<EOL>timebase = templateinfo[magcol]['<STR_LIT>']<EOL>reformed_targetlc = _reform_templatelc_for_tfa((<EOL>lcfile,<EOL>lcformat,<EOL>lcformatdir,<EOL>timecol,<EOL>magcol,<EOL>errcol,<EOL>timebase,<EOL>interp,<EOL>sigclip<EOL>))<EOL>scalar_products = np.dot(tmagseries, reformed_targetlc['<STR_LIT>'])<EOL>corrections = np.dot(normal_matrix_inverse, scalar_products)<EOL>corrected_magseries = (<EOL>reformed_targetlc['<STR_LIT>'] -<EOL>np.dot(tmagseries.T, corrections)<EOL>)<EOL>outdict = {<EOL>'<STR_LIT>':timebase,<EOL>'<STR_LIT>':corrected_magseries,<EOL>'<STR_LIT>':reformed_targetlc['<STR_LIT>'],<EOL>'<STR_LIT>':np.median(corrected_magseries),<EOL>'<STR_LIT>': np.median(np.abs(corrected_magseries -<EOL>np.median(corrected_magseries))),<EOL>'<STR_LIT>':{'<STR_LIT>':tmagseries,<EOL>'<STR_LIT>':normal_matrix,<EOL>'<STR_LIT>':normal_matrix_inverse,<EOL>'<STR_LIT>':scalar_products,<EOL>'<STR_LIT>':corrections,<EOL>'<STR_LIT>':reformed_targetlc},<EOL>}<EOL>lcdict['<STR_LIT>'] = outdict<EOL>outfile = os.path.join(<EOL>os.path.dirname(lcfile),<EOL>'<STR_LIT>' % (<EOL>squeeze(objectid).replace('<STR_LIT:U+0020>','<STR_LIT:->'),<EOL>magcol<EOL>)<EOL>)<EOL>with open(outfile,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(lcdict, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>return outfile<EOL>

This applies the TFA correction to an LC given TFA template information.

    Parameters
    ----------

    lcfile : str
        This is the light curve file to apply the TFA correction to.

    timecol,magcol,errcol : str
        These are the column keys in the lcdict for the LC file to apply the TFA
        correction to.

    templateinfo : dict or str
        This is either the dict produced by `tfa_templates_lclist` or the pickle
        produced by the same function.

    mintemplatedist_arcmin : float
        This sets the minimum distance required from the target object for
        objects in the TFA template ensemble. Objects closer than this distance
        will be removed from the ensemble.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    interp : str
        This is passed to scipy.interpolate.interp1d as the kind of
        interpolation to use when reforming this light curve to the timebase of
        the TFA templates.

    sigclip : float or sequence of two floats or None
        This is the sigma clip to apply to this light curve before running TFA
        on it.

    Returns
    -------

    str
        This returns the filename of the light curve file generated after TFA
        applications. This is a pickle (that can be read by `lcproc.read_pklc`)
        in the same directory as `lcfile`. The `magcol` will be encoded in the
        filename, so each `magcol` in `lcfile` gets its own output file.


(lcfile, timecol, magcol, errcol,<EOL>templateinfo, lcformat, lcformatdir,<EOL>interp, sigclip, mintemplatedist_arcmin) = task<EOL>try:<EOL><INDENT>res = apply_tfa_magseries(<EOL>lcfile, timecol, magcol, errcol,<EOL>templateinfo,<EOL>lcformat=lcformat,<EOL>lcformatdir=lcformatdir,<EOL>interp=interp,<EOL>sigclip=sigclip,<EOL>mintemplatedist_arcmin=mintemplatedist_arcmin<EOL>)<EOL>if res:<EOL><INDENT>LOGINFO('<STR_LIT>' % (lcfile, res))<EOL><DEDENT>return res<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' % lcfile)<EOL>return None<EOL><DEDENT>

This is a parallel worker for the function below.

task[0] = lcfile
task[1] = timecol
task[2] = magcol
task[3] = errcol
task[4] = templateinfo
task[5] = lcformat
task[6] = lcformatdir
task[6] = interp
task[7] = sigclip


<EOL>if isinstance(templateinfo,str) and os.path.exists(templateinfo):<EOL><INDENT>with open(templateinfo,'<STR_LIT:rb>') as infd:<EOL><INDENT>templateinfo = pickle.load(infd)<EOL><DEDENT><DEDENT>try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if timecols is None:<EOL><INDENT>timecols = templateinfo['<STR_LIT>']<EOL><DEDENT>if magcols is None:<EOL><INDENT>magcols = templateinfo['<STR_LIT>']<EOL><DEDENT>if errcols is None:<EOL><INDENT>errcols = templateinfo['<STR_LIT>']<EOL><DEDENT>outdict = {}<EOL>for t, m, e in zip(timecols, magcols, errcols):<EOL><INDENT>tasks = [(x, t, m, e, templateinfo,<EOL>lcformat, lcformatdir,<EOL>interp, sigclip) for<EOL>x in lclist]<EOL>pool = mp.Pool(nworkers, maxtasksperchild=maxworkertasks)<EOL>results = pool.map(_parallel_tfa_worker, tasks)<EOL>pool.close()<EOL>pool.join()<EOL>outdict[m] = results<EOL><DEDENT>return outdict<EOL>

This applies TFA in parallel to all LCs in the given list of file names.

    Parameters
    ----------

    lclist : str
        This is a list of light curve files to apply TFA correction to.

    templateinfo : dict or str
        This is either the dict produced by `tfa_templates_lclist` or the pickle
        produced by the same function.

    timecols : list of str or None
        The timecol keys to use from the lcdict in applying TFA corrections.

    magcols : list of str or None
        The magcol keys to use from the lcdict in applying TFA corrections.

    errcols : list of str or None
        The errcol keys to use from the lcdict in applying TFA corrections.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    interp : str
        This is passed to scipy.interpolate.interp1d as the kind of
        interpolation to use when reforming the light curves to the timebase of
        the TFA templates.

    sigclip : float or sequence of two floats or None
        This is the sigma clip to apply to the light curves before running TFA
        on it.

    mintemplatedist_arcmin : float
        This sets the minimum distance required from the target object for
        objects in the TFA template ensemble. Objects closer than this distance
        will be removed from the ensemble.

    nworkers : int
        The number of parallel workers to launch

    maxworkertasks : int
        The maximum number of tasks per worker allowed before it's replaced by a
        fresh one.

    Returns
    -------

    dict
        Contains the input file names and output TFA light curve filenames per
        input file organized by each `magcol` in `magcols`.


<EOL>if isinstance(templateinfo,str) and os.path.exists(templateinfo):<EOL><INDENT>with open(templateinfo,'<STR_LIT:rb>') as infd:<EOL><INDENT>templateinfo = pickle.load(infd)<EOL><DEDENT><DEDENT>try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if lcfileglob is None:<EOL><INDENT>lcfileglob = dfileglob<EOL><DEDENT>lclist = sorted(glob.glob(os.path.join(lcdir, lcfileglob)))<EOL>return parallel_tfa_lclist(<EOL>lclist,<EOL>templateinfo,<EOL>timecols=timecols,<EOL>magcols=magcols,<EOL>errcols=errcols,<EOL>lcformat=lcformat,<EOL>lcformatdir=None,<EOL>interp=interp,<EOL>sigclip=sigclip,<EOL>mintemplatedist_arcmin=mintemplatedist_arcmin,<EOL>nworkers=nworkers,<EOL>maxworkertasks=maxworkertasks<EOL>)<EOL>

This applies TFA in parallel to all LCs in a directory.

    Parameters
    ----------

    lcdir : str
        This is the directory containing the light curve files to process..

    templateinfo : dict or str
        This is either the dict produced by `tfa_templates_lclist` or the pickle
        produced by the same function.

    lcfileglob : str or None
        The UNIX file glob to use when searching for light curve files in
        `lcdir`. If None, the default file glob associated with registered LC
        format provided is used.

    timecols : list of str or None
        The timecol keys to use from the lcdict in applying TFA corrections.

    magcols : list of str or None
        The magcol keys to use from the lcdict in applying TFA corrections.

    errcols : list of str or None
        The errcol keys to use from the lcdict in applying TFA corrections.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    interp : str
        This is passed to scipy.interpolate.interp1d as the kind of
        interpolation to use when reforming the light curves to the timebase of
        the TFA templates.

    sigclip : float or sequence of two floats or None
        This is the sigma clip to apply to the light curves before running TFA
        on it.

    mintemplatedist_arcmin : float
        This sets the minimum distance required from the target object for
        objects in the TFA template ensemble. Objects closer than this distance
        will be removed from the ensemble.

    nworkers : int
        The number of parallel workers to launch

    maxworkertasks : int
        The maximum number of tasks per worker allowed before it's replaced by a
        fresh one.

    Returns
    -------

    dict
        Contains the input file names and output TFA light curve filenames per
        input file organized by each `magcol` in `magcols`.


try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if timecols is None:<EOL><INDENT>timecols = dtimecols<EOL><DEDENT>if magcols is None:<EOL><INDENT>magcols = dmagcols<EOL><DEDENT>if errcols is None:<EOL><INDENT>errcols = derrcols<EOL><DEDENT>pklist = glob.glob(os.path.join(featuresdir, '<STR_LIT>'))<EOL>if maxobjects:<EOL><INDENT>pklist = pklist[:maxobjects]<EOL><DEDENT>allobjects = {}<EOL>for magcol in magcols:<EOL><INDENT>if (isinstance(min_stetj_stdev, list) or<EOL>isinstance(min_stetj_stdev, np.ndarray)):<EOL><INDENT>magcol_min_stetj_stdev = min_stetj_stdev[::]<EOL><DEDENT>else:<EOL><INDENT>magcol_min_stetj_stdev = min_stetj_stdev<EOL><DEDENT>if (isinstance(min_iqr_stdev, list) or<EOL>isinstance(min_iqr_stdev, np.ndarray)):<EOL><INDENT>magcol_min_iqr_stdev = min_iqr_stdev[::]<EOL><DEDENT>else:<EOL><INDENT>magcol_min_iqr_stdev = min_iqr_stdev<EOL><DEDENT>if (isinstance(min_inveta_stdev, list) or<EOL>isinstance(min_inveta_stdev, np.ndarray)):<EOL><INDENT>magcol_min_inveta_stdev = min_inveta_stdev[::]<EOL><DEDENT>else:<EOL><INDENT>magcol_min_inveta_stdev = min_inveta_stdev<EOL><DEDENT>LOGINFO('<STR_LIT>')<EOL>allobjects[magcol] = {<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[]<EOL>}<EOL>if TQDM and verbose:<EOL><INDENT>listiterator = tqdm(pklist)<EOL><DEDENT>else:<EOL><INDENT>listiterator = pklist<EOL><DEDENT>for pkl in listiterator:<EOL><INDENT>with open(pkl,'<STR_LIT:rb>') as infd:<EOL><INDENT>thisfeatures = pickle.load(infd)<EOL><DEDENT>objectid = thisfeatures['<STR_LIT>']<EOL>if ('<STR_LIT:info>' in thisfeatures and<EOL>thisfeatures['<STR_LIT:info>'] and<EOL>'<STR_LIT>' in thisfeatures['<STR_LIT:info>']):<EOL><INDENT>if (thisfeatures['<STR_LIT:info>']['<STR_LIT>'] and<EOL>thisfeatures['<STR_LIT:info>']['<STR_LIT>'] > <NUM_LIT>):<EOL><INDENT>sdssr = thisfeatures['<STR_LIT:info>']['<STR_LIT>']<EOL><DEDENT>elif (magcol in thisfeatures and<EOL>thisfeatures[magcol] and<EOL>'<STR_LIT>' in thisfeatures[magcol] and<EOL>thisfeatures[magcol]['<STR_LIT>'] > <NUM_LIT>):<EOL><INDENT>sdssr = thisfeatures[magcol]['<STR_LIT>']<EOL><DEDENT>elif (thisfeatures['<STR_LIT:info>']['<STR_LIT>'] and<EOL>thisfeatures['<STR_LIT:info>']['<STR_LIT>'] and<EOL>thisfeatures['<STR_LIT:info>']['<STR_LIT>']):<EOL><INDENT>sdssr = jhk_to_sdssr(thisfeatures['<STR_LIT:info>']['<STR_LIT>'],<EOL>thisfeatures['<STR_LIT:info>']['<STR_LIT>'],<EOL>thisfeatures['<STR_LIT:info>']['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>sdssr = np.nan<EOL><DEDENT><DEDENT>else:<EOL><INDENT>sdssr = np.nan<EOL><DEDENT>if (magcol in thisfeatures and<EOL>thisfeatures[magcol] and<EOL>thisfeatures[magcol]['<STR_LIT>']):<EOL><INDENT>lcmad = thisfeatures[magcol]['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>lcmad = np.nan<EOL><DEDENT>if (magcol in thisfeatures and<EOL>thisfeatures[magcol] and<EOL>thisfeatures[magcol]['<STR_LIT>']):<EOL><INDENT>stetsonj = thisfeatures[magcol]['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>stetsonj = np.nan<EOL><DEDENT>if (magcol in thisfeatures and<EOL>thisfeatures[magcol] and<EOL>thisfeatures[magcol]['<STR_LIT>']):<EOL><INDENT>iqr = thisfeatures[magcol]['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>iqr = np.nan<EOL><DEDENT>if (magcol in thisfeatures and<EOL>thisfeatures[magcol] and<EOL>thisfeatures[magcol]['<STR_LIT>']):<EOL><INDENT>eta = thisfeatures[magcol]['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>eta = np.nan<EOL><DEDENT>allobjects[magcol]['<STR_LIT>'].append(objectid)<EOL>allobjects[magcol]['<STR_LIT>'].append(sdssr)<EOL>allobjects[magcol]['<STR_LIT>'].append(lcmad)<EOL>allobjects[magcol]['<STR_LIT>'].append(stetsonj)<EOL>allobjects[magcol]['<STR_LIT>'].append(iqr)<EOL>allobjects[magcol]['<STR_LIT>'].append(eta)<EOL><DEDENT>LOGINFO('<STR_LIT>')<EOL>allobjects[magcol]['<STR_LIT>'] = np.ravel(np.array(<EOL>allobjects[magcol]['<STR_LIT>']<EOL>))<EOL>allobjects[magcol]['<STR_LIT>'] = np.ravel(np.array(<EOL>allobjects[magcol]['<STR_LIT>']<EOL>))<EOL>allobjects[magcol]['<STR_LIT>'] = np.ravel(np.array(<EOL>allobjects[magcol]['<STR_LIT>']<EOL>))<EOL>allobjects[magcol]['<STR_LIT>'] = np.ravel(np.array(<EOL>allobjects[magcol]['<STR_LIT>']<EOL>))<EOL>allobjects[magcol]['<STR_LIT>'] = np.ravel(np.array(<EOL>allobjects[magcol]['<STR_LIT>']<EOL>))<EOL>allobjects[magcol]['<STR_LIT>'] = np.ravel(np.array(<EOL>allobjects[magcol]['<STR_LIT>']<EOL>))<EOL>thisfinind = (<EOL>np.isfinite(allobjects[magcol]['<STR_LIT>']) &<EOL>np.isfinite(allobjects[magcol]['<STR_LIT>']) &<EOL>np.isfinite(allobjects[magcol]['<STR_LIT>']) &<EOL>np.isfinite(allobjects[magcol]['<STR_LIT>']) &<EOL>np.isfinite(allobjects[magcol]['<STR_LIT>'])<EOL>)<EOL>allobjects[magcol]['<STR_LIT>'] = allobjects[magcol]['<STR_LIT>'][<EOL>thisfinind<EOL>]<EOL>allobjects[magcol]['<STR_LIT>'] = allobjects[magcol]['<STR_LIT>'][thisfinind]<EOL>allobjects[magcol]['<STR_LIT>'] = allobjects[magcol]['<STR_LIT>'][thisfinind]<EOL>allobjects[magcol]['<STR_LIT>'] = allobjects[magcol]['<STR_LIT>'][<EOL>thisfinind<EOL>]<EOL>allobjects[magcol]['<STR_LIT>'] = allobjects[magcol]['<STR_LIT>'][thisfinind]<EOL>allobjects[magcol]['<STR_LIT>'] = allobjects[magcol]['<STR_LIT>'][thisfinind]<EOL>allobjects[magcol]['<STR_LIT>'] = <NUM_LIT:1.0>/allobjects[magcol]['<STR_LIT>']<EOL>magbininds = np.digitize(allobjects[magcol]['<STR_LIT>'],<EOL>magbins)<EOL>binned_objectids = []<EOL>binned_sdssr = []<EOL>binned_sdssr_median = []<EOL>binned_lcmad = []<EOL>binned_stetsonj = []<EOL>binned_iqr = []<EOL>binned_inveta = []<EOL>binned_count = []<EOL>binned_objectids_thresh_stetsonj = []<EOL>binned_objectids_thresh_iqr = []<EOL>binned_objectids_thresh_inveta = []<EOL>binned_objectids_thresh_all = []<EOL>binned_lcmad_median = []<EOL>binned_lcmad_stdev = []<EOL>binned_stetsonj_median = []<EOL>binned_stetsonj_stdev = []<EOL>binned_inveta_median = []<EOL>binned_inveta_stdev = []<EOL>binned_iqr_median = []<EOL>binned_iqr_stdev = []<EOL>for mbinind, magi in zip(np.unique(magbininds),<EOL>range(len(magbins)-<NUM_LIT:1>)):<EOL><INDENT>thisbinind = np.where(magbininds == mbinind)<EOL>thisbin_sdssr_median = (magbins[magi] + magbins[magi+<NUM_LIT:1>])/<NUM_LIT><EOL>binned_sdssr_median.append(thisbin_sdssr_median)<EOL>thisbin_objectids = allobjects[magcol]['<STR_LIT>'][thisbinind]<EOL>thisbin_sdssr = allobjects[magcol]['<STR_LIT>'][thisbinind]<EOL>thisbin_lcmad = allobjects[magcol]['<STR_LIT>'][thisbinind]<EOL>thisbin_stetsonj = allobjects[magcol]['<STR_LIT>'][thisbinind]<EOL>thisbin_iqr = allobjects[magcol]['<STR_LIT>'][thisbinind]<EOL>thisbin_inveta = allobjects[magcol]['<STR_LIT>'][thisbinind]<EOL>thisbin_count = thisbin_objectids.size<EOL>if thisbin_count > <NUM_LIT:4>:<EOL><INDENT>thisbin_lcmad_median = np.median(thisbin_lcmad)<EOL>thisbin_lcmad_stdev = np.median(<EOL>np.abs(thisbin_lcmad - thisbin_lcmad_median)<EOL>) * <NUM_LIT><EOL>binned_lcmad_median.append(thisbin_lcmad_median)<EOL>binned_lcmad_stdev.append(thisbin_lcmad_stdev)<EOL>thisbin_stetsonj_median = np.median(thisbin_stetsonj)<EOL>thisbin_stetsonj_stdev = np.median(<EOL>np.abs(thisbin_stetsonj - thisbin_stetsonj_median)<EOL>) * <NUM_LIT><EOL>binned_stetsonj_median.append(thisbin_stetsonj_median)<EOL>binned_stetsonj_stdev.append(thisbin_stetsonj_stdev)<EOL>if isinstance(magcol_min_stetj_stdev, float):<EOL><INDENT>thisbin_objectids_thresh_stetsonj = thisbin_objectids[<EOL>thisbin_stetsonj > (<EOL>thisbin_stetsonj_median +<EOL>magcol_min_stetj_stdev*thisbin_stetsonj_stdev<EOL>)<EOL>]<EOL><DEDENT>elif (isinstance(magcol_min_stetj_stdev, np.ndarray) or<EOL>isinstance(magcol_min_stetj_stdev, list)):<EOL><INDENT>thisbin_min_stetj_stdev = magcol_min_stetj_stdev[magi]<EOL>if not np.isfinite(thisbin_min_stetj_stdev):<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>thisbin_sdssr_median)<EOL>thisbin_min_stetj_stdev = <NUM_LIT><EOL>magcol_min_stetj_stdev[magi] = <NUM_LIT><EOL><DEDENT>thisbin_objectids_thresh_stetsonj = thisbin_objectids[<EOL>thisbin_stetsonj > (<EOL>thisbin_stetsonj_median +<EOL>thisbin_min_stetj_stdev*thisbin_stetsonj_stdev<EOL>)<EOL>]<EOL><DEDENT>thisbin_iqr_median = np.median(thisbin_iqr)<EOL>thisbin_iqr_stdev = np.median(<EOL>np.abs(thisbin_iqr - thisbin_iqr_median)<EOL>) * <NUM_LIT><EOL>binned_iqr_median.append(thisbin_iqr_median)<EOL>binned_iqr_stdev.append(thisbin_iqr_stdev)<EOL>if isinstance(magcol_min_iqr_stdev, float):<EOL><INDENT>thisbin_objectids_thresh_iqr = thisbin_objectids[<EOL>thisbin_iqr > (thisbin_iqr_median +<EOL>magcol_min_iqr_stdev*thisbin_iqr_stdev)<EOL>]<EOL><DEDENT>elif (isinstance(magcol_min_iqr_stdev, np.ndarray) or<EOL>isinstance(magcol_min_iqr_stdev, list)):<EOL><INDENT>thisbin_min_iqr_stdev = magcol_min_iqr_stdev[magi]<EOL>if not np.isfinite(thisbin_min_iqr_stdev):<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>thisbin_sdssr_median)<EOL>thisbin_min_iqr_stdev = <NUM_LIT><EOL>magcol_min_iqr_stdev[magi] = <NUM_LIT><EOL><DEDENT>thisbin_objectids_thresh_iqr = thisbin_objectids[<EOL>thisbin_iqr > (thisbin_iqr_median +<EOL>thisbin_min_iqr_stdev*thisbin_iqr_stdev)<EOL>]<EOL><DEDENT>thisbin_inveta_median = np.median(thisbin_inveta)<EOL>thisbin_inveta_stdev = np.median(<EOL>np.abs(thisbin_inveta - thisbin_inveta_median)<EOL>) * <NUM_LIT><EOL>binned_inveta_median.append(thisbin_inveta_median)<EOL>binned_inveta_stdev.append(thisbin_inveta_stdev)<EOL>if isinstance(magcol_min_inveta_stdev, float):<EOL><INDENT>thisbin_objectids_thresh_inveta = thisbin_objectids[<EOL>thisbin_inveta > (<EOL>thisbin_inveta_median +<EOL>magcol_min_inveta_stdev*thisbin_inveta_stdev<EOL>)<EOL>]<EOL><DEDENT>elif (isinstance(magcol_min_inveta_stdev, np.ndarray) or<EOL>isinstance(magcol_min_inveta_stdev, list)):<EOL><INDENT>thisbin_min_inveta_stdev = magcol_min_inveta_stdev[magi]<EOL>if not np.isfinite(thisbin_min_inveta_stdev):<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>thisbin_sdssr_median)<EOL>thisbin_min_inveta_stdev = <NUM_LIT><EOL>magcol_min_inveta_stdev[magi] = <NUM_LIT><EOL><DEDENT>thisbin_objectids_thresh_inveta = thisbin_objectids[<EOL>thisbin_inveta > (<EOL>thisbin_inveta_median +<EOL>thisbin_min_inveta_stdev*thisbin_inveta_stdev<EOL>)<EOL>]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>thisbin_objectids_thresh_stetsonj = (<EOL>np.array([],dtype=np.unicode_)<EOL>)<EOL>thisbin_objectids_thresh_iqr = (<EOL>np.array([],dtype=np.unicode_)<EOL>)<EOL>thisbin_objectids_thresh_inveta = (<EOL>np.array([],dtype=np.unicode_)<EOL>)<EOL><DEDENT>thisbin_objectids_thresh_all = reduce(<EOL>np.intersect1d,<EOL>(thisbin_objectids_thresh_stetsonj,<EOL>thisbin_objectids_thresh_iqr,<EOL>thisbin_objectids_thresh_inveta)<EOL>)<EOL>binned_objectids.append(thisbin_objectids)<EOL>binned_sdssr.append(thisbin_sdssr)<EOL>binned_lcmad.append(thisbin_lcmad)<EOL>binned_stetsonj.append(thisbin_stetsonj)<EOL>binned_iqr.append(thisbin_iqr)<EOL>binned_inveta.append(thisbin_inveta)<EOL>binned_count.append(thisbin_objectids.size)<EOL>binned_objectids_thresh_stetsonj.append(<EOL>thisbin_objectids_thresh_stetsonj<EOL>)<EOL>binned_objectids_thresh_iqr.append(<EOL>thisbin_objectids_thresh_iqr<EOL>)<EOL>binned_objectids_thresh_inveta.append(<EOL>thisbin_objectids_thresh_inveta<EOL>)<EOL>binned_objectids_thresh_all.append(<EOL>thisbin_objectids_thresh_all<EOL>)<EOL><DEDENT>allobjects[magcol]['<STR_LIT>'] = magbins<EOL>allobjects[magcol]['<STR_LIT>'] = binned_objectids<EOL>allobjects[magcol]['<STR_LIT>'] = binned_sdssr_median<EOL>allobjects[magcol]['<STR_LIT>'] = binned_sdssr<EOL>allobjects[magcol]['<STR_LIT>'] = binned_count<EOL>allobjects[magcol]['<STR_LIT>'] = binned_lcmad<EOL>allobjects[magcol]['<STR_LIT>'] = binned_lcmad_median<EOL>allobjects[magcol]['<STR_LIT>'] = binned_lcmad_stdev<EOL>allobjects[magcol]['<STR_LIT>'] = binned_stetsonj<EOL>allobjects[magcol]['<STR_LIT>'] = binned_stetsonj_median<EOL>allobjects[magcol]['<STR_LIT>'] = binned_stetsonj_stdev<EOL>allobjects[magcol]['<STR_LIT>'] = binned_iqr<EOL>allobjects[magcol]['<STR_LIT>'] = binned_iqr_median<EOL>allobjects[magcol]['<STR_LIT>'] = binned_iqr_stdev<EOL>allobjects[magcol]['<STR_LIT>'] = binned_inveta<EOL>allobjects[magcol]['<STR_LIT>'] = binned_inveta_median<EOL>allobjects[magcol]['<STR_LIT>'] = binned_inveta_stdev<EOL>allobjects[magcol]['<STR_LIT>'] = (<EOL>binned_objectids_thresh_stetsonj<EOL>)<EOL>allobjects[magcol]['<STR_LIT>'] = (<EOL>binned_objectids_thresh_iqr<EOL>)<EOL>allobjects[magcol]['<STR_LIT>'] = (<EOL>binned_objectids_thresh_inveta<EOL>)<EOL>allobjects[magcol]['<STR_LIT>'] = (<EOL>binned_objectids_thresh_all<EOL>)<EOL>try:<EOL><INDENT>allobjects[magcol]['<STR_LIT>'] = np.unique(<EOL>np.concatenate(<EOL>allobjects[magcol]['<STR_LIT>']<EOL>)<EOL>)<EOL><DEDENT>except ValueError:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL>allobjects[magcol]['<STR_LIT>'] = (<EOL>np.array([])<EOL>)<EOL><DEDENT>allobjects[magcol]['<STR_LIT>'] = np.unique(<EOL>np.concatenate(<EOL>allobjects[magcol]['<STR_LIT>']<EOL>)<EOL>)<EOL>allobjects[magcol]['<STR_LIT>'] = np.unique(<EOL>np.concatenate(allobjects[magcol]['<STR_LIT>'])<EOL>)<EOL>allobjects[magcol]['<STR_LIT>'] = np.unique(<EOL>np.concatenate(allobjects[magcol]['<STR_LIT>'])<EOL>)<EOL>if isinstance(min_stetj_stdev, list):<EOL><INDENT>allobjects[magcol]['<STR_LIT>'] = np.array(<EOL>magcol_min_stetj_stdev<EOL>)<EOL><DEDENT>else:<EOL><INDENT>allobjects[magcol]['<STR_LIT>'] = magcol_min_stetj_stdev<EOL><DEDENT>if isinstance(min_iqr_stdev, list):<EOL><INDENT>allobjects[magcol]['<STR_LIT>'] = np.array(<EOL>magcol_min_iqr_stdev<EOL>)<EOL><DEDENT>else:<EOL><INDENT>allobjects[magcol]['<STR_LIT>'] = magcol_min_iqr_stdev<EOL><DEDENT>if isinstance(min_inveta_stdev, list):<EOL><INDENT>allobjects[magcol]['<STR_LIT>'] = np.array(<EOL>magcol_min_inveta_stdev<EOL>)<EOL><DEDENT>else:<EOL><INDENT>allobjects[magcol]['<STR_LIT>'] = magcol_min_inveta_stdev<EOL><DEDENT>allobjects[magcol]['<STR_LIT>'] = min_lcmad_stdev<EOL><DEDENT>allobjects['<STR_LIT>'] = magbins<EOL>with open(outfile,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(allobjects, outfd, protocol=pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>return allobjects<EOL>

This generates a list of objects with stetson J, IQR, and 1.0/eta
    above some threshold value to select them as potential variable stars.

    Use this to pare down the objects to review and put through
    period-finding. This does the thresholding per magnitude bin; this should be
    better than one single cut through the entire magnitude range. Set the
    magnitude bins using the magbins kwarg.

    FIXME: implement a voting classifier here. this will choose variables based
    on the thresholds in IQR, stetson, and inveta based on weighting carried
    over from the variability recovery sims.

    Parameters
    ----------

    featuresdir : str
        This is the directory containing variability feature pickles created by
        :py:func:`astrobase.lcproc.lcpfeatures.parallel_varfeatures` or similar.

    outfile : str
        This is the output pickle file that will contain all the threshold
        information.

    magbins : np.array of floats
        This sets the magnitude bins to use for calculating thresholds.

    maxobjects : int or None
        This is the number of objects to process. If None, all objects with
        feature pickles in `featuresdir` will be processed.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the thresholds.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the thresholds.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the thresholds.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    min_lcmad_stdev,min_stetj_stdev,min_iqr_stdev,min_inveta_stdev : float or np.array
        These are all the standard deviation multiplier for the distributions of
        light curve standard deviation, Stetson J variability index, the light
        curve interquartile range, and 1/eta variability index
        respectively. These multipliers set the minimum values of these measures
        to use for selecting variable stars. If provided as floats, the same
        value will be used for all magbins. If provided as np.arrays of `size =
        magbins.size - 1`, will be used to apply possibly different sigma cuts
        for each magbin.

    verbose : bool
        If True, will report progress and warn about any problems.

    Returns
    -------

    dict
        Contains all of the variability threshold information along with indices
        into the array of the object IDs chosen as variables.


try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if magcols is None:<EOL><INDENT>magcols = dmagcols<EOL><DEDENT>with open(varthreshpkl,'<STR_LIT:rb>') as infd:<EOL><INDENT>allobjects = pickle.load(infd)<EOL><DEDENT>magbins = allobjects['<STR_LIT>']<EOL>for magcol in magcols:<EOL><INDENT>min_lcmad_stdev = (<EOL>xmin_lcmad_stdev or allobjects[magcol]['<STR_LIT>']<EOL>)<EOL>min_stetj_stdev = (<EOL>xmin_stetj_stdev or allobjects[magcol]['<STR_LIT>']<EOL>)<EOL>min_iqr_stdev = (<EOL>xmin_iqr_stdev or allobjects[magcol]['<STR_LIT>']<EOL>)<EOL>min_inveta_stdev = (<EOL>xmin_inveta_stdev or allobjects[magcol]['<STR_LIT>']<EOL>)<EOL>fig = plt.figure(figsize=(<NUM_LIT:20>,<NUM_LIT:16>))<EOL>plt.subplot(<NUM_LIT>)<EOL>plt.plot(allobjects[magcol]['<STR_LIT>'],<EOL>allobjects[magcol]['<STR_LIT>']*<NUM_LIT>,<EOL>marker='<STR_LIT:.>',ms=<NUM_LIT:1.0>, linestyle='<STR_LIT:none>',<EOL>rasterized=True)<EOL>plt.plot(allobjects[magcol]['<STR_LIT>'],<EOL>np.array(allobjects[magcol]['<STR_LIT>'])*<NUM_LIT>,<EOL>linewidth=<NUM_LIT>)<EOL>plt.plot(<EOL>allobjects[magcol]['<STR_LIT>'],<EOL>np.array(allobjects[magcol]['<STR_LIT>'])*<NUM_LIT> +<EOL>min_lcmad_stdev*np.array(<EOL>allobjects[magcol]['<STR_LIT>']<EOL>),<EOL>linewidth=<NUM_LIT>, linestyle='<STR_LIT>'<EOL>)<EOL>plt.xlim((magbins.min()-<NUM_LIT>, magbins.max()))<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel(r'<STR_LIT>')<EOL>plt.title('<STR_LIT>' % magcol)<EOL>plt.yscale('<STR_LIT>')<EOL>plt.tight_layout()<EOL>plt.subplot(<NUM_LIT>)<EOL>plt.plot(allobjects[magcol]['<STR_LIT>'],<EOL>allobjects[magcol]['<STR_LIT>'],<EOL>marker='<STR_LIT:.>',ms=<NUM_LIT:1.0>, linestyle='<STR_LIT:none>',<EOL>rasterized=True)<EOL>plt.plot(allobjects[magcol]['<STR_LIT>'],<EOL>allobjects[magcol]['<STR_LIT>'],<EOL>linewidth=<NUM_LIT>)<EOL>plt.plot(<EOL>allobjects[magcol]['<STR_LIT>'],<EOL>np.array(allobjects[magcol]['<STR_LIT>']) +<EOL>min_stetj_stdev*np.array(<EOL>allobjects[magcol]['<STR_LIT>']<EOL>),<EOL>linewidth=<NUM_LIT>, linestyle='<STR_LIT>'<EOL>)<EOL>plt.xlim((magbins.min()-<NUM_LIT>, magbins.max()))<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>' % magcol)<EOL>plt.yscale('<STR_LIT>')<EOL>plt.tight_layout()<EOL>plt.subplot(<NUM_LIT>)<EOL>plt.plot(allobjects[magcol]['<STR_LIT>'],<EOL>allobjects[magcol]['<STR_LIT>'],<EOL>marker='<STR_LIT:.>',ms=<NUM_LIT:1.0>, linestyle='<STR_LIT:none>',<EOL>rasterized=True)<EOL>plt.plot(allobjects[magcol]['<STR_LIT>'],<EOL>allobjects[magcol]['<STR_LIT>'],<EOL>linewidth=<NUM_LIT>)<EOL>plt.plot(<EOL>allobjects[magcol]['<STR_LIT>'],<EOL>np.array(allobjects[magcol]['<STR_LIT>']) +<EOL>min_iqr_stdev*np.array(<EOL>allobjects[magcol]['<STR_LIT>']<EOL>),<EOL>linewidth=<NUM_LIT>, linestyle='<STR_LIT>'<EOL>)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>' % magcol)<EOL>plt.xlim((magbins.min()-<NUM_LIT>, magbins.max()))<EOL>plt.yscale('<STR_LIT>')<EOL>plt.tight_layout()<EOL>plt.subplot(<NUM_LIT>)<EOL>plt.plot(allobjects[magcol]['<STR_LIT>'],<EOL>allobjects[magcol]['<STR_LIT>'],<EOL>marker='<STR_LIT:.>',ms=<NUM_LIT:1.0>, linestyle='<STR_LIT:none>',<EOL>rasterized=True)<EOL>plt.plot(allobjects[magcol]['<STR_LIT>'],<EOL>allobjects[magcol]['<STR_LIT>'],<EOL>linewidth=<NUM_LIT>)<EOL>plt.plot(<EOL>allobjects[magcol]['<STR_LIT>'],<EOL>np.array(allobjects[magcol]['<STR_LIT>']) +<EOL>min_inveta_stdev*np.array(<EOL>allobjects[magcol]['<STR_LIT>']<EOL>),<EOL>linewidth=<NUM_LIT>, linestyle='<STR_LIT>'<EOL>)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel(r'<STR_LIT>')<EOL>plt.title(r'<STR_LIT>' % magcol)<EOL>plt.xlim((magbins.min()-<NUM_LIT>, magbins.max()))<EOL>plt.yscale('<STR_LIT>')<EOL>plt.tight_layout()<EOL>plt.savefig('<STR_LIT>' % (varthreshpkl,<EOL>magcol),<EOL>bbox_inches='<STR_LIT>')<EOL>plt.close('<STR_LIT:all>')<EOL><DEDENT>

This makes plots for the variability threshold distributions.

    Parameters
    ----------

    varthreshpkl : str
        The pickle produced by the function above.

    xmin_lcmad_stdev,xmin_stetj_stdev,xmin_iqr_stdev,xmin_inveta_stdev : float or np.array
        Values of the threshold values to override the ones in the
        `vartresholdpkl`. If provided, will plot the thresholds accordingly
        instead of using the ones in the input pickle directly.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    magcols : list of str or None
        The magcol keys to use from the lcdict.

    Returns
    -------

    str
        The file name of the threshold plot generated.


lcf, columns, lcformat, lcformatdir, lcndetkey = task<EOL>try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>lcobjdict = {'<STR_LIT>':os.path.abspath(lcf)}<EOL>try:<EOL><INDENT>lcdict = readerfunc(lcf)<EOL>if ( (isinstance(lcdict, (list, tuple))) and<EOL>(isinstance(lcdict[<NUM_LIT:0>], dict)) ):<EOL><INDENT>lcdict = lcdict[<NUM_LIT:0>]<EOL><DEDENT>for colkey in columns:<EOL><INDENT>if '<STR_LIT:.>' in colkey:<EOL><INDENT>getkey = colkey.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>getkey = [colkey]<EOL><DEDENT>try:<EOL><INDENT>thiscolval = _dict_get(lcdict, getkey)<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGWARNING('<STR_LIT>' %<EOL>(colkey, lcf))<EOL>thiscolval = np.nan<EOL><DEDENT>lcobjdict[getkey[-<NUM_LIT:1>]] = thiscolval<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' % lcf)<EOL>for colkey in columns:<EOL><INDENT>if '<STR_LIT:.>' in colkey:<EOL><INDENT>getkey = colkey.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>getkey = [colkey]<EOL><DEDENT>thiscolval = np.nan<EOL>lcobjdict[getkey[-<NUM_LIT:1>]] = thiscolval<EOL><DEDENT><DEDENT>for dk in lcndetkey:<EOL><INDENT>try:<EOL><INDENT>if '<STR_LIT:.>' in dk:<EOL><INDENT>getdk = dk.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>getdk = [dk]<EOL><DEDENT>ndetcol = _dict_get(lcdict, getdk)<EOL>actualndets = ndetcol[np.isfinite(ndetcol)].size<EOL>lcobjdict['<STR_LIT>' % getdk[-<NUM_LIT:1>]] = actualndets<EOL><DEDENT>except Exception as e:<EOL><INDENT>lcobjdict['<STR_LIT>' % getdk[-<NUM_LIT:1>]] = np.nan<EOL><DEDENT><DEDENT>return lcobjdict<EOL>

This is a parallel worker for makelclist.

    Parameters
    ----------

    task : tuple
        This is a tuple containing the following items:

        task[0] = lcf
        task[1] = columns
        task[2] = lcformat
        task[3] = lcformatdir
        task[4] = lcndetkey

    Returns
    -------

    dict or None
        This contains all of the info for the object processed in this LC read
        operation. If this fails, returns None


try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if not fileglob:<EOL><INDENT>fileglob = dfileglob<EOL><DEDENT>lcndetkey = dmagcols<EOL>if isinstance(use_list_of_filenames, list):<EOL><INDENT>matching = use_list_of_filenames<EOL><DEDENT>else:<EOL><INDENT>if isinstance(basedir, list):<EOL><INDENT>matching = []<EOL>for bdir in basedir:<EOL><INDENT>LOGINFO('<STR_LIT>' % (lcformat,<EOL>bdir))<EOL>if recursive is False:<EOL><INDENT>matching.extend(glob.glob(os.path.join(bdir, fileglob)))<EOL><DEDENT>else:<EOL><INDENT>if sys.version_info[:<NUM_LIT:2>] > (<NUM_LIT:3>,<NUM_LIT:4>):<EOL><INDENT>matching.extend(glob.glob(os.path.join(bdir,<EOL>'<STR_LIT>',<EOL>fileglob),<EOL>recursive=True))<EOL><DEDENT>else:<EOL><INDENT>walker = os.walk(bdir)<EOL>for root, dirs, _files in walker:<EOL><INDENT>for sdir in dirs:<EOL><INDENT>searchpath = os.path.join(root,<EOL>sdir,<EOL>fileglob)<EOL>foundfiles = glob.glob(searchpath)<EOL>if foundfiles:<EOL><INDENT>matching.extend(foundfiles)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>LOGINFO('<STR_LIT>' %<EOL>(lcformat, basedir))<EOL>if recursive is False:<EOL><INDENT>matching = glob.glob(os.path.join(basedir, fileglob))<EOL><DEDENT>else:<EOL><INDENT>if sys.version_info[:<NUM_LIT:2>] > (<NUM_LIT:3>,<NUM_LIT:4>):<EOL><INDENT>matching = glob.glob(os.path.join(basedir,<EOL>'<STR_LIT>',<EOL>fileglob),recursive=True)<EOL><DEDENT>else:<EOL><INDENT>walker = os.walk(basedir)<EOL>matching = []<EOL>for root, dirs, _files in walker:<EOL><INDENT>for sdir in dirs:<EOL><INDENT>searchpath = os.path.join(root,<EOL>sdir,<EOL>fileglob)<EOL>foundfiles = glob.glob(searchpath)<EOL>if foundfiles:<EOL><INDENT>matching.extend(foundfiles)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if matching and len(matching) > <NUM_LIT:0>:<EOL><INDENT>LOGINFO('<STR_LIT>' % len(matching))<EOL>if maxlcs:<EOL><INDENT>matching = matching[:maxlcs]<EOL><DEDENT>lclistdict = {<EOL>'<STR_LIT>':basedir,<EOL>'<STR_LIT>':lcformat,<EOL>'<STR_LIT>':fileglob,<EOL>'<STR_LIT>':recursive,<EOL>'<STR_LIT>':columns,<EOL>'<STR_LIT>':makecoordindex,<EOL>'<STR_LIT>':len(matching),<EOL>'<STR_LIT>': {<EOL>}<EOL>}<EOL>derefcols = ['<STR_LIT>']<EOL>derefcols.extend(['<STR_LIT>' % x.split('<STR_LIT:.>')[-<NUM_LIT:1>] for x in lcndetkey])<EOL>for dc in derefcols:<EOL><INDENT>lclistdict['<STR_LIT>'][dc] = []<EOL><DEDENT>for col in columns:<EOL><INDENT>thiscol = col.split('<STR_LIT:.>')<EOL>thiscol = thiscol[-<NUM_LIT:1>]<EOL>lclistdict['<STR_LIT>'][thiscol] = []<EOL>derefcols.append(thiscol)<EOL><DEDENT>LOGINFO('<STR_LIT>')<EOL>tasks = [(x, columns, lcformat, lcformatdir, lcndetkey)<EOL>for x in matching]<EOL>with ProcessPoolExecutor(max_workers=nworkers) as executor:<EOL><INDENT>results = executor.map(_lclist_parallel_worker, tasks)<EOL><DEDENT>results = [x for x in results]<EOL>for result in results:<EOL><INDENT>for xcol in derefcols:<EOL><INDENT>lclistdict['<STR_LIT>'][xcol].append(result[xcol])<EOL><DEDENT><DEDENT>executor.shutdown()<EOL>for col in lclistdict['<STR_LIT>']:<EOL><INDENT>lclistdict['<STR_LIT>'][col] = np.array(lclistdict['<STR_LIT>'][col])<EOL><DEDENT>uniques, counts = np.unique(lclistdict['<STR_LIT>']['<STR_LIT>'],<EOL>return_counts=True)<EOL>duplicated_objectids = uniques[counts > <NUM_LIT:1>]<EOL>if duplicated_objectids.size > <NUM_LIT:0>:<EOL><INDENT>dt = lclistdict['<STR_LIT>']['<STR_LIT>'].dtype.str<EOL>dt = '<STR_LIT>' % (<EOL>int(dt.replace('<STR_LIT:<>','<STR_LIT>').replace('<STR_LIT>','<STR_LIT>').replace('<STR_LIT:S>','<STR_LIT>')) + <NUM_LIT:3><EOL>)<EOL>lclistdict['<STR_LIT>']['<STR_LIT>'] = np.array(<EOL>lclistdict['<STR_LIT>']['<STR_LIT>'],<EOL>dtype=dt<EOL>)<EOL>for objid in duplicated_objectids:<EOL><INDENT>objid_inds = np.where(<EOL>lclistdict['<STR_LIT>']['<STR_LIT>'] == objid<EOL>)<EOL>for ncounter, nind in enumerate(objid_inds[<NUM_LIT:0>][<NUM_LIT:1>:]):<EOL><INDENT>lclistdict['<STR_LIT>']['<STR_LIT>'][nind] = '<STR_LIT>' % (<EOL>lclistdict['<STR_LIT>']['<STR_LIT>'][nind],<EOL>ncounter+<NUM_LIT:2><EOL>)<EOL>LOGWARNING(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(ncounter+<NUM_LIT:2>, objid, objid, ncounter+<NUM_LIT:2>,<EOL>lclistdict['<STR_LIT>']['<STR_LIT>'][nind])<EOL>)<EOL><DEDENT><DEDENT><DEDENT>if (makecoordindex and<EOL>isinstance(makecoordindex, (list, tuple)) and<EOL>len(makecoordindex) == <NUM_LIT:2>):<EOL><INDENT>try:<EOL><INDENT>racol, declcol = makecoordindex<EOL>racol = racol.split('<STR_LIT:.>')[-<NUM_LIT:1>]<EOL>declcol = declcol.split('<STR_LIT:.>')[-<NUM_LIT:1>]<EOL>objra, objdecl = (lclistdict['<STR_LIT>'][racol],<EOL>lclistdict['<STR_LIT>'][declcol])<EOL>cosdecl = np.cos(np.radians(objdecl))<EOL>sindecl = np.sin(np.radians(objdecl))<EOL>cosra = np.cos(np.radians(objra))<EOL>sinra = np.sin(np.radians(objra))<EOL>xyz = np.column_stack((cosra*cosdecl,sinra*cosdecl, sindecl))<EOL>kdt = sps.cKDTree(xyz,copy_data=True)<EOL>lclistdict['<STR_LIT>'] = kdt<EOL>LOGINFO('<STR_LIT>' %<EOL>(makecoordindex[<NUM_LIT:0>], makecoordindex[<NUM_LIT:1>]))<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' %<EOL>(makecoordindex[<NUM_LIT:0>], makecoordindex[<NUM_LIT:1>]))<EOL>raise<EOL><DEDENT><DEDENT>if field_fitsfile and os.path.exists(field_fitsfile):<EOL><INDENT>if field_wcsfrom is None:<EOL><INDENT>hdulist = pyfits.open(field_fitsfile)<EOL>hdr = hdulist[<NUM_LIT:0>].header<EOL>hdulist.close()<EOL>w = WCS(hdr)<EOL>wcsok = True<EOL><DEDENT>elif os.path.exists(field_wcsfrom):<EOL><INDENT>w = WCS(field_wcsfrom)<EOL>wcsok = True<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>' %<EOL>field_fitsfile)<EOL>wcsok = False<EOL><DEDENT>if wcsok:<EOL><INDENT>radecl = np.column_stack((objra, objdecl))<EOL>lclistdict['<STR_LIT>']['<STR_LIT>'] = w.all_world2pix(<EOL>radecl,<EOL><NUM_LIT:1><EOL>)<EOL>finder_outfile = os.path.join(<EOL>os.path.dirname(outfile),<EOL>os.path.splitext(os.path.basename(outfile))[<NUM_LIT:0>] + '<STR_LIT>'<EOL>)<EOL>finder_png = fits_finder_chart(<EOL>field_fitsfile,<EOL>finder_outfile,<EOL>wcsfrom=field_wcsfrom,<EOL>scale=field_scale,<EOL>stretch=field_stretch,<EOL>colormap=field_colormap,<EOL>findersize=field_findersize,<EOL>overlay_ra=objra,<EOL>overlay_decl=objdecl,<EOL>overlay_pltopts=field_pltopts,<EOL>overlay_zoomcontain=field_zoomcontain,<EOL>grid=field_grid,<EOL>gridcolor=field_gridcolor<EOL>)<EOL>if finder_png is not None:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % finder_png)<EOL><DEDENT><DEDENT><DEDENT>with open(outfile,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(lclistdict, outfd, protocol=pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>LOGINFO('<STR_LIT>' % outfile)<EOL>return outfile<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>' % (basedir, fileglob))<EOL>return None<EOL><DEDENT>

This generates a light curve catalog for all light curves in a directory.

    Given a base directory where all the files are, and a light curve format,
    this will find all light curves, pull out the keys in each lcdict requested
    in the `columns` kwarg for each object, and write them to the requested
    output pickle file. These keys should be pointers to scalar values
    (i.e. something like `objectinfo.ra` is OK, but something like 'times' won't
    work because it's a vector).

    Generally, this works with light curve reading functions that produce
    lcdicts as detailed in the docstring for `lcproc.register_lcformat`. Once
    you've registered your light curve reader functions using the
    `lcproc.register_lcformat` function, pass in the `formatkey` associated with
    your light curve format, and this function will be able to read all light
    curves in that format as well as the object information stored in their
    `objectinfo` dict.

    Parameters
    ----------

    basedir : str or list of str
        If this is a str, points to a single directory to search for light
        curves. If this is a list of str, it must be a list of directories to
        search for light curves. All of these will be searched to find light
        curve files matching either your light curve format's default fileglob
        (when you registered your LC format), or a specific fileglob that you
        can pass in using the `fileglob` kwargh here. If the `recursive` kwarg
        is set, the provided directories will be searched recursively.

        If `use_list_of_filenames` is not None, it will override this argument
        and the function will take those light curves as the list of files it
        must process instead of whatever is specified in `basedir`.

    outfile : str
        This is the name of the output file to write. This will be a pickle
        file, so a good convention to use for this name is something like
        'my-lightcurve-catalog.pkl'.

    use_list_of_filenames : list of str or None
        Use this kwarg to override whatever is provided in `basedir` and
        directly pass in a list of light curve files to process. This can speed
        up this function by a lot because no searches on disk will be performed
        to find light curve files matching `basedir` and `fileglob`.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    fileglob : str or None
        If provided, is a string that is a valid UNIX filename glob. Used to
        override the default fileglob for this LC format when searching for
        light curve files in `basedir`.

    recursive : bool
        If True, the directories specified in `basedir` will be searched
        recursively for all light curve files that match the default fileglob
        for this LC format or a specific one provided in `fileglob`.

    columns : list of str
        This is a list of keys in the lcdict produced by your light curve reader
        function that contain object information, which will be extracted and
        put into the output light curve catalog. It's highly recommended that
        your LC reader function produce a lcdict that contains at least the
        default keys shown here.

        The lcdict keys to extract are specified by using an address scheme:

        - First level dict keys can be specified directly:
          e.g., 'objectid' will extract lcdict['objectid']
        - Keys at other levels can be specified by using a period to indicate
          the level:

          - e.g., 'objectinfo.ra' will extract lcdict['objectinfo']['ra']
          - e.g., 'objectinfo.varinfo.features.stetsonj' will extract
            lcdict['objectinfo']['varinfo']['features']['stetsonj']

    makecoordindex : list of two str or None
        This is used to specify which lcdict keys contain the right ascension
        and declination coordinates for this object. If these are provided, the
        output light curve catalog will have a kdtree built on all object
        coordinates, which enables fast spatial searches and cross-matching to
        external catalogs by `checkplot` and `lcproc` functions.

    field_fitsfile : str or None
        If this is not None, it should be the path to a FITS image containing
        the objects these light curves are for. If this is provided,
        `make_lclist` will use the WCS information in the FITS itself if
        `field_wcsfrom` is None (or from a WCS header file pointed to by
        `field_wcsfrom`) to obtain x and y pixel coordinates for all of the
        objects in the field. A finder chart will also be made using
        `astrobase.plotbase.fits_finder_chart` using the corresponding
        `field_scale`, `_stretch`, `_colormap`, `_findersize`, `_pltopts`,
        `_grid`, and `_gridcolors` kwargs for that function, reproduced here to
        enable customization of the finder chart plot.

    field_wcsfrom : str or None
        If `wcsfrom` is None, the WCS to transform the RA/Dec to pixel x/y will
        be taken from the FITS header of `fitsfile`. If this is not None, it
        must be a FITS or similar file that contains a WCS header in its first
        extension.

    field_scale : astropy.visualization.Interval object
        `scale` sets the normalization for the FITS pixel values. This is an
        astropy.visualization Interval object.
        See http://docs.astropy.org/en/stable/visualization/normalization.html
        for details on `scale` and `stretch` objects.

    field_stretch : astropy.visualization.Stretch object
        `stretch` sets the stretch function for mapping FITS pixel values to
        output pixel values. This is an astropy.visualization Stretch object.
        See http://docs.astropy.org/en/stable/visualization/normalization.html
        for details on `scale` and `stretch` objects.

    field_colormap : matplotlib Colormap object
        `colormap` is a matplotlib color map object to use for the output image.

    field_findersize : None or tuple of two ints
        If `findersize` is None, the output image size will be set by the NAXIS1
        and NAXIS2 keywords in the input `fitsfile` FITS header. Otherwise,
        `findersize` must be a tuple with the intended x and y size of the image
        in inches (all output images will use a DPI = 100).

    field_pltopts : dict
        `field_pltopts` controls how the overlay points will be plotted. This
        a dict with standard matplotlib marker, etc. kwargs as key-val pairs,
        e.g. 'markersize', 'markerfacecolor', etc. The default options make red
        outline circles at the location of each object in the overlay.

    field_grid : bool
        `grid` sets if a grid will be made on the output image.

    field_gridcolor : str
        `gridcolor` sets the color of the grid lines. This is a usual matplotib
        color spec string.

    field_zoomcontain : bool
        `field_zoomcontain` controls if the finder chart will be zoomed to
        just contain the overlayed points. Everything outside the footprint of
        these points will be discarded.

    maxlcs : int or None
        This sets how many light curves to process in the input LC list
        generated by searching for LCs in `basedir` or in the list provided as
        `use_list_of_filenames`.

    nworkers : int
        This sets the number of parallel workers to launch to collect
        information from the light curves.

    Returns
    -------

    str
        Returns the path to the generated light curve catalog pickle file.


with open(lc_catalog,'<STR_LIT:rb>') as infd:<EOL><INDENT>lclist = pickle.load(infd)<EOL><DEDENT>xmatch_matching_index = np.full_like(lclist['<STR_LIT>'][objectidcol],<EOL>False,<EOL>dtype=np.bool)<EOL>conesearch_matching_index = np.full_like(lclist['<STR_LIT>'][objectidcol],<EOL>False,<EOL>dtype=np.bool)<EOL>ext_matches = []<EOL>ext_matching_objects = []<EOL>if (xmatchexternal and<EOL>isinstance(xmatchexternal, str) and<EOL>os.path.exists(xmatchexternal)):<EOL><INDENT>try:<EOL><INDENT>extcat = np.genfromtxt(xmatchexternal,<EOL>usecols=externalcolnums,<EOL>delimiter=externalcolsep,<EOL>names=externalcolnames,<EOL>dtype=externalcoldtypes,<EOL>comments=externalcommentchar)<EOL>ext_cosdecl = np.cos(np.radians(extcat['<STR_LIT>']))<EOL>ext_sindecl = np.sin(np.radians(extcat['<STR_LIT>']))<EOL>ext_cosra = np.cos(np.radians(extcat['<STR_LIT>']))<EOL>ext_sinra = np.sin(np.radians(extcat['<STR_LIT>']))<EOL>ext_xyz = np.column_stack((ext_cosra*ext_cosdecl,<EOL>ext_sinra*ext_cosdecl,<EOL>ext_sindecl))<EOL>ext_xyzdist = <NUM_LIT> * np.sin(np.radians(xmatchdistarcsec/<NUM_LIT>)/<NUM_LIT>)<EOL>our_kdt = lclist['<STR_LIT>']<EOL>ext_kdt = sps.cKDTree(ext_xyz)<EOL>extkd_matchinds = ext_kdt.query_ball_tree(our_kdt, ext_xyzdist)<EOL>for extind, mind in enumerate(extkd_matchinds):<EOL><INDENT>if len(mind) > <NUM_LIT:0>:<EOL><INDENT>ext_matches.append(mind[<NUM_LIT:0>])<EOL>ext_matching_objects.append(extcat[extind])<EOL><DEDENT><DEDENT>ext_matches = np.array(ext_matches)<EOL>if ext_matches.size > <NUM_LIT:0>:<EOL><INDENT>xmatch_matching_index[ext_matches] = True<EOL>LOGINFO('<STR_LIT>' %<EOL>(xmatchexternal, xmatchdistarcsec, ext_matches.size))<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>"<EOL>"<STR_LIT>" % xmatchexternal)<EOL>return None, None, None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' %<EOL>repr(xmatchexternal))<EOL>raise<EOL><DEDENT><DEDENT>if (conesearch and<EOL>isinstance(conesearch, (list, tuple)) and<EOL>len(conesearch) == <NUM_LIT:3>):<EOL><INDENT>try:<EOL><INDENT>racenter, declcenter, searchradius = conesearch<EOL>cosdecl = np.cos(np.radians(declcenter))<EOL>sindecl = np.sin(np.radians(declcenter))<EOL>cosra = np.cos(np.radians(racenter))<EOL>sinra = np.sin(np.radians(racenter))<EOL>xyzdist = <NUM_LIT> * np.sin(np.radians(searchradius)/<NUM_LIT>)<EOL>our_kdt = lclist['<STR_LIT>']<EOL>kdtindices = our_kdt.query_ball_point([cosra*cosdecl,<EOL>sinra*cosdecl,<EOL>sindecl],<EOL>xyzdist,<EOL>n_jobs=conesearchworkers)<EOL>if kdtindices and len(kdtindices) > <NUM_LIT:0>:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(searchradius, racenter, declcenter, len(kdtindices)))<EOL>matchingind = kdtindices<EOL>conesearch_matching_index[np.array(matchingind)] = True<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>"<EOL>"<STR_LIT>" %<EOL>(searchradius, racenter, declcenter, len(kdtindices)))<EOL>return None, None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>'<EOL>'<STR_LIT>' % lc_catalog)<EOL>raise<EOL><DEDENT><DEDENT>allfilterinds = []<EOL>if columnfilters and isinstance(columnfilters, list):<EOL><INDENT>for cfilt in columnfilters:<EOL><INDENT>try:<EOL><INDENT>fcol, foperator, foperand = cfilt.split('<STR_LIT:|>')<EOL>foperator = FILTEROPS[foperator]<EOL>filterstr = (<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>) % (fcol, fcol, foperator, foperand)<EOL>filterind = eval(filterstr)<EOL>ngood = lclist['<STR_LIT>'][objectidcol][filterind].size<EOL>LOGINFO('<STR_LIT>' % (cfilt, ngood))<EOL>allfilterinds.append(filterind)<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>'<EOL>% cfilt)<EOL>LOGWARNING('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>if (xmatchexternal or conesearch or columnfilters):<EOL><INDENT>filterstack = []<EOL>if xmatchexternal:<EOL><INDENT>filterstack.append(xmatch_matching_index)<EOL><DEDENT>if conesearch:<EOL><INDENT>filterstack.append(conesearch_matching_index)<EOL><DEDENT>if columnfilters:<EOL><INDENT>filterstack.extend(allfilterinds)<EOL><DEDENT>finalfilterind = np.column_stack(filterstack)<EOL>finalfilterind = np.all(finalfilterind, axis=<NUM_LIT:1>)<EOL>filteredobjectids = lclist['<STR_LIT>'][objectidcol][finalfilterind]<EOL>filteredlcfnames = lclist['<STR_LIT>']['<STR_LIT>'][finalfilterind]<EOL><DEDENT>else:<EOL><INDENT>filteredobjectids = lclist['<STR_LIT>'][objectidcol]<EOL>filteredlcfnames = lclist['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>if field_fitsfile is not None and os.path.exists(field_fitsfile):<EOL><INDENT>matching_ra = lclist['<STR_LIT>'][racol][finalfilterind]<EOL>matching_decl = lclist['<STR_LIT>'][declcol][finalfilterind]<EOL>matching_postfix = []<EOL>if xmatchexternal is not None:<EOL><INDENT>matching_postfix.append(<EOL>'<STR_LIT>' %<EOL>os.path.splitext(os.path.basename(xmatchexternal))[<NUM_LIT:0>]<EOL>)<EOL><DEDENT>if conesearch is not None:<EOL><INDENT>matching_postfix.append('<STR_LIT>' %<EOL>tuple(conesearch))<EOL><DEDENT>if columnfilters is not None:<EOL><INDENT>for cfi, cf in enumerate(columnfilters):<EOL><INDENT>if cfi == <NUM_LIT:0>:<EOL><INDENT>matching_postfix.append('<STR_LIT>' %<EOL>tuple(cf.split('<STR_LIT:|>')))<EOL><DEDENT>else:<EOL><INDENT>matching_postfix.append('<STR_LIT>' %<EOL>tuple(cf.split('<STR_LIT:|>')))<EOL><DEDENT><DEDENT><DEDENT>if len(matching_postfix) > <NUM_LIT:0>:<EOL><INDENT>matching_postfix = '<STR_LIT>' % '<STR_LIT:_>'.join(matching_postfix)<EOL><DEDENT>else:<EOL><INDENT>matching_postfix = '<STR_LIT>'<EOL><DEDENT>finder_outfile = os.path.join(<EOL>os.path.dirname(lc_catalog),<EOL>'<STR_LIT>' %<EOL>(os.path.splitext(os.path.basename(lc_catalog))[<NUM_LIT:0>],<EOL>matching_postfix)<EOL>)<EOL>finder_png = fits_finder_chart(<EOL>field_fitsfile,<EOL>finder_outfile,<EOL>wcsfrom=field_wcsfrom,<EOL>scale=field_scale,<EOL>stretch=field_stretch,<EOL>colormap=field_colormap,<EOL>findersize=field_findersize,<EOL>overlay_ra=matching_ra,<EOL>overlay_decl=matching_decl,<EOL>overlay_pltopts=field_pltopts,<EOL>field_zoomcontain=field_zoomcontain,<EOL>grid=field_grid,<EOL>gridcolor=field_gridcolor<EOL>)<EOL>if finder_png is not None:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % finder_png)<EOL><DEDENT><DEDENT>if copylcsto is not None:<EOL><INDENT>if not os.path.exists(copylcsto):<EOL><INDENT>os.mkdir(copylcsto)<EOL><DEDENT>if TQDM:<EOL><INDENT>lciter = tqdm(filteredlcfnames)<EOL><DEDENT>else:<EOL><INDENT>lciter = filteredlcfnames<EOL><DEDENT>LOGINFO('<STR_LIT>' % copylcsto)<EOL>for lc in lciter:<EOL><INDENT>shutil.copy(lc, copylcsto)<EOL><DEDENT><DEDENT>LOGINFO('<STR_LIT>' % filteredobjectids.size)<EOL>if xmatchexternal and len(ext_matching_objects) > <NUM_LIT:0>:<EOL><INDENT>return filteredlcfnames, filteredobjectids, ext_matching_objects<EOL><DEDENT>else:<EOL><INDENT>return filteredlcfnames, filteredobjectids<EOL><DEDENT>

This is used to perform cone-search, cross-match, and column-filter
    operations on a light curve catalog generated by `make_lclist`.

    Uses the output of `make_lclist` above. This function returns a list of
    light curves matching various criteria specified by the `xmatchexternal`,
    `conesearch`, and `columnfilters kwargs`. Use this function to generate
    input lists for other lcproc functions,
    e.g. `lcproc.lcvfeatures.parallel_varfeatures`,
    `lcproc.periodfinding.parallel_pf`, and `lcproc.lcbin.parallel_timebin`,
    among others.

    The operations are applied in this order if more than one is specified:
    `xmatchexternal` -> `conesearch` -> `columnfilters`. All results from these
    operations are joined using a logical AND operation.

    Parameters
    ----------

    objectidcol : str
        This is the name of the object ID column in the light curve catalog.

    racol : str
        This is the name of the RA column in the light curve catalog.

    declcol : str
        This is the name of the Dec column in the light curve catalog.

    xmatchexternal : str or None
        If provided, this is the filename of a text file containing objectids,
        ras and decs to match the objects in the light curve catalog to by their
        positions.

    xmatchdistarcsec : float
        This is the distance in arcseconds to use when cross-matching to the
        external catalog in `xmatchexternal`.

    externalcolnums : sequence of int
        This a list of the zero-indexed column numbers of columns to extract
        from the external catalog file.

    externalcolnames : sequence of str
        This is a list of names of columns that will be extracted from the
        external catalog file. This is the same length as
        `externalcolnums`. These must contain the names provided as the
        `objectid`, `ra`, and `decl` column names so this function knows which
        column numbers correspond to those columns and can use them to set up
        the cross-match.

    externalcoldtypes : str
        This is a CSV string containing numpy dtype definitions for all columns
        listed to extract from the external catalog file. The number of dtype
        definitions should be equal to the number of columns to extract.

    externalcolsep : str or None
        The column separator to use when extracting columns from the external
        catalog file. If None, any whitespace between columns is used as the
        separator.

    externalcommentchar : str
        The character indicating that a line in the external catalog file is to
        be ignored.

    conesearch : list of float

        This is used to specify cone-search parameters. It should be a three
        element list:

        [center_ra_deg, center_decl_deg, search_radius_deg]

    conesearchworkers : int
        The number of parallel workers to launch for the cone-search operation.

    columnfilters : list of str

        This is a list of strings indicating any filters to apply on each column
        in the light curve catalog. All column filters are applied in the
        specified sequence and are combined with a logical AND operator. The
        format of each filter string should be:

        '<lc_catalog column>|<operator>|<operand>'

        where:

        - <lc_catalog column> is a column in the lc_catalog pickle file
        - <operator> is one of: 'lt', 'gt', 'le', 'ge', 'eq', 'ne', which
          correspond to the usual operators: <, >, <=, >=, ==, != respectively.
        - <operand> is a float, int, or string.

    field_fitsfile : str or None
        If this is not None, it should be the path to a FITS image containing
        the objects these light curves are for. If this is provided,
        `make_lclist` will use the WCS information in the FITS itself if
        `field_wcsfrom` is None (or from a WCS header file pointed to by
        `field_wcsfrom`) to obtain x and y pixel coordinates for all of the
        objects in the field. A finder chart will also be made using
        `astrobase.plotbase.fits_finder_chart` using the corresponding
        `field_scale`, `_stretch`, `_colormap`, `_findersize`, `_pltopts`,
        `_grid`, and `_gridcolors` kwargs for that function, reproduced here to
        enable customization of the finder chart plot.

    field_wcsfrom : str or None
        If `wcsfrom` is None, the WCS to transform the RA/Dec to pixel x/y will
        be taken from the FITS header of `fitsfile`. If this is not None, it
        must be a FITS or similar file that contains a WCS header in its first
        extension.

    field_scale : astropy.visualization.Interval object
        `scale` sets the normalization for the FITS pixel values. This is an
        astropy.visualization Interval object.
        See http://docs.astropy.org/en/stable/visualization/normalization.html
        for details on `scale` and `stretch` objects.

    field_stretch : astropy.visualization.Stretch object
        `stretch` sets the stretch function for mapping FITS pixel values to
        output pixel values. This is an astropy.visualization Stretch object.
        See http://docs.astropy.org/en/stable/visualization/normalization.html
        for details on `scale` and `stretch` objects.

    field_colormap : matplotlib Colormap object
        `colormap` is a matplotlib color map object to use for the output image.

    field_findersize : None or tuple of two ints
        If `findersize` is None, the output image size will be set by the NAXIS1
        and NAXIS2 keywords in the input `fitsfile` FITS header. Otherwise,
        `findersize` must be a tuple with the intended x and y size of the image
        in inches (all output images will use a DPI = 100).

    field_pltopts : dict
        `field_pltopts` controls how the overlay points will be plotted. This
        a dict with standard matplotlib marker, etc. kwargs as key-val pairs,
        e.g. 'markersize', 'markerfacecolor', etc. The default options make red
        outline circles at the location of each object in the overlay.

    field_grid : bool
        `grid` sets if a grid will be made on the output image.

    field_gridcolor : str
        `gridcolor` sets the color of the grid lines. This is a usual matplotib
        color spec string.

    field_zoomcontain : bool
        `field_zoomcontain` controls if the finder chart will be zoomed to
        just contain the overlayed points. Everything outside the footprint of
        these points will be discarded.

    copylcsto : str
        If this is provided, it is interpreted as a directory target to copy
        all the light curves that match the specified conditions.

    Returns
    -------

    tuple
        Returns a two elem tuple: (matching_object_lcfiles, matching_objectids)
        if conesearch and/or column filters are used. If `xmatchexternal` is
        also used, a three-elem tuple is returned: (matching_object_lcfiles,
        matching_objectids, extcat_matched_objectids).


cpfile, keyspeclist = task<EOL>keystoget = [x[<NUM_LIT:0>] for x in keyspeclist]<EOL>nonesubs = [x[-<NUM_LIT:2>] for x in keyspeclist]<EOL>nansubs = [x[-<NUM_LIT:1>] for x in keyspeclist]<EOL>for i, k in enumerate(keystoget):<EOL><INDENT>thisk = k.split('<STR_LIT:.>')<EOL>if sys.version_info[:<NUM_LIT:2>] < (<NUM_LIT:3>,<NUM_LIT:4>):<EOL><INDENT>thisk = [(int(x) if x.isdigit() else x) for x in thisk]<EOL><DEDENT>else:<EOL><INDENT>thisk = [(int(x) if x.isdecimal() else x) for x in thisk]<EOL><DEDENT>keystoget[i] = thisk<EOL><DEDENT>keystoget.insert(<NUM_LIT:0>,['<STR_LIT>'])<EOL>nonesubs.insert(<NUM_LIT:0>, '<STR_LIT>')<EOL>nansubs.insert(<NUM_LIT:0>,'<STR_LIT>')<EOL>vals = checkplot_infokey_worker((cpfile, keystoget))<EOL>for val, nonesub, nansub, valind in zip(vals, nonesubs,<EOL>nansubs, range(len(vals))):<EOL><INDENT>if val is None:<EOL><INDENT>outval = nonesub<EOL><DEDENT>elif isinstance(val, float) and not np.isfinite(val):<EOL><INDENT>outval = nansub<EOL><DEDENT>elif isinstance(val, (list, tuple)):<EOL><INDENT>outval = '<STR_LIT:U+002CU+0020>'.join(val)<EOL><DEDENT>else:<EOL><INDENT>outval = val<EOL><DEDENT>vals[valind] = outval<EOL><DEDENT>return vals<EOL>

This wraps `checkplotlist.checkplot_infokey_worker`.

    This is used to get the correct dtype for each element in retrieved results.

    Parameters
    ----------

    task : tuple
        task[0] = cpfile
        task[1] = keyspeclist (infokeys kwarg from `add_cpinfo_to_lclist`)

    Returns
    -------

    dict
        All of the requested keys from the checkplot are returned along with
        their values in a dict.


def add_cpinfo_to_lclist(<EOL>checkplots,  <EOL>initial_lc_catalog,<EOL>magcol,  <EOL>outfile,<EOL>checkplotglob='<STR_LIT>',<EOL>infokeys=CPINFO_DEFAULTKEYS,<EOL>nworkers=NCPUS<EOL>):

<EOL>if not isinstance(checkplots, list) and os.path.exists(checkplots):<EOL><INDENT>checkplots = sorted(glob.glob(os.path.join(checkplots, checkplotglob)))<EOL><DEDENT>tasklist = [(cpf, infokeys) for cpf in checkplots]<EOL>with ProcessPoolExecutor(max_workers=nworkers) as executor:<EOL><INDENT>resultfutures = executor.map(_cpinfo_key_worker, tasklist)<EOL><DEDENT>results = [x for x in resultfutures]<EOL>executor.shutdown()<EOL>with open(initial_lc_catalog,'<STR_LIT:rb>') as infd:<EOL><INDENT>lc_catalog = pickle.load(infd)<EOL><DEDENT>catalog_objectids = np.array(lc_catalog['<STR_LIT>']['<STR_LIT>'])<EOL>checkplot_objectids = np.array([x[<NUM_LIT:0>] for x in results])<EOL>extrainfokeys = []<EOL>actualkeys = []<EOL>for keyspec in infokeys:<EOL><INDENT>key, dtype, firstlevel, overwrite_append, nonesub, nansub = keyspec<EOL>if firstlevel:<EOL><INDENT>eik = key<EOL><DEDENT>else:<EOL><INDENT>eik = '<STR_LIT>' % (magcol, key)<EOL><DEDENT>extrainfokeys.append(eik)<EOL>eactual = eik.split('<STR_LIT:.>')<EOL>if not eactual[-<NUM_LIT:1>].isdigit():<EOL><INDENT>if not firstlevel:<EOL><INDENT>eactual = '<STR_LIT:.>'.join([eactual[<NUM_LIT:0>], eactual[-<NUM_LIT:1>]])<EOL><DEDENT>else:<EOL><INDENT>eactual = eactual[-<NUM_LIT:1>]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>elastkey = eactual[-<NUM_LIT:2>]<EOL>if elastkey.endswith('<STR_LIT>'):<EOL><INDENT>elastkey = elastkey[:-<NUM_LIT:2>]<EOL><DEDENT>elif elastkey.endswith('<STR_LIT:s>'):<EOL><INDENT>elastkey = elastkey[:-<NUM_LIT:1>]<EOL><DEDENT>if not firstlevel:<EOL><INDENT>eactual = '<STR_LIT:.>'.join([eactual[<NUM_LIT:0>], elastkey])<EOL><DEDENT>else:<EOL><INDENT>eactual = elastkey<EOL><DEDENT><DEDENT>actualkeys.append(eactual)<EOL>if eactual not in lc_catalog['<STR_LIT>']:<EOL><INDENT>lc_catalog['<STR_LIT>'].append(eactual)<EOL><DEDENT>lc_catalog['<STR_LIT>'][eactual] = []<EOL><DEDENT>for catobj in tqdm(catalog_objectids):<EOL><INDENT>cp_objind = np.where(checkplot_objectids == catobj)<EOL>if len(cp_objind[<NUM_LIT:0>]) > <NUM_LIT:0>:<EOL><INDENT>thiscpinfo = results[cp_objind[<NUM_LIT:0>][<NUM_LIT:0>]]<EOL>thiscpinfo = thiscpinfo[<NUM_LIT:1>:]<EOL>for ekind, ek in enumerate(actualkeys):<EOL><INDENT>lc_catalog['<STR_LIT>'][ek].append(<EOL>thiscpinfo[ekind]<EOL>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for ekind, ek in enumerate(actualkeys):<EOL><INDENT>thiskeyspec = infokeys[ekind]<EOL>nonesub = thiskeyspec[-<NUM_LIT:2>]<EOL>lc_catalog['<STR_LIT>'][ek].append(<EOL>nonesub<EOL>)<EOL><DEDENT><DEDENT><DEDENT>for ek in actualkeys:<EOL><INDENT>lc_catalog['<STR_LIT>'][ek] = np.array(<EOL>lc_catalog['<STR_LIT>'][ek]<EOL>)<EOL><DEDENT>if '<STR_LIT>' in lc_catalog:<EOL><INDENT>if magcol not in lc_catalog['<STR_LIT>']:<EOL><INDENT>lc_catalog['<STR_LIT>'].append(magcol)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>lc_catalog['<STR_LIT>'] = [magcol]<EOL><DEDENT>with open(outfile, '<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(lc_catalog, outfd, protocol=pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>return outfile<EOL>

This adds checkplot info to the initial light curve catalogs generated by
    `make_lclist`.

    This is used to incorporate all the extra info checkplots can have for
    objects back into columns in the light curve catalog produced by
    `make_lclist`. Objects are matched between the checkplots and the light
    curve catalog using their `objectid`. This then allows one to search this
    'augmented' light curve catalog by these extra columns. The 'augmented'
    light curve catalog also forms the basis for search interface provided by
    the LCC-Server.

    The default list of keys that will be extracted from a checkplot and added
    as columns in the initial light curve catalog is listed above in the
    `CPINFO_DEFAULTKEYS` list.

    Parameters
    ----------

    checkplots : str or list
        If this is a str, is interpreted as a directory which will be searched
        for checkplot pickle files using `checkplotglob`. If this is a list, it
        will be interpreted as a list of checkplot pickle files to process.

    initial_lc_catalog : str
        This is the path to the light curve catalog pickle made by
        `make_lclist`.

    magcol : str
        This is used to indicate the light curve magnitude column to extract
        magnitude column specific information. For example, Stetson variability
        indices can be generated using magnitude measurements in separate
        photometric apertures, which appear in separate `magcols` in the
        checkplot. To associate each such feature of the object with its
        specific `magcol`, pass that `magcol` in here. This `magcol` will then
        be added as a prefix to the resulting column in the 'augmented' LC
        catalog, e.g. Stetson J will appear as `magcol1_stetsonj` and
        `magcol2_stetsonj` for two separate magcols.

    outfile : str
        This is the file name of the output 'augmented' light curve catalog
        pickle file that will be written.

    infokeys : list of tuples

        This is a list of keys to extract from the checkplot and some info on
        how this extraction is to be done. Each key entry is a six-element
        tuple of the following form:

        - key name in the checkplot
        - numpy dtype of the value of this key
        - False if key is associated with a magcol or True otherwise
        - False if subsequent updates to the same column name will append to
          existing key values in the output augmented light curve catalog or
          True if these will overwrite the existing key value
        - character to use to substitute a None value of the key in the
          checkplot in the output light curve catalog column
        - character to use to substitute a nan value of the key in the
          checkplot in the output light curve catalog column

        See the `CPFINFO_DEFAULTKEYS` list above for examples.

    nworkers : int
        The number of parallel workers to launch to extract checkplot
        information.

    Returns
    -------

    str
        Returns the path to the generated 'augmented' light curve catalog pickle
        file.


raise KeyboardInterrupt<EOL>

handler function to receive and process a SIGINT