try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(fileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if timecols is None:<EOL><INDENT>timecols = dtimecols<EOL><DEDENT>if magcols is None:<EOL><INDENT>magcols = dmagcols<EOL><DEDENT>if errcols is None:<EOL><INDENT>errcols = derrcols<EOL><DEDENT>lcdict = readerfunc(lcfile)<EOL>if isinstance(lcdict, tuple) and isinstance(lcdict[<NUM_LIT:0>],dict):<EOL><INDENT>lcdict = lcdict[<NUM_LIT:0>]<EOL><DEDENT>fakeobjectid = sha512(npr.bytes(<NUM_LIT:12>)).hexdigest()[-<NUM_LIT:8>:]<EOL>fakelcdict = {<EOL>'<STR_LIT>':fakeobjectid,<EOL>'<STR_LIT>':{'<STR_LIT>':fakeobjectid},<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':{},<EOL>'<STR_LIT>':lcformat,<EOL>}<EOL>if ('<STR_LIT>' in lcdict and<EOL>isinstance(lcdict['<STR_LIT>'], dict)):<EOL><INDENT>objectinfo = lcdict['<STR_LIT>']<EOL>if (not randomizecoords and '<STR_LIT>' in objectinfo and<EOL>objectinfo['<STR_LIT>'] is not None and<EOL>np.isfinite(objectinfo['<STR_LIT>'])):<EOL><INDENT>fakelcdict['<STR_LIT>']['<STR_LIT>'] = objectinfo['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>' % lcfile)<EOL>fakelcdict['<STR_LIT>']['<STR_LIT>'] = npr.random()*<NUM_LIT><EOL><DEDENT>if (not randomizecoords and '<STR_LIT>' in objectinfo and<EOL>objectinfo['<STR_LIT>'] is not None and<EOL>np.isfinite(objectinfo['<STR_LIT>'])):<EOL><INDENT>fakelcdict['<STR_LIT>']['<STR_LIT>'] = objectinfo['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>' % lcfile)<EOL>fakelcdict['<STR_LIT>']['<STR_LIT>'] = npr.random()*<NUM_LIT> - <NUM_LIT><EOL><DEDENT>if ((not randomizemags) and '<STR_LIT>' in objectinfo and<EOL>objectinfo['<STR_LIT>'] is not None and<EOL>np.isfinite(objectinfo['<STR_LIT>'])):<EOL><INDENT>fakelcdict['<STR_LIT>']['<STR_LIT>'] = objectinfo['<STR_LIT>']<EOL><DEDENT>elif ((not randomizemags) and ('<STR_LIT>' in objectinfo and<EOL>objectinfo['<STR_LIT>'] is not None and<EOL>np.isfinite(objectinfo['<STR_LIT>'])) and<EOL>('<STR_LIT>' in objectinfo and<EOL>objectinfo['<STR_LIT>'] is not None and<EOL>np.isfinite(objectinfo['<STR_LIT>'])) and<EOL>('<STR_LIT>' in objectinfo and<EOL>objectinfo['<STR_LIT>'] is not None and<EOL>np.isfinite(objectinfo['<STR_LIT>']))):<EOL><INDENT>LOGWARNING('<STR_LIT>' %<EOL>lcfile)<EOL>fakelcdict['<STR_LIT>']['<STR_LIT>'] = jhk_to_sdssr(<EOL>objectinfo['<STR_LIT>'],<EOL>objectinfo['<STR_LIT>'],<EOL>objectinfo['<STR_LIT>']<EOL>)<EOL><DEDENT>elif randomizemags and magrms:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>' % lcfile)<EOL>magbins = magrms[magcols[<NUM_LIT:0>]]['<STR_LIT>']<EOL>binprobs = magrms[magcols[<NUM_LIT:0>]]['<STR_LIT>']<EOL>magbincenter = npr.choice(magbins,size=<NUM_LIT:1>,p=binprobs)<EOL>chosenmag = (<EOL>npr.random()*((magbincenter+<NUM_LIT>) - (magbincenter-<NUM_LIT>)) +<EOL>(magbincenter-<NUM_LIT>)<EOL>)<EOL>fakelcdict['<STR_LIT>']['<STR_LIT>'] = np.asscalar(chosenmag)<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>' % lcfile)<EOL>fakelcdict['<STR_LIT>']['<STR_LIT>'] = npr.random()*<NUM_LIT> + <NUM_LIT><EOL><DEDENT><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>lcfile)<EOL>fakelcdict['<STR_LIT>']['<STR_LIT>'] = npr.random()*<NUM_LIT><EOL>fakelcdict['<STR_LIT>']['<STR_LIT>'] = npr.random()*<NUM_LIT> - <NUM_LIT><EOL>fakelcdict['<STR_LIT>']['<STR_LIT>'] = npr.random()*<NUM_LIT> + <NUM_LIT><EOL><DEDENT>for tcind, tcol in enumerate(timecols):<EOL><INDENT>if '<STR_LIT:.>' in tcol:<EOL><INDENT>tcolget = tcol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>tcolget = [tcol]<EOL><DEDENT>if tcol not in fakelcdict:<EOL><INDENT>fakelcdict[tcol] = _dict_get(lcdict, tcolget)<EOL>fakelcdict['<STR_LIT>'].append(tcol)<EOL>if tcind == <NUM_LIT:0>:<EOL><INDENT>fakelcdict['<STR_LIT>']['<STR_LIT>'] = fakelcdict[tcol].size<EOL><DEDENT><DEDENT><DEDENT>for mcol in magcols:<EOL><INDENT>if '<STR_LIT:.>' in mcol:<EOL><INDENT>mcolget = mcol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>mcolget = [mcol]<EOL><DEDENT>if mcol not in fakelcdict:<EOL><INDENT>measuredmags = _dict_get(lcdict, mcolget)<EOL>measuredmags = measuredmags[np.isfinite(measuredmags)]<EOL>if (randomizemags and magrms and mcol in magrms and<EOL>'<STR_LIT>' in magrms[mcol] and<EOL>magrms[mcol]['<STR_LIT>'] is not None):<EOL><INDENT>interpfunc = magrms[mcol]['<STR_LIT>']<EOL>lcmad = interpfunc(fakelcdict['<STR_LIT>']['<STR_LIT>'])<EOL>fakelcdict['<STR_LIT>'][mcol] = {<EOL>'<STR_LIT>': fakelcdict['<STR_LIT>']['<STR_LIT>'],<EOL>'<STR_LIT>': lcmad<EOL>}<EOL><DEDENT>else:<EOL><INDENT>if measuredmags.size > <NUM_LIT:9>:<EOL><INDENT>measuredmedian = np.median(measuredmags)<EOL>measuredmad = np.median(<EOL>np.abs(measuredmags - measuredmedian)<EOL>)<EOL>fakelcdict['<STR_LIT>'][mcol] = {'<STR_LIT>':measuredmedian,<EOL>'<STR_LIT>':measuredmad}<EOL><DEDENT>else:<EOL><INDENT>if (magrms and mcol in magrms and<EOL>'<STR_LIT>' in magrms[mcol] and<EOL>magrms[mcol]['<STR_LIT>'] is not None):<EOL><INDENT>LOGWARNING(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % lcfile<EOL>)<EOL>interpfunc = magrms[mcol]['<STR_LIT>']<EOL>lcmad = interpfunc(fakelcdict['<STR_LIT>']['<STR_LIT>'])<EOL>fakelcdict['<STR_LIT>'][mcol] = {<EOL>'<STR_LIT>': fakelcdict['<STR_LIT>']['<STR_LIT>'],<EOL>'<STR_LIT>': lcmad<EOL>}<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% lcfile<EOL>)<EOL>fakelcdict['<STR_LIT>'][mcol] = {<EOL>'<STR_LIT>':fakelcdict['<STR_LIT>']['<STR_LIT>'],<EOL>'<STR_LIT>':npr.random()*(<NUM_LIT:0.1> - <NUM_LIT>) + <NUM_LIT><EOL>}<EOL><DEDENT><DEDENT><DEDENT>fakelcdict[mcol] = np.full_like(_dict_get(lcdict, mcolget), <NUM_LIT:0.0>)<EOL>fakelcdict['<STR_LIT>'].append(mcol)<EOL><DEDENT><DEDENT>for mcol, ecol in zip(magcols, errcols):<EOL><INDENT>if '<STR_LIT:.>' in ecol:<EOL><INDENT>ecolget = ecol.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>ecolget = [ecol]<EOL><DEDENT>if ecol not in fakelcdict:<EOL><INDENT>measurederrs = _dict_get(lcdict, ecolget)<EOL>measurederrs = measurederrs[np.isfinite(measurederrs)]<EOL>if (randomizemags and magrms and mcol in magrms and<EOL>'<STR_LIT>' in magrms[mcol] and<EOL>magrms[mcol]['<STR_LIT>'] is not None):<EOL><INDENT>interpfunc = magrms[mcol]['<STR_LIT>']<EOL>lcmad = interpfunc(fakelcdict['<STR_LIT>']['<STR_LIT>'])<EOL>fakelcdict['<STR_LIT>'][ecol] = {<EOL>'<STR_LIT>': lcmad,<EOL>'<STR_LIT>': <NUM_LIT:0.1>*lcmad<EOL>}<EOL><DEDENT>else:<EOL><INDENT>if measurederrs.size > <NUM_LIT:9>:<EOL><INDENT>measuredmedian = np.median(measurederrs)<EOL>measuredmad = np.median(<EOL>np.abs(measurederrs - measuredmedian)<EOL>)<EOL>fakelcdict['<STR_LIT>'][ecol] = {'<STR_LIT>':measuredmedian,<EOL>'<STR_LIT>':measuredmad}<EOL><DEDENT>else:<EOL><INDENT>if (magrms and mcol in magrms and<EOL>'<STR_LIT>' in magrms[mcol] and<EOL>magrms[mcol]['<STR_LIT>'] is not None):<EOL><INDENT>LOGWARNING(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % lcfile<EOL>)<EOL>interpfunc = magrms[mcol]['<STR_LIT>']<EOL>lcmad = interpfunc(fakelcdict['<STR_LIT>']['<STR_LIT>'])<EOL>fakelcdict['<STR_LIT>'][ecol] = {<EOL>'<STR_LIT>': lcmad,<EOL>'<STR_LIT>': <NUM_LIT:0.1>*lcmad<EOL>}<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % lcfile<EOL>)<EOL>fakelcdict['<STR_LIT>'][ecol] = {<EOL>'<STR_LIT>':npr.random()*(<NUM_LIT> - <NUM_LIT>) + <NUM_LIT>,<EOL>'<STR_LIT>':npr.random()*(<NUM_LIT> - <NUM_LIT>) + <NUM_LIT><EOL>}<EOL><DEDENT><DEDENT><DEDENT>fakelcdict[ecol] = np.full_like(_dict_get(lcdict, ecolget), <NUM_LIT:0.0>)<EOL>fakelcdict['<STR_LIT>'].append(ecol)<EOL><DEDENT><DEDENT>fakelcdict['<STR_LIT>'] = timecols<EOL>fakelcdict['<STR_LIT>'] = magcols<EOL>fakelcdict['<STR_LIT>'] = errcols<EOL>fakelcfname = '<STR_LIT>' % fakelcdict['<STR_LIT>']<EOL>fakelcfpath = os.path.abspath(os.path.join(outdir, fakelcfname))<EOL>with open(fakelcfpath,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(fakelcdict, outfd, protocol=pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>LOGINFO('<STR_LIT>' % (lcfile, fakelcfpath))<EOL>return (fakelcfpath, fakelcdict['<STR_LIT>'],<EOL>fakelcdict['<STR_LIT>'], fakelcdict['<STR_LIT>'])<EOL>

This preprocesses an input real LC and sets it up to be a fake LC.

    Parameters
    ----------

    lcfile : str
        This is an input light curve file that will be used to copy over the
        time-base. This will be used to generate the time-base for fake light
        curves to provide a realistic simulation of the observing window
        function.

    outdir : str
        The output directory where the the fake light curve will be written.

    magrms : dict
        This is a dict containing the SDSS r mag-RMS (SDSS rmag-MAD preferably)
        relation based on all light curves that the input lcfile is from. This
        will be used to generate the median mag and noise corresponding to the
        magnitude chosen for this fake LC.

    randomizemags : bool
        If this is True, then a random mag between the first and last magbin in
        magrms will be chosen as the median mag for this light curve. This
        choice will be weighted by the mag bin probability obtained from the
        magrms kwarg. Otherwise, the median mag will be taken from the input
        lcfile's lcdict['objectinfo']['sdssr'] key or a transformed SDSS r mag
        generated from the input lcfile's lcdict['objectinfo']['jmag'],
        ['hmag'], and ['kmag'] keys. The magrms relation for each magcol will be
        used to generate Gaussian noise at the correct level for the magbin this
        light curve's median mag falls into.

    randomizecoords : bool
        If this is True, will randomize the RA, DEC of the output fake object
        and not copy over the RA/DEC from the real input object.

    lcformat : str
        This is the `formatkey` associated with your input real light curve
        format, which you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curve specified in `lcfile`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols : list of str or None
        The timecol keys to use from the input lcdict in generating the fake
        light curve. Fake LCs will be generated for each each
        timecol/magcol/errcol combination in the input light curve.

    magcols : list of str or None
        The magcol keys to use from the input lcdict in generating the fake
        light curve. Fake LCs will be generated for each each
        timecol/magcol/errcol combination in the input light curve.

    errcols : list of str or None
        The errcol keys to use from the input lcdict in generating the fake
        light curve. Fake LCs will be generated for each each
        timecol/magcol/errcol combination in the input light curve.

    Returns
    -------

    tuple
        A tuple of the following form is returned::

            (fakelc_fpath,
             fakelc_lcdict['columns'],
             fakelc_lcdict['objectinfo'],
             fakelc_lcdict['moments'])


lcfile, outdir, kwargs = task<EOL>try:<EOL><INDENT>fakelcresults = make_fakelc(<EOL>lcfile,<EOL>outdir,<EOL>**kwargs<EOL>)<EOL>return fakelcresults<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' % lcfile)<EOL>return None<EOL><DEDENT>

This wraps `process_fakelc` for `make_fakelc_collection` below.

Parameters
----------

task : tuple
    This is of the form::

        task[0] = lcfile
        task[1] = outdir
        task[2] = magrms
        task[3] = dict with keys: {'lcformat', 'timecols', 'magcols',
                                   'errcols', 'randomizeinfo'}

Returns
-------

tuple
    This returns a tuple of the form::

        (fakelc_fpath,
         fakelc_lcdict['columns'],
         fakelc_lcdict['objectinfo'],
         fakelc_lcdict['moments'])


try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(fileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if timecols is None:<EOL><INDENT>timecols = dtimecols<EOL><DEDENT>if magcols is None:<EOL><INDENT>magcols = dmagcols<EOL><DEDENT>if errcols is None:<EOL><INDENT>errcols = derrcols<EOL><DEDENT>if not isinstance(lclist, np.ndarray):<EOL><INDENT>lclist = np.array(lclist)<EOL><DEDENT>chosenlcs = npr.choice(lclist, maxlcs, replace=False)<EOL>fakelcdir = os.path.join(simbasedir, '<STR_LIT>')<EOL>if not os.path.exists(fakelcdir):<EOL><INDENT>os.makedirs(fakelcdir)<EOL><DEDENT>if isinstance(magrmsfrom, str) and os.path.exists(magrmsfrom):<EOL><INDENT>with open(magrmsfrom,'<STR_LIT:rb>') as infd:<EOL><INDENT>xmagrms = pickle.load(infd)<EOL><DEDENT><DEDENT>elif isinstance(magrmsfrom, dict):<EOL><INDENT>xmagrms = magrmsfrom<EOL><DEDENT>magrms = {}<EOL>for magcol in magcols:<EOL><INDENT>if (magcol in xmagrms and<EOL>'<STR_LIT>' in xmagrms[magcol] and<EOL>'<STR_LIT>' in xmagrms[magcol]):<EOL><INDENT>magrms[magcol] = {<EOL>'<STR_LIT>':np.array(<EOL>xmagrms[magcol]['<STR_LIT>']<EOL>),<EOL>'<STR_LIT>':np.array(<EOL>xmagrms[magcol]['<STR_LIT>']<EOL>),<EOL>}<EOL>interpolated_magmad = spi.interp1d(<EOL>xmagrms[magcol]['<STR_LIT>'],<EOL>xmagrms[magcol]['<STR_LIT>'],<EOL>kind=magrms_interpolate,<EOL>fill_value=magrms_fillvalue,<EOL>)<EOL>magrms[magcol]['<STR_LIT>'] = interpolated_magmad<EOL>magbins = np.array(xmagrms[magcol]['<STR_LIT>'])<EOL>bincounts = np.array(xmagrms[magcol]['<STR_LIT>'])<EOL>binprobs = bincounts/np.sum(bincounts)<EOL>magrms[magcol]['<STR_LIT>'] = binprobs<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>' % magcol)<EOL>magrms[magcol] = {<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':None,<EOL>}<EOL><DEDENT><DEDENT>tasks = [(x, fakelcdir, {'<STR_LIT>':lcformat,<EOL>'<STR_LIT>':timecols,<EOL>'<STR_LIT>':magcols,<EOL>'<STR_LIT>':errcols,<EOL>'<STR_LIT>':magrms,<EOL>'<STR_LIT>':randomizemags,<EOL>'<STR_LIT>':randomizecoords})<EOL>for x in chosenlcs]<EOL>fakeresults = [collection_worker(task) for task in tasks]<EOL>fakedb = {'<STR_LIT>':simbasedir,<EOL>'<STR_LIT>':lcformat,<EOL>'<STR_LIT>':timecols,<EOL>'<STR_LIT>':magcols,<EOL>'<STR_LIT>':errcols,<EOL>'<STR_LIT>':magsarefluxes}<EOL>fobjects, fpaths = [], []<EOL>fras, fdecls, fndets = [], [], []<EOL>fmags, fmagmads = [], []<EOL>ferrmeds, ferrmads = [], []<EOL>totalvars = <NUM_LIT:0><EOL>isvariableind = npr.randint(<NUM_LIT:0>,high=len(fakeresults), size=maxvars)<EOL>isvariable = np.full(len(fakeresults), False, dtype=np.bool)<EOL>isvariable[isvariableind] = True<EOL>fakedb['<STR_LIT>'] = isvariable<EOL>LOGINFO('<STR_LIT>' % maxvars)<EOL>vartypeind = npr.randint(<NUM_LIT:0>,high=len(vartypes), size=maxvars)<EOL>vartypearr = np.array([vartypes[x] for x in vartypeind])<EOL>fakedb['<STR_LIT>'] = vartypearr<EOL>for vt in sorted(vartypes):<EOL><INDENT>LOGINFO('<STR_LIT>' % (vt, vartypearr[vartypearr == vt].size))<EOL><DEDENT>LOGINFO('<STR_LIT>')<EOL>for fr in fakeresults:<EOL><INDENT>if fr is not None:<EOL><INDENT>fpath, fcols, finfo, fmoments = fr<EOL>fobjects.append(finfo['<STR_LIT>'])<EOL>fpaths.append(fpath)<EOL>fras.append(finfo['<STR_LIT>'])<EOL>fdecls.append(finfo['<STR_LIT>'])<EOL>fndets.append(finfo['<STR_LIT>'])<EOL>fmags.append(finfo['<STR_LIT>'])<EOL>fmagmads.append([fmoments[x]['<STR_LIT>'] for x in magcols])<EOL>ferrmeds.append([fmoments[x]['<STR_LIT>'] for x in errcols])<EOL>ferrmads.append([fmoments[x]['<STR_LIT>'] for x in errcols])<EOL><DEDENT><DEDENT>fobjects = np.array(fobjects)<EOL>fpaths = np.array(fpaths)<EOL>fras = np.array(fras)<EOL>fdecls = np.array(fdecls)<EOL>fndets = np.array(fndets)<EOL>fmags = np.array(fmags)<EOL>fmagmads = np.array(fmagmads)<EOL>ferrmeds = np.array(ferrmeds)<EOL>ferrmads = np.array(ferrmads)<EOL>fakedb['<STR_LIT>'] = fobjects<EOL>fakedb['<STR_LIT>'] = fpaths<EOL>fakedb['<STR_LIT>'] = fras<EOL>fakedb['<STR_LIT>'] = fdecls<EOL>fakedb['<STR_LIT>'] = fndets<EOL>fakedb['<STR_LIT>'] = fmags<EOL>fakedb['<STR_LIT>'] = fmagmads<EOL>fakedb['<STR_LIT>'] = ferrmeds<EOL>fakedb['<STR_LIT>'] = ferrmads<EOL>fakedb['<STR_LIT>'] = magrms<EOL>dboutfname = os.path.join(simbasedir,'<STR_LIT>')<EOL>with open(dboutfname, '<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(fakedb, outfd)<EOL><DEDENT>LOGINFO('<STR_LIT>' % (len(fakeresults), simbasedir))<EOL>LOGINFO('<STR_LIT>' % dboutfname)<EOL>return dboutfname<EOL>

This prepares light curves for the recovery sim.

    Collects light curves from `lclist` using a uniform sampling among
    them. Copies them to the `simbasedir`, zeroes out their mags and errs but
    keeps their time bases, also keeps their RMS and median mags for later
    use. Calculates the mag-rms relation for the entire collection and writes
    that to the `simbasedir` as well.

    The purpose of this function is to copy over the time base and mag-rms
    relation of an existing light curve collection to use it as the basis for a
    variability recovery simulation.

    This returns a pickle written to the `simbasedir` that contains all the
    information for the chosen ensemble of fake light curves and writes all
    generated light curves to the `simbasedir/lightcurves` directory. Run the
    `add_variability_to_fakelc_collection` function after this function to add
    variability of the specified type to these generated light curves.

    Parameters
    ----------

    lclist : list of str
        This is a list of existing project light curves. This can be generated
        from :py:func:`astrobase.lcproc.catalogs.make_lclist` or similar.

    simbasedir : str
        This is the directory to where the fake light curves and their
        information will be copied to.

    magrmsfrom : str or dict
        This is used to generate magnitudes and RMSes for the objects in the
        output collection of fake light curves. This arg is either a string
        pointing to an existing pickle file that must contain a dict or a dict
        variable that MUST have the following key-vals at a minimum::

            {'<magcol1_name>': {
                  'binned_sdssr_median': array of median mags for each magbin
                  'binned_lcmad_median': array of LC MAD values per magbin
             },
             '<magcol2_name>': {
                  'binned_sdssr_median': array of median mags for each magbin
                  'binned_lcmad_median': array of LC MAD values per magbin
             },
             .
             .
             ...}

        where `magcol1_name`, etc. are the same as the `magcols` listed in the
        magcols kwarg (or the default magcols for the specified
        lcformat). Examples of the magrmsfrom dict (or pickle) required can be
        generated by the
        :py:func:`astrobase.lcproc.varthreshold.variability_threshold` function.

    magrms_interpolate,magrms_fillvalue : str
        These are arguments that will be passed directly to the
        scipy.interpolate.interp1d function to generate interpolating functions
        for the mag-RMS relation. See:

        https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.interp1d.html

        for details.

    maxlcs : int
        This is the total number of light curves to choose from `lclist` and
        generate as fake LCs.

    maxvars : int
        This is the total number of fake light curves that will be marked as
        variable.

    vartypes : list of str
        This is a list of variable types to put into the collection. The
        vartypes for each fake variable star will be chosen uniformly from this
        list.

    lcformat : str
        This is the `formatkey` associated with your input real light curves'
        format, which you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `lclist`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols : list of str or None
        The timecol keys to use from the input lcdict in generating the fake
        light curve. Fake LCs will be generated for each each
        timecol/magcol/errcol combination in the input light curves.

    magcols : list of str or None
        The magcol keys to use from the input lcdict in generating the fake
        light curve. Fake LCs will be generated for each each
        timecol/magcol/errcol combination in the input light curves.

    errcols : list of str or None
        The errcol keys to use from the input lcdict in generating the fake
        light curve. Fake LCs will be generated for each each
        timecol/magcol/errcol combination in the input light curves.

    Returns
    -------

    str
        Returns the string file name of a pickle containing all of the
        information for the fake LC collection that has been generated.


<EOL>lcdict = _read_pklc(fakelcfile)<EOL>if ('<STR_LIT>' in lcdict and<EOL>'<STR_LIT>' in lcdict and<EOL>not overwrite):<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(fakelcfile, lcdict['<STR_LIT>'],<EOL>repr(lcdict['<STR_LIT>'])))<EOL>return None<EOL><DEDENT>timecols, magcols, errcols = (lcdict['<STR_LIT>'],<EOL>lcdict['<STR_LIT>'],<EOL>lcdict['<STR_LIT>'])<EOL>if vartype in VARTYPE_LCGEN_MAP:<EOL><INDENT>vargenfunc = VARTYPE_LCGEN_MAP[vartype]<EOL><DEDENT>elif vartype is None:<EOL><INDENT>vargenfunc = None<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>' %<EOL>(vartype, repr(list(VARTYPE_LCGEN_MAP.keys()))))<EOL>return None<EOL><DEDENT>if vargenfunc is not None:<EOL><INDENT>if (override_paramdists is not None and<EOL>isinstance(override_paramdists,dict)):<EOL><INDENT>variablelc = vargenfunc(lcdict[timecols[<NUM_LIT:0>]],<EOL>paramdists=override_paramdists,<EOL>magsarefluxes=magsarefluxes)<EOL><DEDENT>else:<EOL><INDENT>variablelc = vargenfunc(lcdict[timecols[<NUM_LIT:0>]],<EOL>magsarefluxes=magsarefluxes)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>variablelc = {'<STR_LIT>':None,<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':lcdict[timecols[<NUM_LIT:0>]],<EOL>'<STR_LIT>':np.full_like(lcdict[timecols[<NUM_LIT:0>]], <NUM_LIT:0.0>),<EOL>'<STR_LIT>':np.full_like(lcdict[timecols[<NUM_LIT:0>]], <NUM_LIT:0.0>)}<EOL><DEDENT>for tcol, mcol, ecol in zip(timecols, magcols, errcols):<EOL><INDENT>times, mags, errs = lcdict[tcol], lcdict[mcol], lcdict[ecol]<EOL>mag_median = lcdict['<STR_LIT>'][mcol]['<STR_LIT>']<EOL>mag_mad = lcdict['<STR_LIT>'][mcol]['<STR_LIT>']<EOL>mag_rms = mag_mad*<NUM_LIT><EOL>err_median = lcdict['<STR_LIT>'][ecol]['<STR_LIT>']<EOL>err_mad = lcdict['<STR_LIT>'][ecol]['<STR_LIT>']<EOL>err_rms = err_mad*<NUM_LIT><EOL>magnoise = npr.normal(size=variablelc['<STR_LIT>'].size)*mag_rms<EOL>errnoise = npr.normal(size=variablelc['<STR_LIT>'].size)*err_rms<EOL>finalmags = mag_median + (variablelc['<STR_LIT>'] + magnoise)<EOL>finalerrs = err_median + (variablelc['<STR_LIT>'] + errnoise)<EOL>lcdict[mcol] = finalmags<EOL>lcdict[ecol] = finalerrs<EOL><DEDENT>lcdict['<STR_LIT>'] = variablelc['<STR_LIT>']<EOL>lcdict['<STR_LIT>'] = variablelc['<STR_LIT>']<EOL>if vartype is not None:<EOL><INDENT>lcdict['<STR_LIT>'] = variablelc['<STR_LIT>']<EOL>lcdict['<STR_LIT>'] = variablelc['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>lcdict['<STR_LIT>'] = np.nan<EOL>lcdict['<STR_LIT>'] = np.nan<EOL><DEDENT>tempoutf = '<STR_LIT>' % (fakelcfile, md5(npr.bytes(<NUM_LIT:4>)).hexdigest()[-<NUM_LIT:8>:])<EOL>with open(tempoutf, '<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(lcdict, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>if os.path.exists(tempoutf):<EOL><INDENT>shutil.copy(tempoutf, fakelcfile)<EOL>os.remove(tempoutf)<EOL><DEDENT>else:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' %<EOL>os.path.dirname(tempoutf))<EOL>raise<EOL><DEDENT>LOGINFO('<STR_LIT>' % (<EOL>lcdict['<STR_LIT>'],<EOL>vartype,<EOL>fakelcfile)<EOL>)<EOL>return {'<STR_LIT>':lcdict['<STR_LIT>'],<EOL>'<STR_LIT>':fakelcfile,<EOL>'<STR_LIT>':vartype,<EOL>'<STR_LIT>':lcdict['<STR_LIT>']}<EOL>

This adds variability of the specified type to the fake LC.

    The procedure is (for each `magcol`):

    - read the fakelcfile, get the stored moments and vartype info

    - add the periodic variability specified in vartype and varparamdists. if
      `vartype == None`, then do nothing in this step. If `override_vartype` is
      not None, override stored vartype with specified vartype. If
      `override_varparamdists` provided, override with specified
      `varparamdists`. NOTE: the varparamdists must make sense for the vartype,
      otherwise, weird stuff will happen.

    - add the median mag level stored in `fakelcfile` to the time series

    - add Gaussian noise to the light curve as specified in `fakelcfile`

    - add a varinfo key and dict to the lcdict with `varperiod`, `varepoch`,
      `varparams`

    - write back to fake LC pickle

    - return the `varinfo` dict to the caller

    Parameters
    ----------

    fakelcfile : str
        The name of the fake LC file to process.

    vartype : str
        The type of variability to add to this fake LC file.

    override_paramdists : dict
        A parameter distribution dict as in the `generate_XX_lightcurve`
        functions above. If provided, will override the distribution stored in
        the input fake LC file itself.

    magsarefluxes : bool
        Sets if the variability amplitude is in fluxes and not magnitudes.

    overwite : bool
        This overwrites the input fake LC file with a new variable LC even if
        it's been processed before.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'objectid':lcdict['objectid'],
             'lcfname':fakelcfile,
             'actual_vartype':vartype,
             'actual_varparams':lcdict['actual_varparams']}


<EOL>infof = os.path.join(simbasedir,'<STR_LIT>')<EOL>with open(infof, '<STR_LIT:rb>') as infd:<EOL><INDENT>lcinfo = pickle.load(infd)<EOL><DEDENT>lclist = lcinfo['<STR_LIT>']<EOL>varflag = lcinfo['<STR_LIT>']<EOL>vartypes = lcinfo['<STR_LIT>']<EOL>vartind = <NUM_LIT:0><EOL>varinfo = {}<EOL>for lc, varf, _lcind in zip(lclist, varflag, range(len(lclist))):<EOL><INDENT>if varf:<EOL><INDENT>thisvartype = vartypes[vartind]<EOL>if (override_paramdists and<EOL>isinstance(override_paramdists, dict) and<EOL>thisvartype in override_paramdists and<EOL>isinstance(override_paramdists[thisvartype], dict)):<EOL><INDENT>thisoverride_paramdists = override_paramdists[thisvartype]<EOL><DEDENT>else:<EOL><INDENT>thisoverride_paramdists = None<EOL><DEDENT>varlc = add_fakelc_variability(<EOL>lc, thisvartype,<EOL>override_paramdists=thisoverride_paramdists,<EOL>overwrite=overwrite_existingvar<EOL>)<EOL>varinfo[varlc['<STR_LIT>']] = {'<STR_LIT>': varlc['<STR_LIT>'],<EOL>'<STR_LIT>': varlc['<STR_LIT>']}<EOL>vartind = vartind + <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>varlc = add_fakelc_variability(<EOL>lc, None,<EOL>overwrite=overwrite_existingvar<EOL>)<EOL>varinfo[varlc['<STR_LIT>']] = {'<STR_LIT>': varlc['<STR_LIT>'],<EOL>'<STR_LIT>': varlc['<STR_LIT>']}<EOL><DEDENT><DEDENT>lcinfo['<STR_LIT>'] = varinfo<EOL>tempoutf = '<STR_LIT>' % (infof, md5(npr.bytes(<NUM_LIT:4>)).hexdigest()[-<NUM_LIT:8>:])<EOL>with open(tempoutf, '<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(lcinfo, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>if os.path.exists(tempoutf):<EOL><INDENT>shutil.copy(tempoutf, infof)<EOL>os.remove(tempoutf)<EOL><DEDENT>else:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' %<EOL>os.path.dirname(tempoutf))<EOL>raise<EOL><DEDENT>return lcinfo<EOL>

This adds variability and noise to all fake LCs in `simbasedir`.

    If an object is marked as variable in the `fakelcs-info`.pkl file in
    `simbasedir`, a variable signal will be added to its light curve based on
    its selected type, default period and amplitude distribution, the
    appropriate params, etc. the epochs for each variable object will be chosen
    uniformly from its time-range (and may not necessarily fall on a actual
    observed time). Nonvariable objects will only have noise added as determined
    by their params, but no variable signal will be added.

    Parameters
    ----------

    simbasedir : str
        The directory containing the fake LCs to process.

    override_paramdists : dict
        This can be used to override the stored variable parameters in each fake
        LC. It should be a dict of the following form::

            {'<vartype1>': {'<param1>: a scipy.stats distribution function or
                                       the np.random.randint function,
                            .
                            .
                            .
                            '<paramN>: a scipy.stats distribution function
                                       or the np.random.randint function}

        for any vartype in VARTYPE_LCGEN_MAP. These are used to override the
        default parameter distributions for each variable type.

    overwrite_existingvar : bool
        If this is True, then will overwrite any existing variability in the
        input fake LCs in `simbasedir`.

    Returns
    -------

    dict
        This returns a dict containing the fake LC filenames as keys and
        variability info for each as values.


try:<EOL><INDENT>with open(fakelcfile,'<STR_LIT:rb>') as infd:<EOL><INDENT>lcdict = pickle.load(infd)<EOL><DEDENT><DEDENT>except UnicodeDecodeError:<EOL><INDENT>with open(fakelcfile,'<STR_LIT:rb>') as infd:<EOL><INDENT>lcdict = pickle.load(infd, encoding='<STR_LIT>')<EOL><DEDENT><DEDENT>return lcdict<EOL>

This just reads a pickled fake LC.

Parameters
----------

fakelcfile : str
    The fake LC file to read.

Returns
-------

dict
    This returns an lcdict.


<EOL>with open(os.path.join(simbasedir, '<STR_LIT>'),'<STR_LIT:rb>') as infd:<EOL><INDENT>siminfo = pickle.load(infd)<EOL><DEDENT>lcfpaths = siminfo['<STR_LIT>']<EOL>varfeaturedir = os.path.join(simbasedir,'<STR_LIT>')<EOL>timecols = siminfo['<STR_LIT>']<EOL>magcols = siminfo['<STR_LIT>']<EOL>errcols = siminfo['<STR_LIT>']<EOL>timecols = siminfo['<STR_LIT>']<EOL>magcols = siminfo['<STR_LIT>']<EOL>errcols = siminfo['<STR_LIT>']<EOL>fakelc_formatkey = '<STR_LIT>' % siminfo['<STR_LIT>']<EOL>lcproc.register_lcformat(<EOL>fakelc_formatkey,<EOL>'<STR_LIT>',<EOL>timecols,<EOL>magcols,<EOL>errcols,<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>magsarefluxes=siminfo['<STR_LIT>']<EOL>)<EOL>varinfo = lcvfeatures.parallel_varfeatures(lcfpaths,<EOL>varfeaturedir,<EOL>lcformat=fakelc_formatkey,<EOL>mindet=mindet,<EOL>nworkers=nworkers)<EOL>with open(os.path.join(simbasedir,'<STR_LIT>'),'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(varinfo, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>return os.path.join(simbasedir,'<STR_LIT>')<EOL>

This runs `lcproc.lcvfeatures.parallel_varfeatures` on fake LCs in
    `simbasedir`.

    Parameters
    ----------

    simbasedir : str
        The directory containing the fake LCs to process.

    mindet : int
        The minimum number of detections needed to accept an LC and process it.

    nworkers : int or None
        The number of parallel workers to use when extracting variability
        features from the input light curves.

    Returns
    -------

    str
        The path to the `varfeatures` pickle created after running the
        `lcproc.lcvfeatures.parallel_varfeatures` function.


if (ntp+nfp) > <NUM_LIT:0>:<EOL><INDENT>return ntp/(ntp+nfp)<EOL><DEDENT>else:<EOL><INDENT>return np.nan<EOL><DEDENT>

This calculates precision.

https://en.wikipedia.org/wiki/Precision_and_recall

Parameters
----------

ntp : int
    The number of true positives.

nfp : int
    The number of false positives.

Returns
-------

float
    The precision calculated using `ntp/(ntp + nfp)`.


if (ntp+nfn) > <NUM_LIT:0>:<EOL><INDENT>return ntp/(ntp+nfn)<EOL><DEDENT>else:<EOL><INDENT>return np.nan<EOL><DEDENT>

This calculates recall.

https://en.wikipedia.org/wiki/Precision_and_recall

Parameters
----------

ntp : int
    The number of true positives.

nfn : int
    The number of false negatives.

Returns
-------

float
    The precision calculated using `ntp/(ntp + nfn)`.


def matthews_correl_coeff(ntp, ntn, nfp, nfn):

mcc_top = (ntp*ntn - nfp*nfn)<EOL>mcc_bot = msqrt((ntp + nfp)*(ntp + nfn)*(ntn + nfp)*(ntn + nfn))<EOL>if mcc_bot > <NUM_LIT:0>:<EOL><INDENT>return mcc_top/mcc_bot<EOL><DEDENT>else:<EOL><INDENT>return np.nan<EOL><DEDENT>

This calculates the Matthews correlation coefficent.

https://en.wikipedia.org/wiki/Matthews_correlation_coefficient

Parameters
----------

ntp : int
    The number of true positives.

ntn : int
    The number of true negatives

nfp : int
    The number of false positives.

nfn : int
    The number of false negatives.

Returns
-------

float
    The Matthews correlation coefficient.


<EOL>with open(os.path.join(simbasedir, '<STR_LIT>'),'<STR_LIT:rb>') as infd:<EOL><INDENT>siminfo = pickle.load(infd)<EOL><DEDENT>objectids = siminfo['<STR_LIT>']<EOL>varflags = siminfo['<STR_LIT>']<EOL>sdssr = siminfo['<STR_LIT>']<EOL>timecols = siminfo['<STR_LIT>']<EOL>magcols = siminfo['<STR_LIT>']<EOL>errcols = siminfo['<STR_LIT>']<EOL>fakelc_formatkey = '<STR_LIT>' % siminfo['<STR_LIT>']<EOL>lcproc.register_lcformat(<EOL>fakelc_formatkey,<EOL>'<STR_LIT>',<EOL>timecols,<EOL>magcols,<EOL>errcols,<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>magsarefluxes=siminfo['<STR_LIT>']<EOL>)<EOL>outdir = os.path.join(simbasedir, '<STR_LIT>')<EOL>if not os.path.exists(outdir):<EOL><INDENT>os.mkdir(outdir)<EOL><DEDENT>varfeaturedir = os.path.join(simbasedir, '<STR_LIT>')<EOL>varthreshinfof = os.path.join(<EOL>outdir,<EOL>'<STR_LIT>' % (magbinmedian,<EOL>stetson_stdev_min,<EOL>inveta_stdev_min)<EOL>)<EOL>varthresh = varthreshold.variability_threshold(<EOL>varfeaturedir,<EOL>varthreshinfof,<EOL>lcformat=fakelc_formatkey,<EOL>min_stetj_stdev=stetson_stdev_min,<EOL>min_inveta_stdev=inveta_stdev_min,<EOL>min_iqr_stdev=iqr_stdev_min,<EOL>verbose=False<EOL>)<EOL>magbins = varthresh['<STR_LIT>']<EOL>magbininds = np.digitize(sdssr, magbins)<EOL>binned_objectids = []<EOL>binned_actualvars = []<EOL>binned_actualnotvars = []<EOL>for mbinind, _magi in zip(np.unique(magbininds),<EOL>range(len(magbins)-<NUM_LIT:1>)):<EOL><INDENT>thisbinind = np.where(magbininds == mbinind)<EOL>thisbin_objectids = objectids[thisbinind]<EOL>thisbin_varflags = varflags[thisbinind]<EOL>thisbin_actualvars = thisbin_objectids[thisbin_varflags]<EOL>thisbin_actualnotvars = thisbin_objectids[~thisbin_varflags]<EOL>binned_objectids.append(thisbin_objectids)<EOL>binned_actualvars.append(thisbin_actualvars)<EOL>binned_actualnotvars.append(thisbin_actualnotvars)<EOL><DEDENT>recdict = {<EOL>'<STR_LIT>':simbasedir,<EOL>'<STR_LIT>':timecols,<EOL>'<STR_LIT>':magcols,<EOL>'<STR_LIT>':errcols,<EOL>'<STR_LIT>':siminfo['<STR_LIT>'],<EOL>'<STR_LIT>':stetson_stdev_min,<EOL>'<STR_LIT>':inveta_stdev_min,<EOL>'<STR_LIT>':iqr_stdev_min,<EOL>'<STR_LIT>':magbinmedian,<EOL>}<EOL>for magcol in magcols:<EOL><INDENT>magbinind = np.where(<EOL>np.array(varthresh[magcol]['<STR_LIT>']) == magbinmedian<EOL>)<EOL>magbinind = np.asscalar(magbinind[<NUM_LIT:0>])<EOL>thisbin_objectids = binned_objectids[magbinind]<EOL>thisbin_actualvars = binned_actualvars[magbinind]<EOL>thisbin_actualnotvars = binned_actualnotvars[magbinind]<EOL>stet_recoveredvars = varthresh[magcol][<EOL>'<STR_LIT>'<EOL>][magbinind]<EOL>stet_recoverednotvars = np.setdiff1d(thisbin_objectids,<EOL>stet_recoveredvars)<EOL>stet_truepositives = np.intersect1d(stet_recoveredvars,<EOL>thisbin_actualvars)<EOL>stet_falsepositives = np.intersect1d(stet_recoveredvars,<EOL>thisbin_actualnotvars)<EOL>stet_truenegatives = np.intersect1d(stet_recoverednotvars,<EOL>thisbin_actualnotvars)<EOL>stet_falsenegatives = np.intersect1d(stet_recoverednotvars,<EOL>thisbin_actualvars)<EOL>stet_recall = recall(stet_truepositives.size,<EOL>stet_falsenegatives.size)<EOL>stet_precision = precision(stet_truepositives.size,<EOL>stet_falsepositives.size)<EOL>stet_mcc = matthews_correl_coeff(stet_truepositives.size,<EOL>stet_truenegatives.size,<EOL>stet_falsepositives.size,<EOL>stet_falsenegatives.size)<EOL>inveta_recoveredvars = varthresh[magcol][<EOL>'<STR_LIT>'<EOL>][magbinind]<EOL>inveta_recoverednotvars = np.setdiff1d(thisbin_objectids,<EOL>inveta_recoveredvars)<EOL>inveta_truepositives = np.intersect1d(inveta_recoveredvars,<EOL>thisbin_actualvars)<EOL>inveta_falsepositives = np.intersect1d(inveta_recoveredvars,<EOL>thisbin_actualnotvars)<EOL>inveta_truenegatives = np.intersect1d(inveta_recoverednotvars,<EOL>thisbin_actualnotvars)<EOL>inveta_falsenegatives = np.intersect1d(inveta_recoverednotvars,<EOL>thisbin_actualvars)<EOL>inveta_recall = recall(inveta_truepositives.size,<EOL>inveta_falsenegatives.size)<EOL>inveta_precision = precision(inveta_truepositives.size,<EOL>inveta_falsepositives.size)<EOL>inveta_mcc = matthews_correl_coeff(inveta_truepositives.size,<EOL>inveta_truenegatives.size,<EOL>inveta_falsepositives.size,<EOL>inveta_falsenegatives.size)<EOL>iqr_recoveredvars = varthresh[magcol][<EOL>'<STR_LIT>'<EOL>][magbinind]<EOL>iqr_recoverednotvars = np.setdiff1d(thisbin_objectids,<EOL>iqr_recoveredvars)<EOL>iqr_truepositives = np.intersect1d(iqr_recoveredvars,<EOL>thisbin_actualvars)<EOL>iqr_falsepositives = np.intersect1d(iqr_recoveredvars,<EOL>thisbin_actualnotvars)<EOL>iqr_truenegatives = np.intersect1d(iqr_recoverednotvars,<EOL>thisbin_actualnotvars)<EOL>iqr_falsenegatives = np.intersect1d(iqr_recoverednotvars,<EOL>thisbin_actualvars)<EOL>iqr_recall = recall(iqr_truepositives.size,<EOL>iqr_falsenegatives.size)<EOL>iqr_precision = precision(iqr_truepositives.size,<EOL>iqr_falsepositives.size)<EOL>iqr_mcc = matthews_correl_coeff(iqr_truepositives.size,<EOL>iqr_truenegatives.size,<EOL>iqr_falsepositives.size,<EOL>iqr_falsenegatives.size)<EOL>stet_missed_inveta_found = np.setdiff1d(inveta_truepositives,<EOL>stet_truepositives)<EOL>stet_missed_iqr_found = np.setdiff1d(iqr_truepositives,<EOL>stet_truepositives)<EOL>inveta_missed_stet_found = np.setdiff1d(stet_truepositives,<EOL>inveta_truepositives)<EOL>inveta_missed_iqr_found = np.setdiff1d(iqr_truepositives,<EOL>inveta_truepositives)<EOL>iqr_missed_stet_found = np.setdiff1d(stet_truepositives,<EOL>iqr_truepositives)<EOL>iqr_missed_inveta_found = np.setdiff1d(inveta_truepositives,<EOL>iqr_truepositives)<EOL>if not statsonly:<EOL><INDENT>recdict[magcol] = {<EOL>'<STR_LIT>':stet_recoveredvars,<EOL>'<STR_LIT>':stet_truepositives,<EOL>'<STR_LIT>':stet_falsepositives,<EOL>'<STR_LIT>':stet_truenegatives,<EOL>'<STR_LIT>':stet_falsenegatives,<EOL>'<STR_LIT>':stet_precision,<EOL>'<STR_LIT>':stet_recall,<EOL>'<STR_LIT>':stet_mcc,<EOL>'<STR_LIT>':inveta_recoveredvars,<EOL>'<STR_LIT>':inveta_truepositives,<EOL>'<STR_LIT>':inveta_falsepositives,<EOL>'<STR_LIT>':inveta_truenegatives,<EOL>'<STR_LIT>':inveta_falsenegatives,<EOL>'<STR_LIT>':inveta_precision,<EOL>'<STR_LIT>':inveta_recall,<EOL>'<STR_LIT>':inveta_mcc,<EOL>'<STR_LIT>':iqr_recoveredvars,<EOL>'<STR_LIT>':iqr_truepositives,<EOL>'<STR_LIT>':iqr_falsepositives,<EOL>'<STR_LIT>':iqr_truenegatives,<EOL>'<STR_LIT>':iqr_falsenegatives,<EOL>'<STR_LIT>':iqr_precision,<EOL>'<STR_LIT>':iqr_recall,<EOL>'<STR_LIT>':iqr_mcc,<EOL>'<STR_LIT>':stet_missed_inveta_found,<EOL>'<STR_LIT>':stet_missed_iqr_found,<EOL>'<STR_LIT>':inveta_missed_stet_found,<EOL>'<STR_LIT>':inveta_missed_iqr_found,<EOL>'<STR_LIT>':iqr_missed_stet_found,<EOL>'<STR_LIT>':iqr_missed_inveta_found,<EOL>'<STR_LIT>':thisbin_actualvars,<EOL>'<STR_LIT>':thisbin_actualnotvars,<EOL>'<STR_LIT>':thisbin_objectids,<EOL>'<STR_LIT>':magbinind,<EOL>}<EOL><DEDENT>else:<EOL><INDENT>recdict[magcol] = {<EOL>'<STR_LIT>':stet_recoveredvars.size,<EOL>'<STR_LIT>':stet_truepositives.size,<EOL>'<STR_LIT>':stet_falsepositives.size,<EOL>'<STR_LIT>':stet_truenegatives.size,<EOL>'<STR_LIT>':stet_falsenegatives.size,<EOL>'<STR_LIT>':stet_precision,<EOL>'<STR_LIT>':stet_recall,<EOL>'<STR_LIT>':stet_mcc,<EOL>'<STR_LIT>':inveta_recoveredvars.size,<EOL>'<STR_LIT>':inveta_truepositives.size,<EOL>'<STR_LIT>':inveta_falsepositives.size,<EOL>'<STR_LIT>':inveta_truenegatives.size,<EOL>'<STR_LIT>':inveta_falsenegatives.size,<EOL>'<STR_LIT>':inveta_precision,<EOL>'<STR_LIT>':inveta_recall,<EOL>'<STR_LIT>':inveta_mcc,<EOL>'<STR_LIT>':iqr_recoveredvars.size,<EOL>'<STR_LIT>':iqr_truepositives.size,<EOL>'<STR_LIT>':iqr_falsepositives.size,<EOL>'<STR_LIT>':iqr_truenegatives.size,<EOL>'<STR_LIT>':iqr_falsenegatives.size,<EOL>'<STR_LIT>':iqr_precision,<EOL>'<STR_LIT>':iqr_recall,<EOL>'<STR_LIT>':iqr_mcc,<EOL>'<STR_LIT>':stet_missed_inveta_found.size,<EOL>'<STR_LIT>':stet_missed_iqr_found.size,<EOL>'<STR_LIT>':inveta_missed_stet_found.size,<EOL>'<STR_LIT>':inveta_missed_iqr_found.size,<EOL>'<STR_LIT>':iqr_missed_stet_found.size,<EOL>'<STR_LIT>':iqr_missed_inveta_found.size,<EOL>'<STR_LIT>':thisbin_actualvars.size,<EOL>'<STR_LIT>':thisbin_actualnotvars.size,<EOL>'<STR_LIT>':thisbin_objectids.size,<EOL>'<STR_LIT>':magbinind,<EOL>}<EOL><DEDENT><DEDENT>return recdict<EOL>

This runs variability selection for the given magbinmedian.

    To generate a full recovery matrix over all magnitude bins, run this
    function for each magbin over the specified stetson_stdev_min and
    inveta_stdev_min grid.

    Parameters
    ----------

    simbasedir : str
        The input directory of fake LCs.

    magbinmedian : float
        The magbin to run the variable recovery for. This is an item from the
        dict from `simbasedir/fakelcs-info.pkl: `fakelcinfo['magrms'][magcol]`
        list for each magcol and designates which magbin to get the recovery
        stats for.

    stetson_stdev_min : float
        The minimum sigma above the trend in the Stetson J variability index
        distribution for this magbin to use to consider objects as variable.

    inveta_stdev_min : float
        The minimum sigma above the trend in the 1/eta variability index
        distribution for this magbin to use to consider objects as variable.

    iqr_stdev_min : float
        The minimum sigma above the trend in the IQR variability index
        distribution for this magbin to use to consider objects as variable.

    statsonly : bool
        If this is True, only the final stats will be returned. If False, the
        full arrays used to generate the stats will also be returned.

    Returns
    -------

    dict
        The returned dict contains statistics for this magbin and if requested,
        the full arrays used to calculate the statistics.


simbasedir, gridpoint, magbinmedian = task<EOL>try:<EOL><INDENT>res = get_recovered_variables_for_magbin(simbasedir,<EOL>magbinmedian,<EOL>stetson_stdev_min=gridpoint[<NUM_LIT:0>],<EOL>inveta_stdev_min=gridpoint[<NUM_LIT:1>],<EOL>iqr_stdev_min=gridpoint[<NUM_LIT:2>],<EOL>statsonly=True)<EOL>return res<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' % gridpoint)<EOL>return None<EOL><DEDENT>

This is a parallel grid search worker for the function below.


<EOL>outdir = os.path.join(simbasedir,'<STR_LIT>')<EOL>if not os.path.exists(outdir):<EOL><INDENT>os.mkdir(outdir)<EOL><DEDENT>with open(os.path.join(simbasedir, '<STR_LIT>'),'<STR_LIT:rb>') as infd:<EOL><INDENT>siminfo = pickle.load(infd)<EOL><DEDENT>timecols = siminfo['<STR_LIT>']<EOL>magcols = siminfo['<STR_LIT>']<EOL>errcols = siminfo['<STR_LIT>']<EOL>magbinmedians = siminfo['<STR_LIT>'][magcols[<NUM_LIT:0>]]['<STR_LIT>']<EOL>stetson_grid = np.linspace(stetson_stdev_range[<NUM_LIT:0>],<EOL>stetson_stdev_range[<NUM_LIT:1>],<EOL>num=ngridpoints)<EOL>inveta_grid = np.linspace(inveta_stdev_range[<NUM_LIT:0>],<EOL>inveta_stdev_range[<NUM_LIT:1>],<EOL>num=ngridpoints)<EOL>iqr_grid = np.linspace(iqr_stdev_range[<NUM_LIT:0>],<EOL>iqr_stdev_range[<NUM_LIT:1>],<EOL>num=ngridpoints)<EOL>stet_inveta_iqr_grid = []<EOL>for stet in stetson_grid:<EOL><INDENT>for inveta in inveta_grid:<EOL><INDENT>for iqr in iqr_grid:<EOL><INDENT>grid_point = [stet, inveta, iqr]<EOL>stet_inveta_iqr_grid.append(grid_point)<EOL><DEDENT><DEDENT><DEDENT>grid_results = {'<STR_LIT>':stetson_grid,<EOL>'<STR_LIT>':inveta_grid,<EOL>'<STR_LIT>':iqr_grid,<EOL>'<STR_LIT>':stet_inveta_iqr_grid,<EOL>'<STR_LIT>':magbinmedians,<EOL>'<STR_LIT>':timecols,<EOL>'<STR_LIT>':magcols,<EOL>'<STR_LIT>':errcols,<EOL>'<STR_LIT>':os.path.abspath(simbasedir),<EOL>'<STR_LIT>':[]}<EOL>pool = mp.Pool(ngridworkers)<EOL>for magbinmedian in magbinmedians:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>' % magbinmedian)<EOL>tasks = [(simbasedir, gp, magbinmedian) for gp in stet_inveta_iqr_grid]<EOL>thisbin_results = pool.map(magbin_varind_gridsearch_worker, tasks)<EOL>grid_results['<STR_LIT>'].append(thisbin_results)<EOL><DEDENT>pool.close()<EOL>pool.join()<EOL>LOGINFO('<STR_LIT>')<EOL>with open(os.path.join(simbasedir,<EOL>'<STR_LIT>'),'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(grid_results,outfd,pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>return grid_results<EOL>

This runs a variable index grid search per magbin.

    For each magbin, this does a grid search using the stetson and inveta ranges
    provided and tries to optimize the Matthews Correlation Coefficient (best
    value is +1.0), indicating the best possible separation of variables
    vs. nonvariables. The thresholds on these two variable indexes that produce
    the largest coeff for the collection of fake LCs will probably be the ones
    that work best for actual variable classification on the real LCs.

    https://en.wikipedia.org/wiki/Matthews_correlation_coefficient

    For each grid-point, calculates the true positives, false positives, true
    negatives, false negatives. Then gets the precision and recall, confusion
    matrix, and the ROC curve for variable vs. nonvariable.

    Once we've identified the best thresholds to use, we can then calculate
    variable object numbers:

    - as a function of magnitude
    - as a function of period
    - as a function of number of detections
    - as a function of amplitude of variability

    Writes everything back to `simbasedir/fakevar-recovery.pkl`. Use the
    plotting function below to make plots for the results.

    Parameters
    ----------

    simbasedir : str
        The directory where the fake LCs are located.

    stetson_stdev_range : sequence of 2 floats
        The min and max values of the Stetson J variability index to generate a
        grid over these to test for the values of this index that produce the
        'best' recovery rate for the injected variable stars.

    inveta_stdev_range : sequence of 2 floats
        The min and max values of the 1/eta variability index to generate a
        grid over these to test for the values of this index that produce the
        'best' recovery rate for the injected variable stars.

    iqr_stdev_range : sequence of 2 floats
        The min and max values of the IQR variability index to generate a
        grid over these to test for the values of this index that produce the
        'best' recovery rate for the injected variable stars.

    ngridpoints : int
        The number of grid points for each variability index grid. Remember that
        this function will be searching in 3D and will require lots of time to
        run if ngridpoints is too large.

        For the default number of grid points and 25000 simulated light curves,
        this takes about 3 days to run on a 40 (effective) core machine with 2 x
        Xeon E5-2650v3 CPUs.

    ngridworkers : int or None
        The number of parallel grid search workers that will be launched.

    Returns
    -------

    dict
        The returned dict contains a list of recovery stats for each magbin and
        each grid point in the variability index grids that were used. This dict
        can be passed to the plotting function below to plot the results.


<EOL>if (isinstance(gridsearch_results, str) and<EOL>os.path.exists(gridsearch_results)):<EOL><INDENT>with open(gridsearch_results,'<STR_LIT:rb>') as infd:<EOL><INDENT>gridresults = pickle.load(infd)<EOL><DEDENT><DEDENT>elif isinstance(gridsearch_results, dict):<EOL><INDENT>gridresults = gridsearch_results<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>')<EOL>return None<EOL><DEDENT>plotres = {'<STR_LIT>':gridresults['<STR_LIT>']}<EOL>recgrid = gridresults['<STR_LIT>']<EOL>simbasedir = gridresults['<STR_LIT>']<EOL>for magcol in gridresults['<STR_LIT>']:<EOL><INDENT>plotres[magcol] = {'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':gridresults['<STR_LIT>']}<EOL>for magbinind, magbinmedian in enumerate(gridresults['<STR_LIT>']):<EOL><INDENT>LOGINFO('<STR_LIT>' %<EOL>(magcol, magbinmedian))<EOL>stet_mcc = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[::(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)]<EOL>stet_precision = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[::(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)]<EOL>stet_recall = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[::(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)]<EOL>stet_missed_inveta_found = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[::(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)]<EOL>stet_missed_iqr_found = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[::(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)]<EOL>inveta_mcc = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>::gridresults['<STR_LIT>'].size<EOL>]<EOL>inveta_precision = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>::gridresults['<STR_LIT>'].size<EOL>]<EOL>inveta_recall = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>::gridresults['<STR_LIT>'].size<EOL>]<EOL>inveta_missed_stet_found = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>::gridresults['<STR_LIT>'].size<EOL>]<EOL>inveta_missed_iqr_found = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>::gridresults['<STR_LIT>'].size<EOL>]<EOL>iqr_mcc = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>:gridresults['<STR_LIT>'].size<EOL>]<EOL>iqr_precision = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>:gridresults['<STR_LIT>'].size<EOL>]<EOL>iqr_recall = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>:gridresults['<STR_LIT>'].size<EOL>]<EOL>iqr_missed_stet_found = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>:gridresults['<STR_LIT>'].size<EOL>]<EOL>iqr_missed_inveta_found = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>:gridresults['<STR_LIT>'].size<EOL>]<EOL>fig = plt.figure(figsize=(<NUM_LIT>*<NUM_LIT:5>, <NUM_LIT>*<NUM_LIT:3>))<EOL>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:1>)<EOL>if np.any(np.isfinite(stet_mcc)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>stet_mcc)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:2>)<EOL>if np.any(np.isfinite(stet_precision)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>stet_precision)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:3>)<EOL>if np.any(np.isfinite(stet_recall)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>stet_recall)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:4>)<EOL>if np.any(np.isfinite(stet_missed_inveta_found)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>stet_missed_inveta_found)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:5>)<EOL>if np.any(np.isfinite(stet_missed_iqr_found)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>stet_missed_iqr_found)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:6>)<EOL>if np.any(np.isfinite(inveta_mcc)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>inveta_mcc)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:7>)<EOL>if np.any(np.isfinite(inveta_precision)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>inveta_precision)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:8>)<EOL>if np.any(np.isfinite(inveta_recall)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>inveta_recall)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:9>)<EOL>if np.any(np.isfinite(inveta_missed_stet_found)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>inveta_missed_stet_found)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:10>)<EOL>if np.any(np.isfinite(inveta_missed_iqr_found)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>inveta_missed_iqr_found)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:11>)<EOL>if np.any(np.isfinite(iqr_mcc)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>iqr_mcc)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:12>)<EOL>if np.any(np.isfinite(iqr_precision)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>iqr_precision)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT>)<EOL>if np.any(np.isfinite(iqr_recall)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>iqr_recall)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT>)<EOL>if np.any(np.isfinite(iqr_missed_stet_found)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>iqr_missed_stet_found)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:15>)<EOL>if np.any(np.isfinite(iqr_missed_inveta_found)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>iqr_missed_inveta_found)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplots_adjust(hspace=<NUM_LIT>,wspace=<NUM_LIT>)<EOL>plt.suptitle('<STR_LIT>' % (magcol, magbinmedian))<EOL>plotdir = os.path.join(gridresults['<STR_LIT>'],<EOL>'<STR_LIT>')<EOL>if not os.path.exists(plotdir):<EOL><INDENT>os.mkdir(plotdir)<EOL><DEDENT>gridplotf = os.path.join(<EOL>plotdir,<EOL>'<STR_LIT>' %<EOL>(magcol, magbinmedian)<EOL>)<EOL>plt.savefig(gridplotf,dpi=<NUM_LIT:100>,bbox_inches='<STR_LIT>')<EOL>plt.close('<STR_LIT:all>')<EOL>stet_mcc_maxind = np.where(stet_mcc == np.max(stet_mcc))<EOL>stet_precision_maxind = np.where(<EOL>stet_precision == np.max(stet_precision)<EOL>)<EOL>stet_recall_maxind = np.where(stet_recall == np.max(stet_recall))<EOL>best_stet_mcc = stet_mcc[stet_mcc_maxind]<EOL>best_stet_precision = stet_mcc[stet_precision_maxind]<EOL>best_stet_recall = stet_mcc[stet_recall_maxind]<EOL>stet_with_best_mcc = gridresults['<STR_LIT>'][stet_mcc_maxind]<EOL>stet_with_best_precision = gridresults['<STR_LIT>'][<EOL>stet_precision_maxind<EOL>]<EOL>stet_with_best_recall = (<EOL>gridresults['<STR_LIT>'][stet_recall_maxind]<EOL>)<EOL>inveta_mcc_maxind = np.where(inveta_mcc == np.max(inveta_mcc))<EOL>inveta_precision_maxind = np.where(<EOL>inveta_precision == np.max(inveta_precision)<EOL>)<EOL>inveta_recall_maxind = (<EOL>np.where(inveta_recall == np.max(inveta_recall))<EOL>)<EOL>best_inveta_mcc = inveta_mcc[inveta_mcc_maxind]<EOL>best_inveta_precision = inveta_mcc[inveta_precision_maxind]<EOL>best_inveta_recall = inveta_mcc[inveta_recall_maxind]<EOL>inveta_with_best_mcc = gridresults['<STR_LIT>'][inveta_mcc_maxind]<EOL>inveta_with_best_precision = gridresults['<STR_LIT>'][<EOL>inveta_precision_maxind<EOL>]<EOL>inveta_with_best_recall = gridresults['<STR_LIT>'][<EOL>inveta_recall_maxind<EOL>]<EOL>iqr_mcc_maxind = np.where(iqr_mcc == np.max(iqr_mcc))<EOL>iqr_precision_maxind = np.where(<EOL>iqr_precision == np.max(iqr_precision)<EOL>)<EOL>iqr_recall_maxind = (<EOL>np.where(iqr_recall == np.max(iqr_recall))<EOL>)<EOL>best_iqr_mcc = iqr_mcc[iqr_mcc_maxind]<EOL>best_iqr_precision = iqr_mcc[iqr_precision_maxind]<EOL>best_iqr_recall = iqr_mcc[iqr_recall_maxind]<EOL>iqr_with_best_mcc = gridresults['<STR_LIT>'][iqr_mcc_maxind]<EOL>iqr_with_best_precision = gridresults['<STR_LIT>'][<EOL>iqr_precision_maxind<EOL>]<EOL>iqr_with_best_recall = gridresults['<STR_LIT>'][<EOL>iqr_recall_maxind<EOL>]<EOL>plotres[magcol][magbinmedian] = {<EOL>'<STR_LIT>':gridresults['<STR_LIT>'],<EOL>'<STR_LIT>':stet_mcc,<EOL>'<STR_LIT>':stet_precision,<EOL>'<STR_LIT>':stet_recall,<EOL>'<STR_LIT>':stet_missed_inveta_found,<EOL>'<STR_LIT>':best_stet_mcc,<EOL>'<STR_LIT>':stet_with_best_mcc,<EOL>'<STR_LIT>':best_stet_precision,<EOL>'<STR_LIT>':stet_with_best_precision,<EOL>'<STR_LIT>':best_stet_recall,<EOL>'<STR_LIT>':stet_with_best_recall,<EOL>'<STR_LIT>':gridresults['<STR_LIT>'],<EOL>'<STR_LIT>':inveta_mcc,<EOL>'<STR_LIT>':inveta_precision,<EOL>'<STR_LIT>':inveta_recall,<EOL>'<STR_LIT>':inveta_missed_stet_found,<EOL>'<STR_LIT>':best_inveta_mcc,<EOL>'<STR_LIT>':inveta_with_best_mcc,<EOL>'<STR_LIT>':best_inveta_precision,<EOL>'<STR_LIT>':inveta_with_best_precision,<EOL>'<STR_LIT>':best_inveta_recall,<EOL>'<STR_LIT>':inveta_with_best_recall,<EOL>'<STR_LIT>':gridresults['<STR_LIT>'],<EOL>'<STR_LIT>':iqr_mcc,<EOL>'<STR_LIT>':iqr_precision,<EOL>'<STR_LIT>':iqr_recall,<EOL>'<STR_LIT>':iqr_missed_stet_found,<EOL>'<STR_LIT>':best_iqr_mcc,<EOL>'<STR_LIT>':iqr_with_best_mcc,<EOL>'<STR_LIT>':best_iqr_precision,<EOL>'<STR_LIT>':iqr_with_best_precision,<EOL>'<STR_LIT>':best_iqr_recall,<EOL>'<STR_LIT>':iqr_with_best_recall,<EOL>'<STR_LIT>':gridplotf<EOL>}<EOL>if stet_with_best_mcc.size > <NUM_LIT:1>:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(stet_with_best_mcc[<NUM_LIT:0>])<EOL><DEDENT>elif stet_with_best_mcc.size > <NUM_LIT:0>:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(stet_with_best_mcc[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(np.nan)<EOL><DEDENT>if inveta_with_best_mcc.size > <NUM_LIT:1>:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(inveta_with_best_mcc[<NUM_LIT:0>])<EOL><DEDENT>elif inveta_with_best_mcc.size > <NUM_LIT:0>:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(inveta_with_best_mcc[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(np.nan)<EOL><DEDENT>if iqr_with_best_mcc.size > <NUM_LIT:1>:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(iqr_with_best_mcc[<NUM_LIT:0>])<EOL><DEDENT>elif iqr_with_best_mcc.size > <NUM_LIT:0>:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(iqr_with_best_mcc[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(np.nan)<EOL><DEDENT><DEDENT><DEDENT>plotrespicklef = os.path.join(simbasedir,<EOL>'<STR_LIT>')<EOL>with open(plotrespicklef, '<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(plotres, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>for magcol in gridresults['<STR_LIT>']:<EOL><INDENT>LOGINFO('<STR_LIT>' % magcol)<EOL>LOGINFO('<STR_LIT>')<EOL>for magbin, inveta, stet, iqr in zip(<EOL>plotres[magcol]['<STR_LIT>'],<EOL>plotres[magcol]['<STR_LIT>'],<EOL>plotres[magcol]['<STR_LIT>'],<EOL>plotres[magcol]['<STR_LIT>']):<EOL><INDENT>LOGINFO('<STR_LIT>' % (magbin,<EOL>inveta,<EOL>stet,<EOL>iqr))<EOL><DEDENT><DEDENT>return plotres<EOL>

This plots the gridsearch results from `variable_index_gridsearch_magbin`.

    Parameters
    ----------

    gridsearch_results : dict
        This is the dict produced by `variable_index_gridsearch_magbin` above.

    Returns
    -------

    dict
        The returned dict contains filenames of the recovery rate plots made for
        each variability index. These include plots of the precision, recall,
        and Matthews Correlation Coefficient over each magbin and a heatmap of
        these values over the grid points of the variability index stdev values
        arrays used.


<EOL>with open(os.path.join(simbasedir, '<STR_LIT>'),'<STR_LIT:rb>') as infd:<EOL><INDENT>siminfo = pickle.load(infd)<EOL><DEDENT>lcfpaths = siminfo['<STR_LIT>']<EOL>pfdir = os.path.join(simbasedir,'<STR_LIT>')<EOL>timecols = siminfo['<STR_LIT>']<EOL>magcols = siminfo['<STR_LIT>']<EOL>errcols = siminfo['<STR_LIT>']<EOL>fakelc_formatkey = '<STR_LIT>' % siminfo['<STR_LIT>']<EOL>lcproc.register_lcformat(<EOL>fakelc_formatkey,<EOL>'<STR_LIT>',<EOL>timecols,<EOL>magcols,<EOL>errcols,<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>magsarefluxes=siminfo['<STR_LIT>']<EOL>)<EOL>if liststartindex:<EOL><INDENT>lcfpaths = lcfpaths[liststartindex:]<EOL><DEDENT>if listmaxobjects:<EOL><INDENT>lcfpaths = lcfpaths[:listmaxobjects]<EOL><DEDENT>pfinfo = periodsearch.parallel_pf(lcfpaths,<EOL>pfdir,<EOL>lcformat=fakelc_formatkey,<EOL>pfmethods=pfmethods,<EOL>pfkwargs=pfkwargs,<EOL>getblssnr=getblssnr,<EOL>sigclip=sigclip,<EOL>nperiodworkers=nperiodworkers,<EOL>ncontrolworkers=ncontrolworkers)<EOL>with open(os.path.join(simbasedir,<EOL>'<STR_LIT>'),'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(pfinfo, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>return os.path.join(simbasedir,'<STR_LIT>')<EOL>

This runs periodfinding using several period-finders on a collection of
    fake LCs.

    As a rough benchmark, 25000 fake LCs with 10000--50000 points per LC take
    about 26 days in total to run on an invocation of this function using
    GLS+PDM+BLS and 10 periodworkers and 4 controlworkers (so all 40 'cores') on
    a 2 x Xeon E5-2660v3 machine.

    Parameters
    ----------

    pfmethods : sequence of str
        This is used to specify which periodfinders to run. These must be in the
        `lcproc.periodsearch.PFMETHODS` dict.

    pfkwargs : sequence of dict
        This is used to provide optional kwargs to the period-finders.

    getblssnr : bool
        If this is True, will run BLS SNR calculations for each object and
        magcol. This takes a while to run, so it's disabled (False) by default.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    nperiodworkers : int
        This is the number of parallel period-finding worker processes to use.

    ncontrolworkers : int
        This is the number of parallel period-finding control workers to
        use. Each control worker will launch `nperiodworkers` worker processes.

    liststartindex : int
        The starting index of processing. This refers to the filename list
        generated by running `glob.glob` on the fake LCs in `simbasedir`.

    maxobjects : int
        The maximum number of objects to process in this run. Use this with
        `liststartindex` to effectively distribute working on a large list of
        input light curves over several sessions or machines.

    Returns
    -------

    str
        The path to the output summary pickle produced by
        `lcproc.periodsearch.parallel_pf`


if not (np.isfinite(actualperiod) and np.isfinite(recoveredperiod)):<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>twotimes_p = actualperiod*<NUM_LIT><EOL>half_p = actualperiod*<NUM_LIT:0.5><EOL>alias_1a = actualperiod/(<NUM_LIT:1.0>+actualperiod)<EOL>alias_1b = actualperiod/(<NUM_LIT:1.0>-actualperiod)<EOL>alias_2a = actualperiod/(<NUM_LIT:1.0>+<NUM_LIT>*actualperiod)<EOL>alias_2b = actualperiod/(<NUM_LIT:1.0>-<NUM_LIT>*actualperiod)<EOL>alias_3a = actualperiod/(<NUM_LIT:1.0>+<NUM_LIT>*actualperiod)<EOL>alias_3b = actualperiod/(<NUM_LIT:1.0>-<NUM_LIT>*actualperiod)<EOL>alias_4a = actualperiod/(actualperiod - <NUM_LIT:1.0>)<EOL>alias_4b = actualperiod/(<NUM_LIT>*actualperiod - <NUM_LIT:1.0>)<EOL>aliases = np.ravel(np.array([<EOL>actualperiod,<EOL>twotimes_p,<EOL>half_p,<EOL>alias_1a,<EOL>alias_1b,<EOL>alias_2a,<EOL>alias_2b,<EOL>alias_3a,<EOL>alias_3b,<EOL>alias_4a,<EOL>alias_4b]<EOL>))<EOL>alias_labels = np.array(ALIAS_TYPES)<EOL>closest_alias = np.isclose(recoveredperiod, aliases, atol=tolerance)<EOL>if np.any(closest_alias):<EOL><INDENT>closest_alias_type = alias_labels[closest_alias]<EOL>return '<STR_LIT:U+002C>'.join(closest_alias_type.tolist())<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT><DEDENT>

This determines what kind of aliasing (if any) exists between
    `recoveredperiod` and `actualperiod`.

    Parameters
    ----------

    actualperiod : float
        The actual period of the object.

    recoveredperiod : float
        The recovered period of the object.

    tolerance : float
        The absolute difference required between the input periods to mark the
        recovered period as close to the actual period.

    Returns
    -------

    str
        The type of alias determined for the input combination of periods. This
        will be CSV string with values taken from the following list, based on
        the types of alias found::

            ['actual',
             'twice',
             'half',
             'ratio_over_1plus',
             'ratio_over_1minus',
             'ratio_over_1plus_twice',
             'ratio_over_1minus_twice',
             'ratio_over_1plus_thrice',
             'ratio_over_1minus_thrice',
             'ratio_over_minus1',
             'ratio_over_twice_minus1']


if fakepfpkl.endswith('<STR_LIT>'):<EOL><INDENT>infd = gzip.open(fakepfpkl,'<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>infd = open(fakepfpkl,'<STR_LIT:rb>')<EOL><DEDENT>fakepf = pickle.load(infd)<EOL>infd.close()<EOL>objectid, lcfbasename = fakepf['<STR_LIT>'], fakepf['<STR_LIT>']<EOL>lcfpath = os.path.join(simbasedir,'<STR_LIT>',lcfbasename)<EOL>if not os.path.exists(lcfpath):<EOL><INDENT>LOGERROR('<STR_LIT>' % (objectid,<EOL>lcfpath))<EOL>return None<EOL><DEDENT>fakelc = lcproc._read_pklc(lcfpath)<EOL>actual_varparams, actual_varperiod, actual_varamplitude, actual_vartype = (<EOL>fakelc['<STR_LIT>'],<EOL>fakelc['<STR_LIT>'],<EOL>fakelc['<STR_LIT>'],<EOL>fakelc['<STR_LIT>']<EOL>)<EOL>actual_moments = fakelc['<STR_LIT>']<EOL>magcols = fakelc['<STR_LIT>']<EOL>pfres = {<EOL>'<STR_LIT>':objectid,<EOL>'<STR_LIT>':simbasedir,<EOL>'<STR_LIT>':magcols,<EOL>'<STR_LIT>':os.path.abspath(lcfpath),<EOL>'<STR_LIT>':os.path.abspath(fakepfpkl),<EOL>'<STR_LIT>':actual_vartype,<EOL>'<STR_LIT>':actual_varperiod,<EOL>'<STR_LIT>':actual_varamplitude,<EOL>'<STR_LIT>':actual_varparams,<EOL>'<STR_LIT>':actual_moments,<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>}<EOL>for magcol in magcols:<EOL><INDENT>for pfm in lcproc.PFMETHODS:<EOL><INDENT>if pfm in fakepf[magcol]:<EOL><INDENT>for rpi, rp in enumerate(<EOL>fakepf[magcol][pfm]['<STR_LIT>']<EOL>):<EOL><INDENT>if ((not np.any(np.isclose(<EOL>rp,<EOL>np.array(pfres['<STR_LIT>']),<EOL>rtol=period_tolerance<EOL>))) and np.isfinite(rp)):<EOL><INDENT>pfres['<STR_LIT>'].append(rp)<EOL>pfres['<STR_LIT>'].append(pfm)<EOL>pfres['<STR_LIT>'].append(magcol)<EOL>if pfm == '<STR_LIT>':<EOL><INDENT>this_lspval = (<EOL>np.max(fakepf[magcol][pfm]['<STR_LIT>']) -<EOL>fakepf[magcol][pfm]['<STR_LIT>'][rpi]<EOL>)<EOL><DEDENT>else:<EOL><INDENT>this_lspval = (<EOL>fakepf[magcol][pfm]['<STR_LIT>'][rpi] /<EOL>np.max(fakepf[magcol][pfm]['<STR_LIT>'])<EOL>)<EOL><DEDENT>pfres['<STR_LIT>'].append(this_lspval)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>pfres['<STR_LIT>'] = np.array(pfres['<STR_LIT>'])<EOL>pfres['<STR_LIT>'] = np.array(pfres['<STR_LIT>'])<EOL>pfres['<STR_LIT>'] = np.array(pfres['<STR_LIT>'])<EOL>pfres['<STR_LIT>'] = np.array(pfres['<STR_LIT>'])<EOL>if (actual_vartype and<EOL>actual_vartype in PERIODIC_VARTYPES and<EOL>np.isfinite(actual_varperiod)):<EOL><INDENT>if pfres['<STR_LIT>'].size > <NUM_LIT:0>:<EOL><INDENT>for ri in range(pfres['<STR_LIT>'].size):<EOL><INDENT>pfres['<STR_LIT>'].append(pfres['<STR_LIT>'][ri] -<EOL>np.asscalar(actual_varperiod))<EOL>pfres['<STR_LIT>'].append(<EOL>check_periodrec_alias(actual_varperiod,<EOL>pfres['<STR_LIT>'][ri],<EOL>tolerance=period_tolerance)<EOL>)<EOL><DEDENT>pfres['<STR_LIT>'] = np.array(pfres['<STR_LIT>'])<EOL>pfres['<STR_LIT>'] = np.array(pfres['<STR_LIT>'])<EOL>rec_absdiff = np.abs(pfres['<STR_LIT>'])<EOL>best_recp_ind = rec_absdiff == rec_absdiff.min()<EOL>pfres['<STR_LIT>'] = (<EOL>pfres['<STR_LIT>'][best_recp_ind]<EOL>)<EOL>pfres['<STR_LIT>'] = (<EOL>pfres['<STR_LIT>'][best_recp_ind]<EOL>)<EOL>pfres['<STR_LIT>'] = (<EOL>pfres['<STR_LIT>'][best_recp_ind]<EOL>)<EOL>pfres['<STR_LIT>'] = (<EOL>pfres['<STR_LIT>'][best_recp_ind]<EOL>)<EOL>pfres['<STR_LIT>'] = (<EOL>pfres['<STR_LIT>'][best_recp_ind]<EOL>)<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING(<EOL>'<STR_LIT>' %<EOL>fakepfpkl<EOL>)<EOL>pfres['<STR_LIT>'] = np.array(['<STR_LIT>'])<EOL>pfres['<STR_LIT>'] = np.array([np.nan])<EOL>pfres['<STR_LIT>'] = np.array([np.nan])<EOL>pfres['<STR_LIT>'] = np.array([],dtype=np.unicode_)<EOL>pfres['<STR_LIT>'] = np.array([],dtype=np.unicode_)<EOL>pfres['<STR_LIT>'] = np.array([],dtype=np.unicode_)<EOL>pfres['<STR_LIT>'] = np.array([np.nan])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>pfres['<STR_LIT>'] = np.array(<EOL>['<STR_LIT>']*pfres['<STR_LIT>'].size<EOL>)<EOL>pfres['<STR_LIT>'] = np.zeros(pfres['<STR_LIT>'].size)<EOL>pfres['<STR_LIT>'] = np.array([np.nan])<EOL>pfres['<STR_LIT>'] = np.array([],dtype=np.unicode_)<EOL>pfres['<STR_LIT>'] = np.array([],dtype=np.unicode_)<EOL>pfres['<STR_LIT>'] = np.array(['<STR_LIT>'])<EOL>pfres['<STR_LIT>'] = np.array([np.nan])<EOL><DEDENT>return pfres<EOL>

Recovers the periodic variable status/info for the simulated PF result.

    - Uses simbasedir and the lcfbasename stored in fakepfpkl to figure out
      where the LC for this object is.
    - Gets the actual_varparams, actual_varperiod, actual_vartype,
      actual_varamplitude elements from the LC.
    - Figures out if the current objectid is a periodic variable (using
      actual_vartype).
    - If it is a periodic variable, gets the canonical period assigned to it.
    - Checks if the period was recovered in any of the five best periods
      reported by any of the period-finders, checks if the period recovered was
      a harmonic of the period.
    - Returns the objectid, actual period and vartype, recovered period, and
      recovery status.

    Parameters
    ----------

    fakepfpkl : str
        This is a periodfinding-<objectid>.pkl[.gz] file produced in the
        `simbasedir/periodfinding` subdirectory after `run_periodfinding` above
        is done.

    simbasedir : str
        The base directory where all of the fake LCs and period-finding results
        are.

    period_tolerance : float
        The maximum difference that this function will consider between an
        actual period (or its aliases) and a recovered period to consider it as
        as a 'recovered' period.

    Returns
    -------

    dict
        Returns a dict of period-recovery results.


pfpkl, simbasedir, period_tolerance = task<EOL>try:<EOL><INDENT>return periodicvar_recovery(pfpkl,<EOL>simbasedir,<EOL>period_tolerance=period_tolerance)<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' % repr(task))<EOL>return None<EOL><DEDENT>

This is a parallel worker for running period-recovery.

    Parameters
    ----------

    task : tuple
        This is used to pass args to the `periodicvar_recovery` function::

            task[0] = period-finding result pickle to work on
            task[1] = simbasedir
            task[2] = period_tolerance

    Returns
    -------

    dict
        This is the dict produced by the `periodicvar_recovery` function for the
        input period-finding result pickle.


<EOL>pfpkldir = os.path.join(simbasedir,'<STR_LIT>')<EOL>if not os.path.exists(pfpkldir):<EOL><INDENT>LOGERROR('<STR_LIT>' %<EOL>simbasedir)<EOL>return None<EOL><DEDENT>pfpkl_list = glob.glob(os.path.join(pfpkldir,'<STR_LIT>'))<EOL>if len(pfpkl_list) > <NUM_LIT:0>:<EOL><INDENT>if liststartind:<EOL><INDENT>pfpkl_list = pfpkl_list[liststartind:]<EOL><DEDENT>if listmaxobjects:<EOL><INDENT>pfpkl_list = pfpkl_list[:listmaxobjects]<EOL><DEDENT>tasks = [(x, simbasedir, period_tolerance) for x in pfpkl_list]<EOL>pool = mp.Pool(nworkers)<EOL>results = pool.map(periodrec_worker, tasks)<EOL>pool.close()<EOL>pool.join()<EOL>resdict = {x['<STR_LIT>']:x for x in results if x is not None}<EOL>actual_periodicvars = np.array(<EOL>[x['<STR_LIT>'] for x in results<EOL>if (x is not None and x['<STR_LIT>'] in PERIODIC_VARTYPES)],<EOL>dtype=np.unicode_<EOL>)<EOL>recovered_periodicvars = np.array(<EOL>[x['<STR_LIT>'] for x in results<EOL>if (x is not None and '<STR_LIT>' in x['<STR_LIT>'])],<EOL>dtype=np.unicode_<EOL>)<EOL>alias_twice_periodicvars = np.array(<EOL>[x['<STR_LIT>'] for x in results<EOL>if (x is not None and '<STR_LIT>' in x['<STR_LIT>'])],<EOL>dtype=np.unicode_<EOL>)<EOL>alias_half_periodicvars = np.array(<EOL>[x['<STR_LIT>'] for x in results<EOL>if (x is not None and '<STR_LIT>' in x['<STR_LIT>'])],<EOL>dtype=np.unicode_<EOL>)<EOL>all_objectids = [x['<STR_LIT>'] for x in results]<EOL>outdict = {'<STR_LIT>':os.path.abspath(simbasedir),<EOL>'<STR_LIT>':all_objectids,<EOL>'<STR_LIT>':period_tolerance,<EOL>'<STR_LIT>':actual_periodicvars,<EOL>'<STR_LIT>':recovered_periodicvars,<EOL>'<STR_LIT>':alias_twice_periodicvars,<EOL>'<STR_LIT>':alias_half_periodicvars,<EOL>'<STR_LIT>':resdict}<EOL>outfile = os.path.join(simbasedir,'<STR_LIT>')<EOL>with open(outfile, '<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(outdict, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>return outdict<EOL><DEDENT>else:<EOL><INDENT>LOGERROR(<EOL>'<STR_LIT>' %<EOL>pfpkldir<EOL>)<EOL>return None<EOL><DEDENT>

This is a parallel driver for `periodicvar_recovery`.

    Parameters
    ----------

    simbasedir : str
        The base directory where all of the fake LCs and period-finding results
        are.

    period_tolerance : float
        The maximum difference that this function will consider between an
        actual period (or its aliases) and a recovered period to consider it as
        as a 'recovered' period.

    liststartindex : int
        The starting index of processing. This refers to the filename list
        generated by running `glob.glob` on the period-finding result pickles in
        `simbasedir/periodfinding`.

    listmaxobjects : int
        The maximum number of objects to process in this run. Use this with
        `liststartindex` to effectively distribute working on a large list of
        input period-finding result pickles over several sessions or machines.

    nperiodworkers : int
        This is the number of parallel period-finding worker processes to use.

    Returns
    -------

    str
        Returns the filename of the pickle produced containing all of the period
        recovery results.