if not outfile and lcdict['<STR_LIT>']:<EOL><INDENT>outfile = '<STR_LIT>' % lcdict['<STR_LIT>']<EOL><DEDENT>elif not outfile and not lcdict['<STR_LIT>']:<EOL><INDENT>outfile = '<STR_LIT>'<EOL><DEDENT>with open(outfile,'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(lcdict, outfd, protocol=pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>if os.path.exists(outfile):<EOL><INDENT>LOGINFO('<STR_LIT>' % (lcdict['<STR_LIT>'],<EOL>outfile))<EOL>return outfile<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>')<EOL>return None<EOL><DEDENT>

This just writes the lcdict to a pickle.

    If outfile is None, then will try to get the name from the
    lcdict['objectid'] and write to <objectid>-hptxtlc.pkl. If that fails, will
    write to a file named hptxtlc.pkl'.


try:<EOL><INDENT>if lcfile.endswith('<STR_LIT>'):<EOL><INDENT>infd = gzip.open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>infd = open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>lcdict = pickle.load(infd)<EOL>infd.close()<EOL>return lcdict<EOL><DEDENT>except UnicodeDecodeError:<EOL><INDENT>if lcfile.endswith('<STR_LIT>'):<EOL><INDENT>infd = gzip.open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>infd = open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % lcfile)<EOL>lcdict = pickle.load(infd, encoding='<STR_LIT>')<EOL>infd.close()<EOL>return lcdict<EOL><DEDENT>

This just reads a pickle LC. Returns an lcdict.


<EOL>lcdict = read_hatpi_textlc(lclist[<NUM_LIT:0>])<EOL>lccounter = <NUM_LIT:0><EOL>lcdict['<STR_LIT>'] = {lccounter: os.path.abspath(lclist[<NUM_LIT:0>])}<EOL>lcdict['<STR_LIT>'] = np.full_like(lcdict['<STR_LIT>'], lccounter)<EOL>if normalize:<EOL><INDENT>for col in MAGCOLS:<EOL><INDENT>if col in lcdict:<EOL><INDENT>thismedval = np.nanmedian(lcdict[col])<EOL>if col in ('<STR_LIT>','<STR_LIT>','<STR_LIT>'):<EOL><INDENT>lcdict[col] = lcdict[col] / thismedval<EOL><DEDENT>else:<EOL><INDENT>lcdict[col] = lcdict[col] - thismedval<EOL><DEDENT><DEDENT><DEDENT><DEDENT>for lcf in lclist[<NUM_LIT:1>:]:<EOL><INDENT>thislcd = read_hatpi_textlc(lcf)<EOL>if thislcd['<STR_LIT>'] != lcdict['<STR_LIT>']:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% (lcf, lclist[<NUM_LIT:0>]))<EOL>continue<EOL><DEDENT>else:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>% (lcf,<EOL>thislcd['<STR_LIT>']['<STR_LIT>'],<EOL>lclist[<NUM_LIT:0>],<EOL>lcdict[lcdict['<STR_LIT>'][<NUM_LIT:0>]].size))<EOL>lccounter = lccounter + <NUM_LIT:1><EOL>lcdict['<STR_LIT>'][lccounter] = os.path.abspath(lcf)<EOL>lcdict['<STR_LIT>'] = np.concatenate((<EOL>lcdict['<STR_LIT>'],<EOL>np.full_like(thislcd['<STR_LIT>'],lccounter)<EOL>))<EOL>for col in lcdict['<STR_LIT>']:<EOL><INDENT>if normalize and col in MAGCOLS:<EOL><INDENT>thismedval = np.nanmedian(thislcd[col])<EOL>if col in ('<STR_LIT>','<STR_LIT>','<STR_LIT>'):<EOL><INDENT>thislcd[col] = thislcd[col] / thismedval<EOL><DEDENT>else:<EOL><INDENT>thislcd[col] = thislcd[col] - thismedval<EOL><DEDENT><DEDENT>lcdict[col] = np.concatenate((lcdict[col], thislcd[col]))<EOL><DEDENT><DEDENT><DEDENT>lcdict['<STR_LIT>']['<STR_LIT>'] = lcdict[lcdict['<STR_LIT>'][<NUM_LIT:0>]].size<EOL>lcdict['<STR_LIT>']['<STR_LIT>'] = [<EOL>'<STR_LIT>' % x for x in np.unique(lcdict['<STR_LIT>']).tolist()<EOL>]<EOL>lcdict['<STR_LIT>'] = lccounter + <NUM_LIT:1><EOL>if sortby and sortby in [x[<NUM_LIT:0>] for x in COLDEFS]:<EOL><INDENT>LOGINFO('<STR_LIT>' % sortby)<EOL>sortind = np.argsort(lcdict[sortby])<EOL>for col in lcdict['<STR_LIT>']:<EOL><INDENT>lcdict[col] = lcdict[col][sortind]<EOL><DEDENT>lcdict['<STR_LIT>'] = lcdict['<STR_LIT>'][sortind]<EOL><DEDENT>LOGINFO('<STR_LIT>' %<EOL>lcdict['<STR_LIT>']['<STR_LIT>'])<EOL>return lcdict<EOL>

This concatenates a list of light curves.

    Does not care about overlaps or duplicates. The light curves must all be
    from the same aperture.

    The intended use is to concatenate light curves across CCDs or instrument
    changes for a single object. These can then be normalized later using
    standard astrobase tools to search for variablity and/or periodicity.

    sortby is a column to sort the final concatenated light curve by in
    ascending order.

    If normalize is True, then each light curve's magnitude columns are
    normalized to zero.

    The returned lcdict has an extra column: 'lcn' that tracks which measurement
    belongs to which input light curve. This can be used with
    lcdict['concatenated'] which relates input light curve index to input light
    curve filepath. Finally, there is an 'nconcatenated' key in the lcdict that
    contains the total number of concatenated light curves.


LOGINFO('<STR_LIT>'<EOL>% (objectid, aperture, lcbasedir))<EOL>if recursive is False:<EOL><INDENT>matching = glob.glob(os.path.join(lcbasedir,<EOL>'<STR_LIT>' % (objectid,<EOL>aperture,<EOL>postfix)))<EOL><DEDENT>else:<EOL><INDENT>if sys.version_info[:<NUM_LIT:2>] > (<NUM_LIT:3>,<NUM_LIT:4>):<EOL><INDENT>matching = glob.glob(os.path.join(lcbasedir,<EOL>'<STR_LIT>',<EOL>'<STR_LIT>' % (objectid,<EOL>aperture,<EOL>postfix)),<EOL>recursive=True)<EOL>LOGINFO('<STR_LIT>' % (len(matching), repr(matching)))<EOL><DEDENT>else:<EOL><INDENT>walker = os.walk(lcbasedir)<EOL>matching = []<EOL>for root, dirs, _files in walker:<EOL><INDENT>for sdir in dirs:<EOL><INDENT>searchpath = os.path.join(root,<EOL>sdir,<EOL>'<STR_LIT>' % (objectid,<EOL>aperture,<EOL>postfix))<EOL>foundfiles = glob.glob(searchpath)<EOL>if foundfiles:<EOL><INDENT>matching.extend(foundfiles)<EOL>LOGINFO(<EOL>'<STR_LIT>' % (repr(foundfiles),<EOL>os.path.join(root,sdir))<EOL>)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if matching and len(matching) > <NUM_LIT:0>:<EOL><INDENT>clcdict = concatenate_textlcs(matching,<EOL>sortby=sortby,<EOL>normalize=normalize)<EOL>return clcdict<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>' %<EOL>(objectid, aperture))<EOL>return None<EOL><DEDENT>

This concatenates all text LCs for an objectid with the given aperture.

    Does not care about overlaps or duplicates. The light curves must all be
    from the same aperture.

    The intended use is to concatenate light curves across CCDs or instrument
    changes for a single object. These can then be normalized later using
    standard astrobase tools to search for variablity and/or periodicity.

    lcbasedir is the directory to start searching in.

    objectid is the object to search for.

    aperture is the aperture postfix to use: (TF1 = aperture 1,
                                              TF2 = aperture 2,
                                              TF3 = aperture 3)

    sortby is a column to sort the final concatenated light curve by in
    ascending order.

    If normalize is True, then each light curve's magnitude columns are
    normalized to zero, and the whole light curve is then normalized to the
    global median magnitude for each magnitude column.

    If recursive is True, then the function will search recursively in lcbasedir
    for any light curves matching the specified criteria. This may take a while,
    especially on network filesystems.

    The returned lcdict has an extra column: 'lcn' that tracks which measurement
    belongs to which input light curve. This can be used with
    lcdict['concatenated'] which relates input light curve index to input light
    curve filepath. Finally, there is an 'nconcatenated' key in the lcdict that
    contains the total number of concatenated light curves.


concatlcd = concatenate_textlcs_for_objectid(lcbasedir,<EOL>objectid,<EOL>aperture=aperture,<EOL>sortby=sortby,<EOL>normalize=normalize,<EOL>recursive=recursive)<EOL>if not outdir:<EOL><INDENT>outdir = '<STR_LIT>'<EOL><DEDENT>if not os.path.exists(outdir):<EOL><INDENT>os.mkdir(outdir)<EOL><DEDENT>outfpath = os.path.join(outdir, '<STR_LIT>' % (concatlcd['<STR_LIT>'],<EOL>aperture))<EOL>pklc = lcdict_to_pickle(concatlcd, outfile=outfpath)<EOL>return pklc<EOL>

This concatenates all text LCs for the given object and writes to a pklc.

    Basically a rollup for the concatenate_textlcs_for_objectid and
    lcdict_to_pickle functions.


lcbasedir, objectid, kwargs = task<EOL>try:<EOL><INDENT>return concat_write_pklc(lcbasedir, objectid, **kwargs)<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>'<EOL>% (objectid, lcbasedir))<EOL>return None<EOL><DEDENT>

This is a worker for the function below.

task[0] = lcbasedir
task[1] = objectid
task[2] = {'aperture','postfix','sortby','normalize','outdir','recursive'}


if not outdir:<EOL><INDENT>outdir = '<STR_LIT>'<EOL><DEDENT>if not os.path.exists(outdir):<EOL><INDENT>os.mkdir(outdir)<EOL><DEDENT>tasks = [(lcbasedir, x, {'<STR_LIT>':aperture,<EOL>'<STR_LIT>':postfix,<EOL>'<STR_LIT>':sortby,<EOL>'<STR_LIT>':normalize,<EOL>'<STR_LIT>':outdir,<EOL>'<STR_LIT>':recursive}) for x in objectidlist]<EOL>pool = mp.Pool(nworkers, maxtasksperchild=maxworkertasks)<EOL>results = pool.map(parallel_concat_worker, tasks)<EOL>pool.close()<EOL>pool.join()<EOL>return {x:y for (x,y) in zip(objectidlist, results)}<EOL>

This concatenates all text LCs for the given objectidlist.


lcaps = {}<EOL>framekeys = []<EOL>for lc in lclist:<EOL><INDENT>lcd = read_hatpi_textlc(lc)<EOL>for col in lcd['<STR_LIT>']:<EOL><INDENT>if col.startswith('<STR_LIT>'):<EOL><INDENT>lcaps[col] = lcd<EOL><DEDENT><DEDENT>thisframekeys = lcd['<STR_LIT>'].tolist()<EOL>framekeys.extend(thisframekeys)<EOL><DEDENT>framekeys = sorted(list(set(framekeys)))<EOL>

This merges all TFA text LCs with separate apertures for a single object.

    The framekey column will be used as the join column across all light curves
    in lclist. Missing values will be filled in with nans. This function assumes
    all light curves are in the format specified in COLDEFS above and readable
    by read_hatpi_textlc above (i.e. have a single column for TFA mags for a
    specific aperture at the end).


def read_hatpi_binnedlc(binnedpklf, textlcf, timebinsec):

LOGINFO('<STR_LIT>' % binnedpklf)<EOL>lcdict = read_hatpi_textlc(textlcf)<EOL>if binnedpklf.endswith('<STR_LIT>'):<EOL><INDENT>infd = gzip.open(binnedpklf,'<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>infd = open(binnedpklf,'<STR_LIT:rb>')<EOL><DEDENT>try:<EOL><INDENT>binned = pickle.load(infd)<EOL><DEDENT>except Exception as e:<EOL><INDENT>infd.seek(<NUM_LIT:0>)<EOL>binned = pickle.load(infd, encoding='<STR_LIT>')<EOL><DEDENT>infd.close()<EOL>blckeys = binned.keys()<EOL>lcdict['<STR_LIT>'] = {}<EOL>for key in blckeys:<EOL><INDENT>if (key == '<STR_LIT>' and<EOL>'<STR_LIT>' in binned[key] and<EOL>'<STR_LIT>' in binned[key] and<EOL>'<STR_LIT>' in binned[key]):<EOL><INDENT>ap0mad = np.nanmedian(np.abs(binned[key]['<STR_LIT>'] -<EOL>np.nanmedian(binned[key]['<STR_LIT>'])))<EOL>ap1mad = np.nanmedian(np.abs(binned[key]['<STR_LIT>'] -<EOL>np.nanmedian(binned[key]['<STR_LIT>'])))<EOL>ap2mad = np.nanmedian(np.abs(binned[key]['<STR_LIT>'] -<EOL>np.nanmedian(binned[key]['<STR_LIT>'])))<EOL>lcdict['<STR_LIT>']['<STR_LIT>'] = {'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':np.full_like(binned[key]['<STR_LIT>'],<EOL>ap0mad),<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':timebinsec}<EOL>lcdict['<STR_LIT>']['<STR_LIT>'] = {'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':np.full_like(binned[key]['<STR_LIT>'],<EOL>ap1mad),<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':timebinsec}<EOL>lcdict['<STR_LIT>']['<STR_LIT>'] = {'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':np.full_like(binned[key]['<STR_LIT>'],<EOL>ap2mad),<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':timebinsec}<EOL><DEDENT>if ((key == '<STR_LIT>' or key == '<STR_LIT>') and<EOL>'<STR_LIT>' in binned[key]):<EOL><INDENT>ap0mad = np.nanmedian(np.abs(binned[key]['<STR_LIT>'] -<EOL>np.nanmedian(binned[key]['<STR_LIT>'])))<EOL>lcdict['<STR_LIT>']['<STR_LIT>'] = {'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':np.full_like(binned[key]['<STR_LIT>'],<EOL>ap0mad),<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':timebinsec}<EOL><DEDENT>if ((key == '<STR_LIT>' or key == '<STR_LIT>') and<EOL>'<STR_LIT>' in binned[key]):<EOL><INDENT>ap0mad = np.nanmedian(np.abs(binned[key]['<STR_LIT>'] -<EOL>np.nanmedian(binned[key]['<STR_LIT>'])))<EOL>lcdict['<STR_LIT>']['<STR_LIT>'] = {'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':np.full_like(binned[key]['<STR_LIT>'],<EOL>ap0mad),<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':timebinsec}<EOL><DEDENT>if ((key == '<STR_LIT>' or key == '<STR_LIT>') and<EOL>'<STR_LIT>' in binned[key]):<EOL><INDENT>ap0mad = np.nanmedian(np.abs(binned[key]['<STR_LIT>'] -<EOL>np.nanmedian(binned[key]['<STR_LIT>'])))<EOL>lcdict['<STR_LIT>']['<STR_LIT>'] = {'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':np.full_like(binned[key]['<STR_LIT>'],<EOL>ap0mad),<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':binned[key]['<STR_LIT>'],<EOL>'<STR_LIT>':timebinsec}<EOL><DEDENT><DEDENT>if lcdict['<STR_LIT>']:<EOL><INDENT>return lcdict<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>' % binnedpklf)<EOL>return None<EOL><DEDENT>

This reads a binnedlc pickle produced by the HATPI prototype pipeline.

    Converts it into a standard lcdict as produced by the read_hatpi_textlc
    function above by using the information in unbinnedtextlc for the same
    object.

    Adds a 'binned' key to the standard lcdict containing the binned mags, etc.


def generate_hatpi_binnedlc_pkl(binnedpklf, textlcf, timebinsec,<EOL>outfile=None):

binlcdict = read_hatpi_binnedlc(binnedpklf, textlcf, timebinsec)<EOL>if binlcdict:<EOL><INDENT>if outfile is None:<EOL><INDENT>outfile = os.path.join(<EOL>os.path.dirname(binnedpklf),<EOL>'<STR_LIT>' % (<EOL>os.path.basename(binnedpklf).replace('<STR_LIT>','<STR_LIT>')<EOL>)<EOL>)<EOL><DEDENT>return lcdict_to_pickle(binlcdict, outfile=outfile)<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>' % binnedpklf)<EOL>return None<EOL><DEDENT>

This reads the binned LC and writes it out to a pickle.


binnedpkls = sorted(glob.glob(os.path.join(binnedpkldir, binnedpklglob)))<EOL>textlcs = []<EOL>for bpkl in binnedpkls:<EOL><INDENT>objectid = HATIDREGEX.findall(bpkl)<EOL>if objectid is not None:<EOL><INDENT>objectid = objectid[<NUM_LIT:0>]<EOL><DEDENT>searchpath = os.path.join(textlcdir, '<STR_LIT>' % (objectid, textlcglob))<EOL>textlcf = glob.glob(searchpath)<EOL>if textlcf:<EOL><INDENT>textlcs.append(textlcf)<EOL><DEDENT>else:<EOL><INDENT>textlcs.append(None)<EOL><DEDENT><DEDENT>

This generates the binnedlc pkls for a directory of such files.

FIXME: finish this


if fovcatalog.endswith('<STR_LIT>'):<EOL><INDENT>catfd = gzip.open(fovcatalog)<EOL><DEDENT>else:<EOL><INDENT>catfd = open(fovcatalog)<EOL><DEDENT>fovcat = np.genfromtxt(catfd,<EOL>usecols=fovcatalog_columns,<EOL>names=fovcatalog_colnames,<EOL>dtype=fovcatalog_colformats)<EOL>catfd.close()<EOL>pklclist = sorted(glob.glob(os.path.join(pklcdir, '<STR_LIT>')))<EOL>updatedpklcs, failedpklcs = [], []<EOL>for pklc in pklclist:<EOL><INDENT>lcdict = read_hatpi_pklc(pklc)<EOL>objectid = lcdict['<STR_LIT>']<EOL>catind = np.where(fovcat['<STR_LIT>'] == objectid)<EOL>if len(catind) > <NUM_LIT:0> and catind[<NUM_LIT:0>]:<EOL><INDENT>lcdict['<STR_LIT>'].update(<EOL>{x:y for x,y in zip(<EOL>fovcatalog_colnames,<EOL>[np.asscalar(fovcat[z][catind]) for<EOL>z in fovcatalog_colnames]<EOL>)<EOL>}<EOL>)<EOL>with open(pklc+'<STR_LIT>','<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(lcdict, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>if os.path.exists(pklc+'<STR_LIT>'):<EOL><INDENT>shutil.move(pklc+'<STR_LIT>',pklc)<EOL>LOGINFO('<STR_LIT>' %<EOL>(pklc, objectid,<EOL>lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>lcdict['<STR_LIT>']['<STR_LIT>']))<EOL>updatedpklcs.append(pklc)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>failedpklcs.append(pklc)<EOL><DEDENT><DEDENT>return updatedpklcs, failedpklcs<EOL>

Adds catalog info to objectinfo key of all pklcs in lcdir.

    If fovcatalog, fovcatalog_columns, fovcatalog_colnames are provided, uses
    them to find all the additional information listed in the fovcatalog_colname
    keys, and writes this info to the objectinfo key of each lcdict. This makes
    it easier for astrobase tools to work on these light curve.

    The default set up for fovcatalog is to use a text file generated by the
    HATPI pipeline before auto-calibrating a field. The format is specified as
    above in _columns,  _colnames, and _colformats.


lcfname = os.path.basename(hatlc)<EOL>if '<STR_LIT>' in lcfname:<EOL><INDENT>lcf = gzip.open(hatlc,'<STR_LIT:rb>')<EOL><DEDENT>elif '<STR_LIT>' in lcfname:<EOL><INDENT>lcf = bz2.BZ2File(hatlc, '<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>lcf = open(hatlc,'<STR_LIT:rb>')<EOL><DEDENT>if '<STR_LIT>' in lcfname and HAVEPYFITS:<EOL><INDENT>hdulist = pyfits.open(lcf)<EOL>objectinfo = hdulist[<NUM_LIT:0>].header<EOL>objectlc = hdulist[<NUM_LIT:1>].data<EOL>lccols = objectlc.columns.names<EOL>hdulist.close()<EOL>lcf.close()<EOL>lcdict = {}<EOL>for col in lccols:<EOL><INDENT>lcdict[col] = np.array(objectlc[col])<EOL><DEDENT>lcdict['<STR_LIT>'] = objectinfo['<STR_LIT>']<EOL>lcdict['<STR_LIT>'] = objectinfo['<STR_LIT>']<EOL>lcdict['<STR_LIT>'] = objectinfo['<STR_LIT>']<EOL>lcdict['<STR_LIT>'] = objectinfo['<STR_LIT>']<EOL>lcdict['<STR_LIT>'] = [objectinfo[x] for x in ('<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>')]<EOL>lcdict['<STR_LIT>'] = objectinfo['<STR_LIT>']<EOL>lcdict['<STR_LIT>'] = objectinfo['<STR_LIT>']<EOL>lcdict['<STR_LIT>'] = objectinfo['<STR_LIT>']<EOL>lcdict['<STR_LIT>'] = lccols<EOL>return lcdict<EOL><DEDENT>elif '<STR_LIT>' in lcfname and not HAVEPYFITS:<EOL><INDENT>print("<STR_LIT>" % lcfname)<EOL>return<EOL><DEDENT>elif '<STR_LIT>' in lcfname or '<STR_LIT>' in lcfname:<EOL><INDENT>lcflines = lcf.read().decode().split('<STR_LIT:\n>')<EOL>lcf.close()<EOL>objectdata = [x for x in lcflines if x.startswith('<STR_LIT:#>')]<EOL>objectlc = [x for x in lcflines if not x.startswith('<STR_LIT:#>')]<EOL>objectlc = [x for x in objectlc if len(x) > <NUM_LIT:1>]<EOL>if '<STR_LIT>' in lcfname:<EOL><INDENT>objectlc = [x.split('<STR_LIT:U+002C>') for x in objectlc]<EOL><DEDENT>else:<EOL><INDENT>objectlc = [x.split() for x in objectlc]<EOL><DEDENT>objectlc = list(zip(*objectlc))<EOL>objectdata = [x.strip('<STR_LIT:#>') for x in objectdata]<EOL>objectdata = [x.strip() for x in objectdata]<EOL>objectdata = [x for x in objectdata if len(x) > <NUM_LIT:0>]<EOL>hatid, twomassid = objectdata[<NUM_LIT:0>].split('<STR_LIT>')<EOL>ra, dec = objectdata[<NUM_LIT:1>].split('<STR_LIT:U+002CU+0020>')<EOL>ra = float(ra.split('<STR_LIT>')[-<NUM_LIT:1>].strip('<STR_LIT>'))<EOL>dec = float(dec.split('<STR_LIT>')[-<NUM_LIT:1>].strip('<STR_LIT>'))<EOL>vmag, rmag, imag, jmag, hmag, kmag = objectdata[<NUM_LIT:2>].split('<STR_LIT:U+002CU+0020>')<EOL>vmag = float(vmag.split('<STR_LIT>')[-<NUM_LIT:1>])<EOL>rmag = float(rmag.split('<STR_LIT>')[-<NUM_LIT:1>])<EOL>imag = float(imag.split('<STR_LIT>')[-<NUM_LIT:1>])<EOL>jmag = float(jmag.split('<STR_LIT>')[-<NUM_LIT:1>])<EOL>hmag = float(hmag.split('<STR_LIT>')[-<NUM_LIT:1>])<EOL>kmag = float(kmag.split('<STR_LIT>')[-<NUM_LIT:1>])<EOL>ndet = int(objectdata[<NUM_LIT:3>].split('<STR_LIT>')[-<NUM_LIT:1>])<EOL>hatstations = objectdata[<NUM_LIT:4>].split('<STR_LIT>')[-<NUM_LIT:1>]<EOL>filterhead_ind = objectdata.index('<STR_LIT>')<EOL>columnhead_ind = objectdata.index('<STR_LIT>')<EOL>filters = objectdata[filterhead_ind:columnhead_ind]<EOL>columndefs = objectdata[columnhead_ind+<NUM_LIT:1>:]<EOL>columns = []<EOL>for line in columndefs:<EOL><INDENT>colnum, colname, coldesc = line.split('<STR_LIT>')<EOL>columns.append(colname)<EOL><DEDENT>lcdict = {}<EOL>for ind, col in enumerate(columns):<EOL><INDENT>lcdict[col] = np.array([TEXTLC_OUTPUT_COLUMNS[col][<NUM_LIT:3>](x)<EOL>for x in objectlc[ind]])<EOL><DEDENT>lcdict['<STR_LIT>'] = hatid<EOL>lcdict['<STR_LIT>'] = twomassid.replace('<STR_LIT>','<STR_LIT>')<EOL>lcdict['<STR_LIT>'] = ra<EOL>lcdict['<STR_LIT>'] = dec<EOL>lcdict['<STR_LIT>'] = [vmag, rmag, imag, jmag, hmag, kmag]<EOL>lcdict['<STR_LIT>'] = ndet<EOL>lcdict['<STR_LIT>'] = hatstations.split('<STR_LIT:U+002CU+0020>')<EOL>lcdict['<STR_LIT>'] = filters[<NUM_LIT:1>:]<EOL>lcdict['<STR_LIT>'] = columns<EOL>return lcdict<EOL><DEDENT>

This reads a consolidated HAT LC written by the functions above.

Returns a dict.


LOGINFO('<STR_LIT>'.format(lcpath))<EOL>N_lines_to_parse_comments = <NUM_LIT:50><EOL>with open(lcpath, '<STR_LIT:rb>') as file:<EOL><INDENT>head = [next(file) for ind in range(N_lines_to_parse_comments)]<EOL><DEDENT>N_comment_lines = len([l for l in head if l.decode('<STR_LIT>')[<NUM_LIT:0>] == '<STR_LIT:#>'])<EOL>if N_comment_lines < N_lines_to_parse_comments:<EOL><INDENT>LOGERROR(<EOL>'<STR_LIT>'.format(fpath=lcpath)<EOL>)<EOL>return None<EOL><DEDENT>first_data_line = list(<EOL>filter(None, head[N_comment_lines].decode('<STR_LIT>').split())<EOL>)<EOL>N_cols = len(first_data_line)<EOL>if N_cols == <NUM_LIT>:<EOL><INDENT>colformat = '<STR_LIT>'<EOL><DEDENT>elif N_cols == <NUM_LIT:20>:<EOL><INDENT>colformat = '<STR_LIT>'<EOL><DEDENT>elif N_cols == <NUM_LIT:32>:<EOL><INDENT>colformat = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>"<EOL>"<STR_LIT>".format(fpath=lcpath,<EOL>ncols=N_cols))<EOL>return None<EOL><DEDENT>if colformat == '<STR_LIT>':<EOL><INDENT>col_names = ['<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>']<EOL>col_dtypes = ['<STR_LIT>',float,<EOL>float,float,'<STR_LIT>',<EOL>float,float,'<STR_LIT>',<EOL>float,float,'<STR_LIT>',<EOL>float,float,float,<EOL>float,float,float]<EOL>dtype_pairs = [el for el in zip(col_names, col_dtypes)]<EOL>data = np.genfromtxt(lcpath, names=col_names, dtype=col_dtypes,<EOL>skip_header=N_comment_lines, delimiter=None)<EOL>out = {}<EOL>for ix in range(len(data.dtype.names)):<EOL><INDENT>out[data.dtype.names[ix]] = data[data.dtype.names[ix]]<EOL><DEDENT><DEDENT>elif colformat == '<STR_LIT>':<EOL><INDENT>col_names = ['<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>',<EOL>'<STR_LIT>','<STR_LIT>','<STR_LIT>']<EOL>col_dtypes = ['<STR_LIT>',float,<EOL>float,float,'<STR_LIT>',<EOL>float,float,'<STR_LIT>',<EOL>float,float,'<STR_LIT>',<EOL>float,float,float,<EOL>float,float,float,<EOL>float,float,float]<EOL>dtype_pairs = [el for el in zip(col_names, col_dtypes)]<EOL>data = np.genfromtxt(lcpath, names=col_names, dtype=col_dtypes,<EOL>skip_header=N_comment_lines, delimiter=None)<EOL>out = {}<EOL>for ix in range(len(data.dtype.names)):<EOL><INDENT>out[data.dtype.names[ix]] = data[data.dtype.names[ix]]<EOL><DEDENT><DEDENT>elif colformat == '<STR_LIT>':<EOL><INDENT>col_names = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>out = astascii.read(lcpath, names=col_names, comment='<STR_LIT:#>')<EOL><DEDENT>return out<EOL>

Read .epdlc, and .tfalc light curves and return a corresponding labelled
dict (if LC from <2012) or astropy table (if >=2012). Each has different
keys that can be accessed via .keys()

Input:
lcpath: path (string) to light curve data, which is a textfile with HAT
LC data.

Example:
dat = read_original_textlc('HAT-115-0003266.epdlc')


return re.sub(r"<STR_LIT>", "<STR_LIT:U+0020>", value).strip()<EOL>

Replace all sequences of whitespace chars with a single space.


outfile = '<STR_LIT>' % sqlitecurve<EOL>try:<EOL><INDENT>if os.path.exists(outfile) and not force:<EOL><INDENT>os.remove(sqlitecurve)<EOL>return outfile<EOL><DEDENT>else:<EOL><INDENT>with open(sqlitecurve,'<STR_LIT:rb>') as infd:<EOL><INDENT>with gzip.open(outfile,'<STR_LIT:wb>') as outfd:<EOL><INDENT>shutil.copyfileobj(infd, outfd)<EOL><DEDENT><DEDENT>if os.path.exists(outfile):<EOL><INDENT>os.remove(sqlitecurve)<EOL>return outfile<EOL><DEDENT><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>return None<EOL><DEDENT>

This just compresses the sqlitecurve. Should be independent of OS.


outfile = sqlitecurve.replace('<STR_LIT>','<STR_LIT>')<EOL>try:<EOL><INDENT>if os.path.exists(outfile) and not force:<EOL><INDENT>return outfile<EOL><DEDENT>else:<EOL><INDENT>with gzip.open(sqlitecurve,'<STR_LIT:rb>') as infd:<EOL><INDENT>with open(outfile,'<STR_LIT:wb>') as outfd:<EOL><INDENT>shutil.copyfileobj(infd, outfd)<EOL><DEDENT><DEDENT>if os.path.exists(outfile):<EOL><INDENT>return outfile<EOL><DEDENT><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>return None<EOL><DEDENT>

This just uncompresses the sqlitecurve. Should be independent of OS.


<EOL>if force:<EOL><INDENT>cmd = '<STR_LIT>' % sqlitecurve<EOL><DEDENT>else:<EOL><INDENT>cmd = '<STR_LIT>' % sqlitecurve<EOL><DEDENT>try:<EOL><INDENT>outfile = '<STR_LIT>' % sqlitecurve<EOL>if os.path.exists(outfile) and not force:<EOL><INDENT>os.remove(sqlitecurve)<EOL>return outfile<EOL><DEDENT>else:<EOL><INDENT>subprocess.check_output(cmd, shell=True)<EOL>if os.path.exists(outfile):<EOL><INDENT>return outfile<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT><DEDENT><DEDENT>except subprocess.CalledProcessError:<EOL><INDENT>return None<EOL><DEDENT>

This just compresses the sqlitecurve in gzip format.

    FIXME: this doesn't work with gzip < 1.6 or non-GNU gzip (probably).


<EOL>cmd = '<STR_LIT>' % sqlitecurve<EOL>try:<EOL><INDENT>subprocess.check_output(cmd, shell=True)<EOL>return sqlitecurve.replace('<STR_LIT>','<STR_LIT>')<EOL><DEDENT>except subprocess.CalledProcessError:<EOL><INDENT>return None<EOL><DEDENT>

This just uncompresses the sqlitecurve in gzip format.

    FIXME: this doesn't work with gzip < 1.6 or non-GNU gzip (probably).


<EOL>stringelems = _squeeze(filterstring).lower()<EOL>stringelems = filterstring.replace('<STR_LIT:(>','<STR_LIT>')<EOL>stringelems = stringelems.replace('<STR_LIT:)>','<STR_LIT>')<EOL>stringelems = stringelems.replace('<STR_LIT:U+002C>','<STR_LIT>')<EOL>stringelems = stringelems.replace("<STR_LIT:'>",'<STR_LIT:">')<EOL>stringelems = stringelems.replace('<STR_LIT:\n>','<STR_LIT:U+0020>')<EOL>stringelems = stringelems.replace('<STR_LIT:\t>','<STR_LIT:U+0020>')<EOL>stringelems = _squeeze(stringelems)<EOL>stringelems = stringelems.split('<STR_LIT:U+0020>')<EOL>stringelems = [x.strip() for x in stringelems]<EOL>stringwords = []<EOL>for x in stringelems:<EOL><INDENT>try:<EOL><INDENT>float(x)<EOL><DEDENT>except ValueError as e:<EOL><INDENT>stringwords.append(x)<EOL><DEDENT><DEDENT>stringwords2 = []<EOL>for x in stringwords:<EOL><INDENT>if not(x.startswith('<STR_LIT:">') and x.endswith('<STR_LIT:">')):<EOL><INDENT>stringwords2.append(x)<EOL><DEDENT><DEDENT>stringwords2 = [x for x in stringwords2 if len(x) > <NUM_LIT:0>]<EOL>wordset = set(stringwords2)<EOL>allowedwords = SQLITE_ALLOWED_WORDS + lccolumns<EOL>checkset = set(allowedwords)<EOL>validatecheck = list(wordset - checkset)<EOL>if len(validatecheck) > <NUM_LIT:0>:<EOL><INDENT>LOGWARNING("<STR_LIT>"<EOL>"<STR_LIT>" % filterstring)<EOL>return None<EOL><DEDENT>else:<EOL><INDENT>return filterstring<EOL><DEDENT>

This validates the sqlitecurve filter string.

    This MUST be valid SQL but not contain any commands.


<EOL>try:<EOL><INDENT>if '<STR_LIT>' in lcfile[-<NUM_LIT:4>:]:<EOL><INDENT>lcf = _uncompress_sqlitecurve(lcfile)<EOL><DEDENT>else:<EOL><INDENT>lcf = lcfile<EOL><DEDENT>db = sql.connect(lcf)<EOL>cur = db.cursor()<EOL>query = ("<STR_LIT>")<EOL>cur.execute(query)<EOL>objectinfo = cur.fetchone()<EOL>query = ("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>cur.execute(query)<EOL>lcinfo = cur.fetchone()<EOL>(lcversion, lcdatarelease, lccols, lcsortcol,<EOL>lcapertures, lcbestaperture,<EOL>objinfocols, objidcol,<EOL>lcunixtime, lcgitrev, lccomment) = lcinfo<EOL>lcapertures = json.loads(lcapertures)<EOL>lcbestaperture = json.loads(lcbestaperture)<EOL>objectinfokeys = objinfocols.split('<STR_LIT:U+002C>')<EOL>objectinfodict = {x:y for (x,y) in zip(objectinfokeys, objectinfo)}<EOL>objectid = objectinfodict[objidcol]<EOL>query = ("<STR_LIT>")<EOL>cur.execute(query)<EOL>filterinfo = cur.fetchall()<EOL>if columns and all([x in lccols.split('<STR_LIT:U+002C>') for x in columns]):<EOL><INDENT>LOGINFO('<STR_LIT>' % columns)<EOL>proceed = True<EOL><DEDENT>elif columns is None:<EOL><INDENT>columns = lccols.split('<STR_LIT:U+002C>')<EOL>proceed = True<EOL><DEDENT>else:<EOL><INDENT>proceed = False<EOL><DEDENT>if not proceed:<EOL><INDENT>if '<STR_LIT>' in lcfile[-<NUM_LIT:4>:] and lcf:<EOL><INDENT>_compress_sqlitecurve(lcf, force=forcerecompress)<EOL><DEDENT>LOGERROR('<STR_LIT>')<EOL>return None, "<STR_LIT>"<EOL><DEDENT>lcdict = {'<STR_LIT>':objectid,<EOL>'<STR_LIT>':objectinfodict,<EOL>'<STR_LIT>':objectinfokeys,<EOL>'<STR_LIT>':lcversion,<EOL>'<STR_LIT>':lcdatarelease,<EOL>'<STR_LIT>':columns,<EOL>'<STR_LIT>':lcsortcol,<EOL>'<STR_LIT>':lcapertures,<EOL>'<STR_LIT>':lcbestaperture,<EOL>'<STR_LIT>':lcunixtime,<EOL>'<STR_LIT>':lcgitrev,<EOL>'<STR_LIT>':lccomment,<EOL>'<STR_LIT>':filterinfo}<EOL>if ((sqlfilters is not None) and<EOL>(isinstance(sqlfilters,str) or<EOL>isinstance(sqlfilters, unicode))):<EOL><INDENT>validatedfilters = _validate_sqlitecurve_filters(sqlfilters,<EOL>lccols.split('<STR_LIT:U+002C>'))<EOL>if validatedfilters is not None:<EOL><INDENT>LOGINFO('<STR_LIT>' % validatedfilters)<EOL>filtersok = True<EOL><DEDENT>else:<EOL><INDENT>filtersok = False<EOL><DEDENT><DEDENT>else:<EOL><INDENT>validatedfilters = None<EOL>filtersok = None<EOL><DEDENT>if validatedfilters is not None:<EOL><INDENT>query = (<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>).format(<EOL>columns='<STR_LIT:U+002C>'.join(columns),  <EOL>sqlfilter=validatedfilters,<EOL>sortcol=lcsortcol<EOL>)<EOL>lcdict['<STR_LIT>'] = validatedfilters<EOL><DEDENT>else:<EOL><INDENT>query = ("<STR_LIT>") % (<EOL>'<STR_LIT:U+002C>'.join(columns),<EOL>lcsortcol<EOL>)<EOL><DEDENT>cur.execute(query)<EOL>lightcurve = cur.fetchall()<EOL>if lightcurve and len(lightcurve) > <NUM_LIT:0>:<EOL><INDENT>lightcurve = list(zip(*lightcurve))<EOL>lcdict.update({x:y for (x,y) in zip(lcdict['<STR_LIT>'],<EOL>lightcurve)})<EOL>lcok = True<EOL>lcdict['<STR_LIT>']['<STR_LIT>'] = len(lightcurve[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>' % lcdict['<STR_LIT>'])<EOL>lcdict.update({x:y for (x,y) in<EOL>zip(lcdict['<STR_LIT>'],<EOL>[[] for x in lcdict['<STR_LIT>']])})<EOL>lcok = False<EOL><DEDENT>if filtersok is True and lcok:<EOL><INDENT>statusmsg = '<STR_LIT>'<EOL><DEDENT>elif filtersok is None and lcok:<EOL><INDENT>statusmsg = '<STR_LIT>'<EOL><DEDENT>elif filtersok is False and lcok:<EOL><INDENT>statusmsg = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>statusmsg = '<STR_LIT>'<EOL><DEDENT>returnval = (lcdict, statusmsg)<EOL>if '<STR_LIT>' in lcfile[-<NUM_LIT:4>:] and lcf:<EOL><INDENT>_compress_sqlitecurve(lcf, force=forcerecompress)<EOL><DEDENT>if returnarrays:<EOL><INDENT>for column in lcdict['<STR_LIT>']:<EOL><INDENT>lcdict[column] = np.array([x if x is not None else np.nan<EOL>for x in lcdict[column]])<EOL><DEDENT><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>if not quiet:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' % lcfile)<EOL><DEDENT>returnval = (None, '<STR_LIT>')<EOL>if '<STR_LIT>' in lcfile[-<NUM_LIT:4>:] and lcf:<EOL><INDENT>_compress_sqlitecurve(lcf, force=forcerecompress)<EOL><DEDENT>if raiseonfail:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>return returnval<EOL>

This reads a HAT sqlitecurve and optionally filters it.

    Parameters
    ----------

    lcfile : str
        The path to the HAT sqlitecurve file.

    columns : list
        A list of columns to extract from the ligh curve file. If None, then
        returns all columns present in the latest `columnlist` in the light
        curve.

    sqlfilters : list of str
        If no None, it must be a list of text SQL filters that apply to the
        columns in the lightcurve.

    raiseonfail : bool
        If this is True, an Exception when reading the LC will crash the
        function instead of failing silently and returning None as the result.

    returnarrays : bool
        If this is True, the output lcdict contains columns as np.arrays instead
        of lists. You generally want this to be True.

    forcerecompress : bool
        If True, the sqlitecurve will be recompressed even if a compressed
        version of it is found. This usually happens when sqlitecurve opening is
        interrupted by the OS for some reason, leaving behind a gzipped and
        un-gzipped copy. By default, this function refuses to overwrite the
        existing gzipped version so if the un-gzipped version is corrupt but
        that one isn't, it can be safely recovered.

    quiet : bool
        If True, will not warn about any problems, even if the light curve
        reading fails (the only clue then will be the return value of
        None). Useful for batch processing of many many light curves.

    Returns
    -------

    tuple : (lcdict, status_message)
        A two-element tuple is returned, with the first element being the
        lcdict.


def describe(lcdict, returndesc=False, offsetwith=None):

<EOL>if '<STR_LIT>' in lcdict and '<STR_LIT>' in lcdict['<STR_LIT>'].lower():<EOL><INDENT>return describe_lcc_csv(lcdict, returndesc=returndesc)<EOL><DEDENT>columndefs = []<EOL>for colind, column in enumerate(lcdict['<STR_LIT>']):<EOL><INDENT>if '<STR_LIT:_>' in column:<EOL><INDENT>colkey, colap = column.split('<STR_LIT:_>')<EOL>coldesc = COLUMNDEFS[colkey][<NUM_LIT:0>] % colap<EOL><DEDENT>else:<EOL><INDENT>coldesc = COLUMNDEFS[column][<NUM_LIT:0>]<EOL><DEDENT>columndefstr = '<STR_LIT>' % (colind,<EOL>column,<EOL>coldesc)<EOL>columndefs.append(columndefstr)<EOL><DEDENT>columndefs = '<STR_LIT:\n>'.join(columndefs)<EOL>filterdefs = []<EOL>for row in lcdict['<STR_LIT>']:<EOL><INDENT>filterid, filtername, filterdesc = row<EOL>filterdefstr = '<STR_LIT>' % (filterid,<EOL>filtername,<EOL>filterdesc)<EOL>filterdefs.append(filterdefstr)<EOL><DEDENT>filterdefs = '<STR_LIT:\n>'.join(filterdefs)<EOL>aperturedefs = []<EOL>for key in sorted(lcdict['<STR_LIT>'].keys()):<EOL><INDENT>aperturedefstr = '<STR_LIT>' % (key, lcdict['<STR_LIT>'][key])<EOL>aperturedefs.append(aperturedefstr)<EOL><DEDENT>aperturedefs = '<STR_LIT:\n>'.join(aperturedefs)<EOL>description = DESCTEMPLATE.format(<EOL>objectid=lcdict['<STR_LIT>'],<EOL>hatid=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>twomassid=lcdict['<STR_LIT>']['<STR_LIT>'].strip(),<EOL>ra=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>decl=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>pmra=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>pmra_err=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>pmdecl=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>pmdecl_err=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>jmag=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>hmag=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>kmag=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>bmag=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>vmag=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>sdssg=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>sdssr=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>sdssi=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>ndet=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>lcsortcol=lcdict['<STR_LIT>'],<EOL>lcbestaperture=json.dumps(lcdict['<STR_LIT>'],ensure_ascii=True),<EOL>network=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>stations=lcdict['<STR_LIT>']['<STR_LIT>'],<EOL>lastupdated=lcdict['<STR_LIT>'],<EOL>datarelease=lcdict['<STR_LIT>'],<EOL>lcversion=lcdict['<STR_LIT>'],<EOL>lcserver=lcdict['<STR_LIT>'],<EOL>comment=lcdict['<STR_LIT>'],<EOL>lcfiltersql=(lcdict['<STR_LIT>'] if '<STR_LIT>' in lcdict else '<STR_LIT>'),<EOL>lcnormcols=(lcdict['<STR_LIT>'] if '<STR_LIT>' in lcdict else '<STR_LIT>'),<EOL>filterdefs=filterdefs,<EOL>columndefs=columndefs,<EOL>aperturedefs=aperturedefs<EOL>)<EOL>if offsetwith is not None:<EOL><INDENT>description = textwrap.indent(<EOL>description,<EOL>'<STR_LIT>' % offsetwith,<EOL>lambda line: True<EOL>)<EOL>print(description)<EOL><DEDENT>else:<EOL><INDENT>print(description)<EOL><DEDENT>if returndesc:<EOL><INDENT>return description<EOL><DEDENT>

This describes the light curve object and columns present.

    Parameters
    ----------

    lcdict : dict
        The input lcdict to parse for column and metadata info.

    returndesc : bool
        If True, returns the description string as an str instead of just
        printing it to stdout.

    offsetwith : str
        This is a character to offset the output description lines by. This is
        useful to add comment characters like '#' to the output description
        lines.

    Returns
    -------

    str or None
        If returndesc is True, returns the description lines as a str, otherwise
        returns nothing.


def _smartcast(castee, caster, subval=None):

try:<EOL><INDENT>return caster(castee)<EOL><DEDENT>except Exception as e:<EOL><INDENT>if caster is float or caster is int:<EOL><INDENT>return nan<EOL><DEDENT>elif caster is str:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>return subval<EOL><DEDENT><DEDENT>

This just tries to apply the caster function to castee.

Returns None on failure.


<EOL>headerlines = header.split('<STR_LIT:\n>')<EOL>headerlines = [x.lstrip('<STR_LIT>') for x in headerlines]<EOL>objectstart = headerlines.index('<STR_LIT>')<EOL>metadatastart = headerlines.index('<STR_LIT>')<EOL>camfilterstart = headerlines.index('<STR_LIT>')<EOL>photaperturestart = headerlines.index('<STR_LIT>')<EOL>columnstart = headerlines.index('<STR_LIT>')<EOL>lcstart = headerlines.index('<STR_LIT>')<EOL>objectinfo = headerlines[objectstart+<NUM_LIT:1>:metadatastart-<NUM_LIT:1>]<EOL>metadatainfo = headerlines[metadatastart+<NUM_LIT:1>:camfilterstart-<NUM_LIT:1>]<EOL>camfilterinfo = headerlines[camfilterstart+<NUM_LIT:1>:photaperturestart-<NUM_LIT:1>]<EOL>photapertureinfo = headerlines[photaperturestart+<NUM_LIT:1>:columnstart-<NUM_LIT:1>]<EOL>columninfo = headerlines[columnstart+<NUM_LIT:1>:lcstart-<NUM_LIT:1>]<EOL>metadict = {'<STR_LIT>':{}}<EOL>objectinfo = [x.split('<STR_LIT:;>') for x in objectinfo]<EOL>for elem in objectinfo:<EOL><INDENT>for kvelem in elem:<EOL><INDENT>key, val = kvelem.split('<STR_LIT>',<NUM_LIT:1>)<EOL>metadict['<STR_LIT>'][key.strip()] = (<EOL>_smartcast(val, METAKEYS[key.strip()])<EOL>)<EOL><DEDENT><DEDENT>metadict['<STR_LIT>'] = metadict['<STR_LIT>']['<STR_LIT>'][:]<EOL>del metadict['<STR_LIT>']['<STR_LIT>']<EOL>metadatainfo = [x.split('<STR_LIT:;>') for x in metadatainfo]<EOL>for elem in metadatainfo:<EOL><INDENT>for kvelem in elem:<EOL><INDENT>try:<EOL><INDENT>key, val = kvelem.split('<STR_LIT>',<NUM_LIT:1>)<EOL>if key.strip() == '<STR_LIT>':<EOL><INDENT>val = json.loads(val)<EOL><DEDENT>if key.strip() in ('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>val = int(val)<EOL><DEDENT>if key.strip() == '<STR_LIT>':<EOL><INDENT>val = float(val)<EOL><DEDENT>metadict[key.strip()] = val<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>' % kvelem)<EOL><DEDENT><DEDENT><DEDENT>metadict['<STR_LIT>'] = []<EOL>for row in camfilterinfo:<EOL><INDENT>filterid, filtername, filterdesc = row.split('<STR_LIT>')<EOL>metadict['<STR_LIT>'].append((int(filterid),<EOL>filtername,<EOL>filterdesc))<EOL><DEDENT>metadict['<STR_LIT>'] = {}<EOL>for row in photapertureinfo:<EOL><INDENT>apnum, appix = row.split('<STR_LIT>')<EOL>appix = float(appix.rstrip('<STR_LIT>'))<EOL>metadict['<STR_LIT>'][apnum.strip()] = appix<EOL><DEDENT>metadict['<STR_LIT>'] = []<EOL>for row in columninfo:<EOL><INDENT>colnum, colname, coldesc = row.split('<STR_LIT>')<EOL>metadict['<STR_LIT>'].append(colname)<EOL><DEDENT>return metadict<EOL>

This parses the CSV header from the CSV HAT sqlitecurve.

Returns a dict that can be used to update an existing lcdict with the
relevant metadata info needed to form a full LC.


<EOL>commentchar = headerlines[<NUM_LIT:1>]<EOL>separator = headerlines[<NUM_LIT:2>]<EOL>headerlines = [x.lstrip('<STR_LIT>' % commentchar) for x in headerlines[<NUM_LIT:3>:]]<EOL>metadatastart = headerlines.index('<STR_LIT>')<EOL>columnstart = headerlines.index('<STR_LIT>')<EOL>lcstart = headerlines.index('<STR_LIT>')<EOL>metadata = '<STR_LIT:U+0020>' .join(headerlines[metadatastart+<NUM_LIT:1>:columnstart-<NUM_LIT:1>])<EOL>columns = '<STR_LIT:U+0020>' .join(headerlines[columnstart+<NUM_LIT:1>:lcstart-<NUM_LIT:1>])<EOL>metadata = json.loads(metadata)<EOL>columns = json.loads(columns)<EOL>return metadata, columns, separator<EOL>

This parses the header of the LCC CSV V1 LC format.


<EOL>if '<STR_LIT>' in os.path.basename(lcfile):<EOL><INDENT>infd = gzip.open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>infd = open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>lctext = infd.read().decode()<EOL>infd.close()<EOL>lctextlines = lctext.split('<STR_LIT:\n>')<EOL>lcformat = lctextlines[<NUM_LIT:0>]<EOL>commentchar = lctextlines[<NUM_LIT:1>]<EOL>lcstart = lctextlines.index('<STR_LIT>' % commentchar)<EOL>headerlines = lctextlines[:lcstart+<NUM_LIT:1>]<EOL>lclines = lctextlines[lcstart+<NUM_LIT:1>:]<EOL>metadata, columns, separator = _parse_csv_header_lcc_csv_v1(headerlines)<EOL>objectid = metadata['<STR_LIT>']['<STR_LIT>']<EOL>objectinfo = {key:metadata[key]['<STR_LIT>'] for key in metadata}<EOL>colnames = []<EOL>colnum = []<EOL>coldtypes = []<EOL>for k in columns:<EOL><INDENT>coldef = columns[k]<EOL>colnames.append(k)<EOL>colnum.append(coldef['<STR_LIT>'])<EOL>coldtypes.append(coldef['<STR_LIT>'])<EOL><DEDENT>coldtypes = '<STR_LIT:U+002C>'.join(coldtypes)<EOL>recarr = np.genfromtxt(<EOL>lclines,<EOL>comments=commentchar,<EOL>delimiter=separator,<EOL>usecols=colnum,<EOL>autostrip=True,<EOL>names=colnames,<EOL>dtype=coldtypes<EOL>)<EOL>lcdict = {x:recarr[x] for x in colnames}<EOL>lcdict['<STR_LIT>'] = lcformat<EOL>lcdict['<STR_LIT>'] = objectid<EOL>lcdict['<STR_LIT>'] = objectinfo<EOL>lcdict['<STR_LIT>'] = colnames<EOL>lcdict['<STR_LIT>'] = columns<EOL>lcdict['<STR_LIT>'] = metadata<EOL>return lcdict<EOL>

This reads a CSV LC produced by an `LCC-Server
    <https://github.com/waqasbhatti/lcc-server>`_ instance.

    Parameters
    ----------

    lcfile : str
        The LC file to read.

    Returns
    -------

    dict
        Returns an lcdict that's readable by most astrobase functions for
        further processing.


metadata_lines = []<EOL>coldef_lines = []<EOL>if '<STR_LIT>' in lcdict and '<STR_LIT>' in lcdict['<STR_LIT>'].lower():<EOL><INDENT>metadata = lcdict['<STR_LIT>']<EOL>metakeys = lcdict['<STR_LIT>'].keys()<EOL>coldefs = lcdict['<STR_LIT>']<EOL>for mk in metakeys:<EOL><INDENT>metadata_lines.append(<EOL>'<STR_LIT>' % (<EOL>mk,<EOL>metadata[mk]['<STR_LIT>']<EOL>)<EOL>)<EOL><DEDENT>for ck in lcdict['<STR_LIT>']:<EOL><INDENT>coldef_lines.append('<STR_LIT>'<EOL>% (coldefs[ck]['<STR_LIT>'],<EOL>ck,<EOL>coldefs[ck]['<STR_LIT>'],<EOL>coldefs[ck]['<STR_LIT>']))<EOL><DEDENT>desc = LCC_CSVLC_DESCTEMPLATE.format(<EOL>objectid=lcdict['<STR_LIT>'],<EOL>metadata_desc='<STR_LIT:\n>'.join(metadata_lines),<EOL>metadata=pformat(lcdict['<STR_LIT>']),<EOL>columndefs='<STR_LIT:\n>'.join(coldef_lines)<EOL>)<EOL>print(desc)<EOL>if returndesc:<EOL><INDENT>return desc<EOL><DEDENT><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT>

This describes the LCC CSV format light curve file.

Parameters
----------

lcdict : dict
    The input lcdict to parse for column and metadata info.

returndesc : bool
    If True, returns the description string as an str instead of just
    printing it to stdout.

Returns
-------

str or None
    If returndesc is True, returns the description lines as a str, otherwise
    returns nothing.


<EOL>if '<STR_LIT>' in os.path.basename(lcfile):<EOL><INDENT>LOGINFO('<STR_LIT>' % lcfile)<EOL>infd = gzip.open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>LOGINFO('<STR_LIT>' % lcfile)<EOL>infd = open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>lcformat_check = infd.read(<NUM_LIT:12>).decode()<EOL>if '<STR_LIT>' in lcformat_check:<EOL><INDENT>infd.close()<EOL>return read_lcc_csvlc(lcfile)<EOL><DEDENT>else:<EOL><INDENT>infd.seek(<NUM_LIT:0>)<EOL><DEDENT>lctext = infd.read().decode()  <EOL>infd.close()<EOL>lcstart = lctext.index('<STR_LIT>')<EOL>lcheader = lctext[:lcstart+<NUM_LIT:12>]<EOL>lccolumns = lctext[lcstart+<NUM_LIT>:].split('<STR_LIT:\n>')<EOL>lccolumns = [x for x in lccolumns if len(x) > <NUM_LIT:0>]<EOL>lcdict = _parse_csv_header(lcheader)<EOL>lccolumns = [x.split('<STR_LIT:U+002C>') for x in lccolumns]<EOL>lccolumns = list(zip(*lccolumns))  <EOL>for colind, col in enumerate(lcdict['<STR_LIT>']):<EOL><INDENT>if (col.split('<STR_LIT:_>')[<NUM_LIT:0>] in LC_MAG_COLUMNS or<EOL>col.split('<STR_LIT:_>')[<NUM_LIT:0>] in LC_ERR_COLUMNS or<EOL>col.split('<STR_LIT:_>')[<NUM_LIT:0>] in LC_FLAG_COLUMNS):<EOL><INDENT>lcdict[col] = np.array([_smartcast(x,<EOL>COLUMNDEFS[col.split('<STR_LIT:_>')[<NUM_LIT:0>]][<NUM_LIT:2>])<EOL>for x in lccolumns[colind]])<EOL><DEDENT>elif col in COLUMNDEFS:<EOL><INDENT>lcdict[col] = np.array([_smartcast(x,COLUMNDEFS[col][<NUM_LIT:2>])<EOL>for x in lccolumns[colind]])<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>' % col)<EOL>continue<EOL><DEDENT><DEDENT>return lcdict<EOL>

This reads a HAT data server or LCC-Server produced CSV light curve
    into an lcdict.

    This will automatically figure out the format of the file
    provided. Currently, it can read:

    - legacy HAT data server CSV LCs (e.g. from
      https://hatsouth.org/planets/lightcurves.html) with an extension of the
      form: `.hatlc.csv.gz`.
    - all LCC-Server produced LCC-CSV-V1 LCs (e.g. from
      https://data.hatsurveys.org) with an extension of the form: `-csvlc.gz`.

    Parameters
    ----------

    lcfile : str
        The light curve file to read.

    Returns
    -------

    dict
        Returns an lcdict that can be read and used by many astrobase processing
        functions.


lc_time_diffs = [(lctimes[x] - lctimes[x-<NUM_LIT:1>]) for x in range(<NUM_LIT:1>,len(lctimes))]<EOL>lc_time_diffs = np.array(lc_time_diffs)<EOL>group_start_indices = np.where(lc_time_diffs > mingap)[<NUM_LIT:0>]<EOL>if len(group_start_indices) > <NUM_LIT:0>:<EOL><INDENT>group_indices = []<EOL>for i, gindex in enumerate(group_start_indices):<EOL><INDENT>if i == <NUM_LIT:0>:<EOL><INDENT>group_indices.append(slice(<NUM_LIT:0>,gindex+<NUM_LIT:1>))<EOL><DEDENT>else:<EOL><INDENT>group_indices.append(slice(group_start_indices[i-<NUM_LIT:1>]+<NUM_LIT:1>,gindex+<NUM_LIT:1>))<EOL><DEDENT><DEDENT>group_indices.append(slice(group_start_indices[-<NUM_LIT:1>]+<NUM_LIT:1>,len(lctimes)))<EOL><DEDENT>else:<EOL><INDENT>group_indices = [slice(<NUM_LIT:0>,len(lctimes))]<EOL><DEDENT>return len(group_indices), group_indices<EOL>

This finds the time gaps in the light curve, so we can figure out which
    times are for consecutive observations and which represent gaps
    between seasons.

    Parameters
    ----------

    lctimes : np.array
        This is the input array of times, assumed to be in some form of JD.

    mingap : float
        This defines how much the difference between consecutive measurements is
        allowed to be to consider them as parts of different timegroups. By
        default it is set to 4.0 days.

    Returns
    -------

    tuple
        A tuple of the form below is returned, containing the number of time
        groups found and Python slice objects for each group::

            (ngroups, [slice(start_ind_1, end_ind_1), ...])


<EOL>if '<STR_LIT>' in lcdict and len(lcdict['<STR_LIT>']) > <NUM_LIT:0>:<EOL><INDENT>if not quiet:<EOL><INDENT>LOGWARNING('<STR_LIT>')<EOL><DEDENT>return lcdict<EOL><DEDENT>if timecol in lcdict:<EOL><INDENT>times = lcdict[timecol]<EOL><DEDENT>elif '<STR_LIT>' in lcdict:<EOL><INDENT>times = lcdict['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>" %<EOL>lcdict['<STR_LIT>'])<EOL>return lcdict<EOL><DEDENT>ngroups, timegroups = find_lc_timegroups(np.array(times),<EOL>mingap=mingap)<EOL>if '<STR_LIT>' in lcdict:<EOL><INDENT>apertures = sorted(lcdict['<STR_LIT>'].keys())<EOL><DEDENT>elif '<STR_LIT>' in lcdict and '<STR_LIT>' in lcdict['<STR_LIT>']:<EOL><INDENT>apertures = sorted(lcdict['<STR_LIT>']['<STR_LIT>'].keys())<EOL><DEDENT>aimcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>armcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>aepcols = [('<STR_LIT>' % x)for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>atfcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>psimcols = [x for x in ['<STR_LIT>','<STR_LIT>','<STR_LIT>','<STR_LIT>'] if x in lcdict]<EOL>irmcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>iepcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>itfcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>if magcols == '<STR_LIT:all>':<EOL><INDENT>cols_to_normalize = (aimcols + armcols + aepcols + atfcols +<EOL>psimcols + irmcols + iepcols + itfcols)<EOL><DEDENT>elif magcols == '<STR_LIT>':<EOL><INDENT>cols_to_normalize = (irmcols + (['<STR_LIT>'] if '<STR_LIT>' in lcdict else []) +<EOL>irmcols)<EOL><DEDENT>elif magcols == '<STR_LIT>':<EOL><INDENT>cols_to_normalize = (aepcols + (['<STR_LIT>'] if '<STR_LIT>' in lcdict else []) +<EOL>iepcols)<EOL><DEDENT>elif magcols == '<STR_LIT>':<EOL><INDENT>cols_to_normalize = (atfcols + (['<STR_LIT>'] if '<STR_LIT>' in lcdict else []) +<EOL>itfcols)<EOL><DEDENT>elif magcols == '<STR_LIT>':<EOL><INDENT>cols_to_normalize = (aepcols + (['<STR_LIT>'] if '<STR_LIT>' in lcdict else []) +<EOL>iepcols + atfcols +<EOL>(['<STR_LIT>'] if '<STR_LIT>' in lcdict else []) +<EOL>itfcols)<EOL><DEDENT>else:<EOL><INDENT>cols_to_normalize = magcols.split('<STR_LIT:U+002C>')<EOL>cols_to_normalize = [x.strip() for x in cols_to_normalize]<EOL><DEDENT>colsnormalized = []<EOL>for col in cols_to_normalize:<EOL><INDENT>if col in lcdict:<EOL><INDENT>mags = lcdict[col]<EOL>mags = [(nan if x is None else x) for x in mags]<EOL>mags = np.array(mags)<EOL>colsnormalized.append(col)<EOL>finite_ind = np.isfinite(mags)<EOL>if any(finite_ind):<EOL><INDENT>global_mag_median = np.median(mags[finite_ind])<EOL>for tgind, tg in enumerate(timegroups):<EOL><INDENT>finite_ind = np.isfinite(mags[tg])<EOL>group_median = np.median((mags[tg])[finite_ind])<EOL>mags[tg] = mags[tg] - group_median<EOL>if debugmode:<EOL><INDENT>LOGDEBUG('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(col, tgind,<EOL>len(mags[tg]),<EOL>len(finite_ind),<EOL>group_median))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>' % col)<EOL>continue<EOL><DEDENT>if normto == '<STR_LIT>':<EOL><INDENT>mags = mags + global_mag_median<EOL><DEDENT>elif normto in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>if (normto in lcdict['<STR_LIT>'] and<EOL>lcdict['<STR_LIT>'][normto] is not None):<EOL><INDENT>mags = mags + lcdict['<STR_LIT>'][normto]<EOL><DEDENT>else:<EOL><INDENT>if not quiet:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>' % normto)<EOL><DEDENT>normto = '<STR_LIT>'<EOL>mags = mags + global_mag_median<EOL><DEDENT><DEDENT>lcdict[col] = mags<EOL><DEDENT>else:<EOL><INDENT>if not quiet:<EOL><INDENT>LOGWARNING('<STR_LIT>' % col)<EOL><DEDENT>continue<EOL><DEDENT><DEDENT>lcnormcols = ('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>') % (<EOL>repr(colsnormalized),<EOL>mingap,<EOL>normto<EOL>)<EOL>lcdict['<STR_LIT>'] = lcnormcols<EOL>return lcdict<EOL>

This normalizes magcols in `lcdict` using `timecol` to find timegroups.

    Parameters
    ----------

    lcdict : dict
        The input lcdict to process.

    timecol : str
        The key in the lcdict that is to be used to extract the time column.

    magcols : 'all' or list of str
        If this is 'all', all of the columns in the lcdict that are indicated to
        be magnitude measurement columns are normalized. If this is a list of
        str, must contain the keys of the lcdict specifying which magnitude
        columns will be normalized.

    mingap : float
        This defines how much the difference between consecutive measurements is
        allowed to be to consider them as parts of different timegroups. By
        default it is set to 4.0 days.

    normto : {'globalmedian', 'zero', 'jmag', 'hmag', 'kmag', 'bmag', 'vmag', 'sdssg', 'sdssr', 'sdssi'}
        This indicates which column will be the normalization target. If this is
        'globalmedian', the normalization will be to the global median of each
        LC column. If this is 'zero', will normalize to 0.0 for each LC
        column. Otherwise, will normalize to the value of one of the other keys
        in the lcdict['objectinfo'][magkey], meaning the normalization will be
        to some form of catalog magnitude.

    debugmode : bool
        If True, will indicate progress as time-groups are found and processed.

    quiet : bool
        If True, will not emit any messages when processing.

    Returns
    -------

    dict
        Returns the lcdict with the magnitude measurements normalized as
        specified. The normalization happens IN PLACE.


<EOL>if '<STR_LIT>' in lcdict and len(lcdict['<STR_LIT>']) > <NUM_LIT:0>:<EOL><INDENT>if not quiet:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>return lcdict<EOL><DEDENT>normkeycols = []<EOL>availablenormkeys = []<EOL>for key in normkeylist:<EOL><INDENT>if key in lcdict and lcdict[key] is not None:<EOL><INDENT>normkeycols.append(lcdict[key])<EOL>availablenormkeys.append(key)<EOL><DEDENT><DEDENT>normkeycols = list(zip(*normkeycols))<EOL>allkeys = [repr(x) for x in normkeycols]<EOL>allkeys = [a.replace('<STR_LIT:(>','<STR_LIT>').replace('<STR_LIT:)>','<STR_LIT>').replace("<STR_LIT:'>",'<STR_LIT>').replace('<STR_LIT:U+0020>','<STR_LIT>')<EOL>for a in allkeys]<EOL>allkeys = np.array(allkeys)<EOL>normkeys = np.unique(allkeys)<EOL>if '<STR_LIT>' in lcdict:<EOL><INDENT>apertures = sorted(lcdict['<STR_LIT>'].keys())<EOL><DEDENT>elif '<STR_LIT>' in lcdict and '<STR_LIT>' in lcdict['<STR_LIT>']:<EOL><INDENT>apertures = sorted(lcdict['<STR_LIT>']['<STR_LIT>'].keys())<EOL><DEDENT>aimcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>armcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>aepcols = [('<STR_LIT>' % x)for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>atfcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>psimcols = [x for x in ['<STR_LIT>','<STR_LIT>','<STR_LIT>','<STR_LIT>'] if x in lcdict]<EOL>irmcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>iepcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>itfcols = [('<STR_LIT>' % x) for x in apertures if ('<STR_LIT>' % x) in lcdict]<EOL>if magcols == '<STR_LIT:all>':<EOL><INDENT>cols_to_normalize = (aimcols + armcols + aepcols + atfcols +<EOL>psimcols + irmcols + iepcols + itfcols)<EOL><DEDENT>elif magcols == '<STR_LIT>':<EOL><INDENT>cols_to_normalize = (irmcols + (['<STR_LIT>'] if '<STR_LIT>' in lcdict else []) +<EOL>irmcols)<EOL><DEDENT>elif magcols == '<STR_LIT>':<EOL><INDENT>cols_to_normalize = (aepcols + (['<STR_LIT>'] if '<STR_LIT>' in lcdict else []) +<EOL>iepcols)<EOL><DEDENT>elif magcols == '<STR_LIT>':<EOL><INDENT>cols_to_normalize = (atfcols + (['<STR_LIT>'] if '<STR_LIT>' in lcdict else []) +<EOL>itfcols)<EOL><DEDENT>elif magcols == '<STR_LIT>':<EOL><INDENT>cols_to_normalize = (aepcols + (['<STR_LIT>'] if '<STR_LIT>' in lcdict else []) +<EOL>iepcols + atfcols +<EOL>(['<STR_LIT>'] if '<STR_LIT>' in lcdict else []) +<EOL>itfcols)<EOL><DEDENT>else:<EOL><INDENT>cols_to_normalize = magcols.split('<STR_LIT:U+002C>')<EOL>cols_to_normalize = [x.strip() for x in cols_to_normalize]<EOL><DEDENT>colsnormalized = []<EOL>for col in cols_to_normalize:<EOL><INDENT>if col in lcdict:<EOL><INDENT>thismags = lcdict[col]<EOL>for nkey in normkeys:<EOL><INDENT>thisind = allkeys == nkey<EOL>thismagsize = thismags[thisind].size<EOL>thismagfinite = np.where(np.isfinite(thismags[thisind]))[<NUM_LIT:0>].size<EOL>if thismagsize > <NUM_LIT:2> and thismagfinite > <NUM_LIT:2>:<EOL><INDENT>medmag = np.nanmedian(thismags[thisind])<EOL>lcdict[col][thisind] = lcdict[col][thisind] - medmag<EOL>if debugmode:<EOL><INDENT>LOGDEBUG('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(col, nkey, len(thismags[thisind]), medmag))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>lcdict[col][thisind] = np.nan<EOL><DEDENT><DEDENT>if normto in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>if (normto in lcdict['<STR_LIT>'] and<EOL>lcdict['<STR_LIT>'][normto] is not None):<EOL><INDENT>lcdict[col] = lcdict[col] + lcdict['<STR_LIT>'][normto]<EOL><DEDENT>else:<EOL><INDENT>if not quiet:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>' % normto)<EOL><DEDENT>normto = '<STR_LIT>'<EOL><DEDENT><DEDENT>colsnormalized.append(col)<EOL><DEDENT>else:<EOL><INDENT>if not quiet:<EOL><INDENT>LOGWARNING('<STR_LIT>' % col)<EOL><DEDENT>continue<EOL><DEDENT><DEDENT>lcinstnormcols = ('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>') % (repr(colsnormalized),<EOL>normto,<EOL>repr(availablenormkeys))<EOL>lcdict['<STR_LIT>'] = lcinstnormcols<EOL>return lcdict<EOL>

This is a function to normalize light curves across all instrument
    combinations present.

    Use this to normalize a light curve containing a variety of:

    - HAT station IDs ('stf')
    - camera IDs ('ccd')
    - filters ('flt')
    - observed field names ('fld')
    - HAT project IDs ('prj')
    - exposure times ('exp')

    Parameters
    ----------

    lcdict : dict
        The input lcdict to process.

    magcols : 'all' or list of str
        If this is 'all', all of the columns in the lcdict that are indicated to
        be magnitude measurement columns are normalized. If this is a list of
        str, must contain the keys of the lcdict specifying which magnitude
        columns will be normalized.

    normto : {'zero', 'jmag', 'hmag', 'kmag', 'bmag', 'vmag', 'sdssg', 'sdssr', 'sdssi'}
        This indicates which column will be the normalization target. If this is
        'zero', will normalize to 0.0 for each LC column. Otherwise, will
        normalize to the value of one of the other keys in the
        lcdict['objectinfo'][magkey], meaning the normalization will be to some
        form of catalog magnitude.

    normkeylist : list of str
        These are the column keys to use to form the normalization
        index. Measurements in the specified `magcols` with identical
        normalization index values will be considered as part of a single
        measurement 'era', and will be normalized to zero. Once all eras have
        been normalized this way, the final light curve will be re-normalized as
        specified in `normto`.

    debugmode : bool
        If True, will indicate progress as time-groups are found and processed.

    quiet : bool
        If True, will not emit any messages when processing.

    Returns
    -------

    dict
        Returns the lcdict with the magnitude measurements normalized as
        specified. The normalization happens IN PLACE.


<EOL>import signal<EOL>signal.signal(signal.SIGPIPE, signal.SIG_DFL)<EOL>import argparse<EOL>aparser = argparse.ArgumentParser(<EOL>description='<STR_LIT>'<EOL>)<EOL>aparser.add_argument(<EOL>'<STR_LIT>',<EOL>action='<STR_LIT:store>',<EOL>type=str,<EOL>help=("<STR_LIT>")<EOL>)<EOL>aparser.add_argument(<EOL>'<STR_LIT>',<EOL>action='<STR_LIT:store_true>',<EOL>default=False,<EOL>help=("<STR_LIT>")<EOL>)<EOL>args = aparser.parse_args()<EOL>filetoread = args.hatlcfile<EOL>if not os.path.exists(filetoread):<EOL><INDENT>LOGERROR("<STR_LIT>" % filetoread)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>filename = os.path.basename(filetoread)<EOL>if filename.endswith('<STR_LIT>') or filename.endswith('<STR_LIT>'):<EOL><INDENT>if args.describe:<EOL><INDENT>describe(read_csvlc(filename))<EOL>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>with gzip.open(filename,'<STR_LIT:rb>') as infd:<EOL><INDENT>for line in infd:<EOL><INDENT>print(line.decode(),end='<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT><DEDENT>elif filename.endswith('<STR_LIT>'):<EOL><INDENT>lcdict, msg = read_and_filter_sqlitecurve(filetoread)<EOL>describe(lcdict, offsetwith='<STR_LIT:#>')<EOL>if args.describe:<EOL><INDENT>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>apertures = sorted(lcdict['<STR_LIT>'].keys())<EOL>for aper in apertures:<EOL><INDENT>COLUMNDEFS.update({'<STR_LIT>' % (x, aper): COLUMNDEFS[x] for x in<EOL>LC_MAG_COLUMNS})<EOL>COLUMNDEFS.update({'<STR_LIT>' % (x, aper): COLUMNDEFS[x] for x in<EOL>LC_ERR_COLUMNS})<EOL>COLUMNDEFS.update({'<STR_LIT>' % (x, aper): COLUMNDEFS[x] for x in<EOL>LC_FLAG_COLUMNS})<EOL><DEDENT>formstr = '<STR_LIT:U+002C>'.join([COLUMNDEFS[x][<NUM_LIT:1>] for x in lcdict['<STR_LIT>']])<EOL>ndet = lcdict['<STR_LIT>']['<STR_LIT>']<EOL>for ind in range(ndet):<EOL><INDENT>line = [lcdict[x][ind] for x in lcdict['<STR_LIT>']]<EOL>formline = formstr % tuple(line)<EOL>print(formline)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>' % filetoread)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>

This is called when we're executed from the commandline.

The current usage from the command-line is described below::

    usage: hatlc [-h] [--describe] hatlcfile

    read a HAT LC of any format and output to stdout

    positional arguments:
      hatlcfile   path to the light curve you want to read and pipe to stdout

    optional arguments:
      -h, --help  show this help message and exit
      --describe  don't dump the columns, show only object info and LC metadata


<EOL>headerlines = header.split('<STR_LIT:\n>')<EOL>headerlines = [x.lstrip('<STR_LIT>') for x in headerlines]<EOL>metadatastart = headerlines.index('<STR_LIT>')<EOL>columnstart = headerlines.index('<STR_LIT>')<EOL>lcstart = headerlines.index('<STR_LIT>')<EOL>metadata = headerlines[metadatastart+<NUM_LIT:1>:columnstart-<NUM_LIT:1>]<EOL>columndefs = headerlines[columnstart+<NUM_LIT:1>:lcstart-<NUM_LIT:1>]<EOL>metainfo = [x.split('<STR_LIT:U+002C>') for x in metadata][:-<NUM_LIT:1>]<EOL>aperpixradius = metadata[-<NUM_LIT:1>]<EOL>objectid, kepid, ucac4id, kepmag = metainfo[<NUM_LIT:0>]<EOL>objectid, kepid, ucac4id, kepmag = (objectid.split('<STR_LIT>')[-<NUM_LIT:1>],<EOL>kepid.split('<STR_LIT>')[-<NUM_LIT:1>],<EOL>ucac4id.split('<STR_LIT>')[-<NUM_LIT:1>],<EOL>kepmag.split('<STR_LIT>')[-<NUM_LIT:1>])<EOL>kepmag = float(kepmag) if kepmag else None<EOL>ra, decl, ndet, k2campaign = metainfo[<NUM_LIT:1>]<EOL>ra, decl, ndet, k2campaign = (ra.split('<STR_LIT>')[-<NUM_LIT:1>],<EOL>decl.split('<STR_LIT>')[-<NUM_LIT:1>],<EOL>int(ndet.split('<STR_LIT>')[-<NUM_LIT:1>]),<EOL>int(k2campaign.split('<STR_LIT>')[-<NUM_LIT:1>]))<EOL>fovccd, fovchannel, fovmodule = metainfo[<NUM_LIT:2>]<EOL>fovccd, fovchannel, fovmodule = (int(fovccd.split('<STR_LIT>')[-<NUM_LIT:1>]),<EOL>int(fovchannel.split('<STR_LIT>')[-<NUM_LIT:1>]),<EOL>int(fovmodule.split('<STR_LIT>')[-<NUM_LIT:1>]))<EOL>try:<EOL><INDENT>qualflag, bjdoffset, napertures = metainfo[<NUM_LIT:3>]<EOL>qualflag, bjdoffset, napertures = (int(qualflag.split('<STR_LIT>')[-<NUM_LIT:1>]),<EOL>float(bjdoffset.split('<STR_LIT>')[-<NUM_LIT:1>]),<EOL>int(napertures.split('<STR_LIT>')[-<NUM_LIT:1>]))<EOL>kernelspec = None<EOL><DEDENT>except Exception as e:<EOL><INDENT>qualflag, bjdoffset, napertures, kernelspec = metainfo[<NUM_LIT:3>]<EOL>qualflag, bjdoffset, napertures, kernelspec = (<EOL>int(qualflag.split('<STR_LIT>')[-<NUM_LIT:1>]),<EOL>float(bjdoffset.split('<STR_LIT>')[-<NUM_LIT:1>]),<EOL>int(napertures.split('<STR_LIT>')[-<NUM_LIT:1>]),<EOL>str(kernelspec.split('<STR_LIT>')[-<NUM_LIT:1>])<EOL>)<EOL><DEDENT>aperpixradius = aperpixradius.split('<STR_LIT>')[-<NUM_LIT:1>].split('<STR_LIT:U+002C>')<EOL>aperpixradius = [float(x) for x in aperpixradius]<EOL>columns = [x.split('<STR_LIT>')[<NUM_LIT:1>] for x in columndefs]<EOL>metadict = {'<STR_LIT>':objectid,<EOL>'<STR_LIT>':{<EOL>'<STR_LIT>':objectid,<EOL>'<STR_LIT>':kepid,<EOL>'<STR_LIT>':ucac4id,<EOL>'<STR_LIT>':kepmag,<EOL>'<STR_LIT>':ra,<EOL>'<STR_LIT>':decl,<EOL>'<STR_LIT>':ndet,<EOL>'<STR_LIT>':k2campaign,<EOL>'<STR_LIT>':fovccd,<EOL>'<STR_LIT>':fovchannel,<EOL>'<STR_LIT>':fovmodule,<EOL>'<STR_LIT>':qualflag,<EOL>'<STR_LIT>':bjdoffset,<EOL>'<STR_LIT>':napertures,<EOL>'<STR_LIT>':kernelspec,<EOL>'<STR_LIT>':aperpixradius,<EOL>},<EOL>'<STR_LIT>':columns}<EOL>return metadict<EOL>

This parses a CSV header from a K2 CSV LC.

    Returns a dict that can be used to update an existing lcdict with the
    relevant metadata info needed to form a full LC.


<EOL>if '<STR_LIT>' in os.path.basename(lcfile):<EOL><INDENT>LOGINFO('<STR_LIT>' % lcfile)<EOL>infd = gzip.open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>LOGINFO('<STR_LIT>' % lcfile)<EOL>infd = open(lcfile,'<STR_LIT:rb>')<EOL><DEDENT>lctext = infd.read().decode()<EOL>infd.close()<EOL>lcstart = lctext.index('<STR_LIT>')<EOL>lcheader = lctext[:lcstart+<NUM_LIT:12>]<EOL>lccolumns = lctext[lcstart+<NUM_LIT>:].split('<STR_LIT:\n>')<EOL>lccolumns = [x.split('<STR_LIT:U+002C>') for x in lccolumns if len(x) > <NUM_LIT:0>]<EOL>lcdict = _parse_csv_header(lcheader)<EOL>lccolumns = list(zip(*lccolumns))<EOL>for colind, col in enumerate(lcdict['<STR_LIT>']):<EOL><INDENT>lcdict[col.lower()] = np.array([COLUMNDEFS[col][<NUM_LIT:2>](x)<EOL>for x in lccolumns[colind]])<EOL><DEDENT>lcdict['<STR_LIT>'] = [x.lower() for x in lcdict['<STR_LIT>']]<EOL>return lcdict<EOL>

This reads in a K2 lightcurve in CSV format. Transparently reads gzipped
files.

Parameters
----------

lcfile : str
    The light curve file to read.

Returns
-------

dict
    Returns an lcdict.


if mags is None:<EOL><INDENT>mags = np.full_like(times, <NUM_LIT:0.0>)<EOL><DEDENT>if errs is None:<EOL><INDENT>errs = np.full_like(times, <NUM_LIT:0.0>)<EOL><DEDENT>epoch = npr.random()*(times.max() - times.min()) + times.min()<EOL>period = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>depth = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>duration = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>ingduration = npr.random()*(<NUM_LIT:0.5>*duration - <NUM_LIT>*duration) + <NUM_LIT>*duration<EOL>if magsarefluxes and depth < <NUM_LIT:0.0>:<EOL><INDENT>depth = -depth<EOL><DEDENT>elif not magsarefluxes and depth > <NUM_LIT:0.0>:<EOL><INDENT>depth = -depth<EOL><DEDENT>modelmags, phase, ptimes, pmags, perrs = (<EOL>transits.trapezoid_transit_func([period, epoch, depth,<EOL>duration, ingduration],<EOL>times,<EOL>mags,<EOL>errs)<EOL>)<EOL>timeind = np.argsort(ptimes)<EOL>mtimes = ptimes[timeind]<EOL>mmags = modelmags[timeind]<EOL>merrs = perrs[timeind]<EOL>modeldict = {<EOL>'<STR_LIT>':'<STR_LIT>',<EOL>'<STR_LIT>':{x:np.asscalar(y) for x,y in zip(['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>'],<EOL>[period,<EOL>epoch,<EOL>depth,<EOL>duration,<EOL>ingduration])},<EOL>'<STR_LIT>':mtimes,<EOL>'<STR_LIT>':mmags,<EOL>'<STR_LIT>':merrs,<EOL>'<STR_LIT>':period,<EOL>'<STR_LIT>':depth<EOL>}<EOL>return modeldict<EOL>

This generates fake planet transit light curves.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'transitperiod', 'transitdepth', 'transitduration'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The variability epoch will be automatically chosen from a uniform
        distribution between `times.min()` and `times.max()`.

        The ingress duration will be automatically chosen from a uniform
        distribution ranging from 0.05 to 0.5 of the transitduration.

        The transitdepth will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'planet',
             'params': {'transitperiod': generated value of period,
                        'transitepoch': generated value of epoch,
                        'transitdepth': generated value of transit depth,
                        'transitduration': generated value of transit duration,
                        'ingressduration': generated value of transit ingress
                                           duration},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varperiod': the generated period of variability == 'transitperiod'
             'varamplitude': the generated amplitude of
                             variability == 'transitdepth'}


if mags is None:<EOL><INDENT>mags = np.full_like(times, <NUM_LIT:0.0>)<EOL><DEDENT>if errs is None:<EOL><INDENT>errs = np.full_like(times, <NUM_LIT:0.0>)<EOL><DEDENT>epoch = npr.random()*(times.max() - times.min()) + times.min()<EOL>period = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>pdepth = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>pduration = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>depthratio = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>secphase = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>if magsarefluxes and pdepth < <NUM_LIT:0.0>:<EOL><INDENT>pdepth = -pdepth<EOL><DEDENT>elif not magsarefluxes and pdepth > <NUM_LIT:0.0>:<EOL><INDENT>pdepth = -pdepth<EOL><DEDENT>modelmags, phase, ptimes, pmags, perrs = (<EOL>eclipses.invgauss_eclipses_func([period, epoch, pdepth,<EOL>pduration, depthratio, secphase],<EOL>times,<EOL>mags,<EOL>errs)<EOL>)<EOL>timeind = np.argsort(ptimes)<EOL>mtimes = ptimes[timeind]<EOL>mmags = modelmags[timeind]<EOL>merrs = perrs[timeind]<EOL>modeldict = {<EOL>'<STR_LIT>':'<STR_LIT>',<EOL>'<STR_LIT>':{x:np.asscalar(y) for x,y in zip(['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>'],<EOL>[period,<EOL>epoch,<EOL>pdepth,<EOL>pduration,<EOL>depthratio])},<EOL>'<STR_LIT>':mtimes,<EOL>'<STR_LIT>':mmags,<EOL>'<STR_LIT>':merrs,<EOL>'<STR_LIT>':period,<EOL>'<STR_LIT>':pdepth,<EOL>}<EOL>return modeldict<EOL>

This generates fake EB light curves.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'period', 'pdepth', 'pduration', 'depthratio', 'secphase'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The variability epoch will be automatically chosen from a uniform
        distribution between `times.min()` and `times.max()`.

        The `pdepth` will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'EB',
             'params': {'period': generated value of period,
                        'epoch': generated value of epoch,
                        'pdepth': generated value of priary eclipse depth,
                        'pduration': generated value of prim eclipse duration,
                        'depthratio': generated value of prim/sec eclipse
                                      depth ratio},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varperiod': the generated period of variability == 'period'
             'varamplitude': the generated amplitude of
                             variability == 'pdepth'}


if mags is None:<EOL><INDENT>mags = np.full_like(times, <NUM_LIT:0.0>)<EOL><DEDENT>if errs is None:<EOL><INDENT>errs = np.full_like(times, <NUM_LIT:0.0>)<EOL><DEDENT>nflares = npr.randint(paramdists['<STR_LIT>'][<NUM_LIT:0>],<EOL>high=paramdists['<STR_LIT>'][<NUM_LIT:1>])<EOL>flarepeaktimes = (<EOL>npr.random(<EOL>size=nflares<EOL>)*(times.max() - times.min()) + times.min()<EOL>)<EOL>params = {'<STR_LIT>':nflares}<EOL>for flareind, peaktime in zip(range(nflares), flarepeaktimes):<EOL><INDENT>amp = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>risestdev = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>decayconst = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>if magsarefluxes and amp < <NUM_LIT:0.0>:<EOL><INDENT>amp = -amp<EOL><DEDENT>elif not magsarefluxes and amp > <NUM_LIT:0.0>:<EOL><INDENT>amp = -amp<EOL><DEDENT>modelmags, ptimes, pmags, perrs = (<EOL>flares.flare_model(<EOL>[amp, peaktime, risestdev, decayconst],<EOL>times,<EOL>mags,<EOL>errs<EOL>)<EOL>)<EOL>mags = modelmags<EOL>params[flareind] = {'<STR_LIT>':peaktime,<EOL>'<STR_LIT>':amp,<EOL>'<STR_LIT>':risestdev,<EOL>'<STR_LIT>':decayconst}<EOL><DEDENT>modeldict = {<EOL>'<STR_LIT>':'<STR_LIT>',<EOL>'<STR_LIT>':params,<EOL>'<STR_LIT>':times,<EOL>'<STR_LIT>':mags,<EOL>'<STR_LIT>':errs,<EOL>'<STR_LIT>':None,<EOL>'<STR_LIT>':[params[x]['<STR_LIT>']<EOL>for x in range(params['<STR_LIT>'])],<EOL>}<EOL>return modeldict<EOL>

This generates fake flare light curves.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'amplitude', 'nflares', 'risestdev', 'decayconst'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The `flare_peak_time` for each flare will be generated automatically
        between `times.min()` and `times.max()` using a uniform distribution.

        The `amplitude` will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'flare',
             'params': {'amplitude': generated value of flare amplitudes,
                        'nflares': generated value of number of flares,
                        'risestdev': generated value of stdev of rise time,
                        'decayconst': generated value of decay constant,
                        'peaktime': generated value of flare peak time},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varamplitude': the generated amplitude of
                             variability == 'amplitude'}


if mags is None:<EOL><INDENT>mags = np.full_like(times, <NUM_LIT:0.0>)<EOL><DEDENT>if errs is None:<EOL><INDENT>errs = np.full_like(times, <NUM_LIT:0.0>)<EOL><DEDENT>epoch = npr.random()*(times.max() - times.min()) + times.min()<EOL>period = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>fourierorder = npr.randint(paramdists['<STR_LIT>'][<NUM_LIT:0>],<EOL>high=paramdists['<STR_LIT>'][<NUM_LIT:1>])<EOL>amplitude = paramdists['<STR_LIT>'].rvs(size=<NUM_LIT:1>)<EOL>if magsarefluxes and amplitude < <NUM_LIT:0.0>:<EOL><INDENT>amplitude = -amplitude<EOL><DEDENT>elif not magsarefluxes and amplitude > <NUM_LIT:0.0>:<EOL><INDENT>amplitude = -amplitude<EOL><DEDENT>ampcomps = [abs(amplitude/<NUM_LIT>)/float(x)<EOL>for x in range(<NUM_LIT:1>,fourierorder+<NUM_LIT:1>)]<EOL>phacomps = [paramdists['<STR_LIT>']*float(x)<EOL>for x in range(<NUM_LIT:1>,fourierorder+<NUM_LIT:1>)]<EOL>modelmags, phase, ptimes, pmags, perrs = sinusoidal.sine_series_sum(<EOL>[period, epoch, ampcomps, phacomps],<EOL>times,<EOL>mags,<EOL>errs<EOL>)<EOL>timeind = np.argsort(ptimes)<EOL>mtimes = ptimes[timeind]<EOL>mmags = modelmags[timeind]<EOL>merrs = perrs[timeind]<EOL>mphase = phase[timeind]<EOL>modeldict = {<EOL>'<STR_LIT>':'<STR_LIT>',<EOL>'<STR_LIT>':{x:y for x,y in zip(['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>'],<EOL>[period,<EOL>epoch,<EOL>amplitude,<EOL>fourierorder,<EOL>ampcomps,<EOL>phacomps])},<EOL>'<STR_LIT>':mtimes,<EOL>'<STR_LIT>':mmags,<EOL>'<STR_LIT>':merrs,<EOL>'<STR_LIT>':mphase,<EOL>'<STR_LIT>':period,<EOL>'<STR_LIT>':amplitude<EOL>}<EOL>return modeldict<EOL>

This generates fake sinusoidal light curves.

    This can be used for a variety of sinusoidal variables, e.g. RRab, RRc,
    Cepheids, Miras, etc. The functions that generate these model LCs below
    implement the following table::

        ## FOURIER PARAMS FOR SINUSOIDAL VARIABLES
        #
        # type        fourier           period [days]
        #             order    dist     limits         dist

        # RRab        8 to 10  uniform  0.45--0.80     uniform
        # RRc         3 to 6   uniform  0.10--0.40     uniform
        # HADS        7 to 9   uniform  0.04--0.10     uniform
        # rotator     2 to 5   uniform  0.80--120.0    uniform
        # LPV         2 to 5   uniform  250--500.0     uniform

    FIXME: for better model LCs, figure out how scipy.signal.butter works and
    low-pass filter using scipy.signal.filtfilt.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'period', 'fourierorder', 'amplitude', 'phioffset'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The variability epoch will be automatically chosen from a uniform
        distribution between `times.min()` and `times.max()`.

        The `amplitude` will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'sinusoidal',
             'params': {'period': generated value of period,
                        'epoch': generated value of epoch,
                        'amplitude': generated value of amplitude,
                        'fourierorder': generated value of fourier order,
                        'fourieramps': generated values of fourier amplitudes,
                        'fourierphases': generated values of fourier phases},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varperiod': the generated period of variability == 'period'
             'varamplitude': the generated amplitude of
                             variability == 'amplitude'}


modeldict = generate_sinusoidal_lightcurve(times,<EOL>mags=mags,<EOL>errs=errs,<EOL>paramdists=paramdists,<EOL>magsarefluxes=magsarefluxes)<EOL>modeldict['<STR_LIT>'] = '<STR_LIT>'<EOL>return modeldict<EOL>

This generates fake RRab light curves.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'period', 'fourierorder', 'amplitude'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The variability epoch will be automatically chosen from a uniform
        distribution between `times.min()` and `times.max()`.

        The `amplitude` will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'RRab',
             'params': {'period': generated value of period,
                        'epoch': generated value of epoch,
                        'amplitude': generated value of amplitude,
                        'fourierorder': generated value of fourier order,
                        'fourieramps': generated values of fourier amplitudes,
                        'fourierphases': generated values of fourier phases},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varperiod': the generated period of variability == 'period'
             'varamplitude': the generated amplitude of
                             variability == 'amplitude'}


This generates fake RRc light curves.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'period', 'fourierorder', 'amplitude'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The variability epoch will be automatically chosen from a uniform
        distribution between `times.min()` and `times.max()`.

        The `amplitude` will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'RRc',
             'params': {'period': generated value of period,
                        'epoch': generated value of epoch,
                        'amplitude': generated value of amplitude,
                        'fourierorder': generated value of fourier order,
                        'fourieramps': generated values of fourier amplitudes,
                        'fourierphases': generated values of fourier phases},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varperiod': the generated period of variability == 'period'
             'varamplitude': the generated amplitude of
                             variability == 'amplitude'}


This generates fake HADS light curves.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'period', 'fourierorder', 'amplitude'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The variability epoch will be automatically chosen from a uniform
        distribution between `times.min()` and `times.max()`.

        The `amplitude` will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'HADS',
             'params': {'period': generated value of period,
                        'epoch': generated value of epoch,
                        'amplitude': generated value of amplitude,
                        'fourierorder': generated value of fourier order,
                        'fourieramps': generated values of fourier amplitudes,
                        'fourierphases': generated values of fourier phases},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varperiod': the generated period of variability == 'period'
             'varamplitude': the generated amplitude of
                             variability == 'amplitude'}


This generates fake rotator light curves.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'period', 'fourierorder', 'amplitude'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The variability epoch will be automatically chosen from a uniform
        distribution between `times.min()` and `times.max()`.

        The `amplitude` will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'rotator',
             'params': {'period': generated value of period,
                        'epoch': generated value of epoch,
                        'amplitude': generated value of amplitude,
                        'fourierorder': generated value of fourier order,
                        'fourieramps': generated values of fourier amplitudes,
                        'fourierphases': generated values of fourier phases},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varperiod': the generated period of variability == 'period'
             'varamplitude': the generated amplitude of
                             variability == 'amplitude'}


This generates fake long-period-variable (LPV) light curves.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'period', 'fourierorder', 'amplitude'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The variability epoch will be automatically chosen from a uniform
        distribution between `times.min()` and `times.max()`.

        The `amplitude` will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'LPV',
             'params': {'period': generated value of period,
                        'epoch': generated value of epoch,
                        'amplitude': generated value of amplitude,
                        'fourierorder': generated value of fourier order,
                        'fourieramps': generated values of fourier amplitudes,
                        'fourierphases': generated values of fourier phases},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varperiod': the generated period of variability == 'period'
             'varamplitude': the generated amplitude of
                             variability == 'amplitude'}


This generates fake Cepheid light curves.

    Parameters
    ----------

    times : np.array
        This is an array of time values that will be used as the time base.

    mags,errs : np.array
        These arrays will have the model added to them. If either is
        None, `np.full_like(times, 0.0)` will used as a substitute and the model
        light curve will be centered around 0.0.

    paramdists : dict
        This is a dict containing parameter distributions to use for the
        model params, containing the following keys ::

            {'period', 'fourierorder', 'amplitude'}

        The values of these keys should all be 'frozen' scipy.stats distribution
        objects, e.g.:

        https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions
        The variability epoch will be automatically chosen from a uniform
        distribution between `times.min()` and `times.max()`.

        The `amplitude` will be flipped automatically as appropriate if
        `magsarefluxes=True`.

    magsarefluxes : bool
        If the generated time series is meant to be a flux time-series, set this
        to True to get the correct sign of variability amplitude.

    Returns
    -------

    dict
        A dict of the form below is returned::

            {'vartype': 'cepheid',
             'params': {'period': generated value of period,
                        'epoch': generated value of epoch,
                        'amplitude': generated value of amplitude,
                        'fourierorder': generated value of fourier order,
                        'fourieramps': generated values of fourier amplitudes,
                        'fourierphases': generated values of fourier phases},
             'times': the model times,
             'mags': the model mags,
             'errs': the model errs,
             'varperiod': the generated period of variability == 'period'
             'varamplitude': the generated amplitude of
                             variability == 'amplitude'}