<EOL>if (isinstance(gridsearch_results, str) and<EOL>os.path.exists(gridsearch_results)):<EOL><INDENT>with open(gridsearch_results,'<STR_LIT:rb>') as infd:<EOL><INDENT>gridresults = pickle.load(infd)<EOL><DEDENT><DEDENT>elif isinstance(gridsearch_results, dict):<EOL><INDENT>gridresults = gridsearch_results<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>')<EOL>return None<EOL><DEDENT>plotres = {'<STR_LIT>':gridresults['<STR_LIT>']}<EOL>recgrid = gridresults['<STR_LIT>']<EOL>simbasedir = gridresults['<STR_LIT>']<EOL>for magcol in gridresults['<STR_LIT>']:<EOL><INDENT>plotres[magcol] = {'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':[],<EOL>'<STR_LIT>':gridresults['<STR_LIT>']}<EOL>for magbinind, magbinmedian in enumerate(gridresults['<STR_LIT>']):<EOL><INDENT>LOGINFO('<STR_LIT>' %<EOL>(magcol, magbinmedian))<EOL>stet_mcc = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[::(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)]<EOL>stet_precision = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[::(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)]<EOL>stet_recall = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[::(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)]<EOL>stet_missed_inveta_found = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[::(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)]<EOL>stet_missed_iqr_found = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[::(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)]<EOL>inveta_mcc = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>::gridresults['<STR_LIT>'].size<EOL>]<EOL>inveta_precision = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>::gridresults['<STR_LIT>'].size<EOL>]<EOL>inveta_recall = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>::gridresults['<STR_LIT>'].size<EOL>]<EOL>inveta_missed_stet_found = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>::gridresults['<STR_LIT>'].size<EOL>]<EOL>inveta_missed_iqr_found = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>::gridresults['<STR_LIT>'].size<EOL>]<EOL>iqr_mcc = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>:gridresults['<STR_LIT>'].size<EOL>]<EOL>iqr_precision = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>:gridresults['<STR_LIT>'].size<EOL>]<EOL>iqr_recall = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>:gridresults['<STR_LIT>'].size<EOL>]<EOL>iqr_missed_stet_found = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>:gridresults['<STR_LIT>'].size<EOL>]<EOL>iqr_missed_inveta_found = np.array(<EOL>[x[magcol]['<STR_LIT>']<EOL>for x in recgrid[magbinind]]<EOL>)[:(gridresults['<STR_LIT>'].size *<EOL>gridresults['<STR_LIT>'].size)][<EOL>:gridresults['<STR_LIT>'].size<EOL>]<EOL>fig = plt.figure(figsize=(<NUM_LIT>*<NUM_LIT:5>, <NUM_LIT>*<NUM_LIT:3>))<EOL>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:1>)<EOL>if np.any(np.isfinite(stet_mcc)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>stet_mcc)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:2>)<EOL>if np.any(np.isfinite(stet_precision)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>stet_precision)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:3>)<EOL>if np.any(np.isfinite(stet_recall)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>stet_recall)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:4>)<EOL>if np.any(np.isfinite(stet_missed_inveta_found)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>stet_missed_inveta_found)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:5>)<EOL>if np.any(np.isfinite(stet_missed_iqr_found)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>stet_missed_iqr_found)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:6>)<EOL>if np.any(np.isfinite(inveta_mcc)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>inveta_mcc)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:7>)<EOL>if np.any(np.isfinite(inveta_precision)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>inveta_precision)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:8>)<EOL>if np.any(np.isfinite(inveta_recall)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>inveta_recall)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:9>)<EOL>if np.any(np.isfinite(inveta_missed_stet_found)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>inveta_missed_stet_found)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:10>)<EOL>if np.any(np.isfinite(inveta_missed_iqr_found)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>inveta_missed_iqr_found)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:11>)<EOL>if np.any(np.isfinite(iqr_mcc)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>iqr_mcc)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:12>)<EOL>if np.any(np.isfinite(iqr_precision)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>iqr_precision)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT>)<EOL>if np.any(np.isfinite(iqr_recall)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>iqr_recall)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT>)<EOL>if np.any(np.isfinite(iqr_missed_stet_found)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>iqr_missed_stet_found)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplot(<NUM_LIT:3>,<NUM_LIT:5>,<NUM_LIT:15>)<EOL>if np.any(np.isfinite(iqr_missed_inveta_found)):<EOL><INDENT>plt.plot(gridresults['<STR_LIT>'],<EOL>iqr_missed_inveta_found)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>plt.title('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>plt.text(<NUM_LIT:0.5>,<NUM_LIT:0.5>,<EOL>'<STR_LIT>'<EOL>'<STR_LIT>',<EOL>transform=plt.gca().transAxes,<EOL>horizontalalignment='<STR_LIT>',<EOL>verticalalignment='<STR_LIT>')<EOL>plt.xticks([])<EOL>plt.yticks([])<EOL><DEDENT>plt.subplots_adjust(hspace=<NUM_LIT>,wspace=<NUM_LIT>)<EOL>plt.suptitle('<STR_LIT>' % (magcol, magbinmedian))<EOL>plotdir = os.path.join(gridresults['<STR_LIT>'],<EOL>'<STR_LIT>')<EOL>if not os.path.exists(plotdir):<EOL><INDENT>os.mkdir(plotdir)<EOL><DEDENT>gridplotf = os.path.join(<EOL>plotdir,<EOL>'<STR_LIT>' %<EOL>(magcol, magbinmedian)<EOL>)<EOL>plt.savefig(gridplotf,dpi=<NUM_LIT:100>,bbox_inches='<STR_LIT>')<EOL>plt.close('<STR_LIT:all>')<EOL>stet_mcc_maxind = np.where(stet_mcc == np.max(stet_mcc))<EOL>stet_precision_maxind = np.where(<EOL>stet_precision == np.max(stet_precision)<EOL>)<EOL>stet_recall_maxind = np.where(stet_recall == np.max(stet_recall))<EOL>best_stet_mcc = stet_mcc[stet_mcc_maxind]<EOL>best_stet_precision = stet_mcc[stet_precision_maxind]<EOL>best_stet_recall = stet_mcc[stet_recall_maxind]<EOL>stet_with_best_mcc = gridresults['<STR_LIT>'][stet_mcc_maxind]<EOL>stet_with_best_precision = gridresults['<STR_LIT>'][<EOL>stet_precision_maxind<EOL>]<EOL>stet_with_best_recall = (<EOL>gridresults['<STR_LIT>'][stet_recall_maxind]<EOL>)<EOL>inveta_mcc_maxind = np.where(inveta_mcc == np.max(inveta_mcc))<EOL>inveta_precision_maxind = np.where(<EOL>inveta_precision == np.max(inveta_precision)<EOL>)<EOL>inveta_recall_maxind = (<EOL>np.where(inveta_recall == np.max(inveta_recall))<EOL>)<EOL>best_inveta_mcc = inveta_mcc[inveta_mcc_maxind]<EOL>best_inveta_precision = inveta_mcc[inveta_precision_maxind]<EOL>best_inveta_recall = inveta_mcc[inveta_recall_maxind]<EOL>inveta_with_best_mcc = gridresults['<STR_LIT>'][inveta_mcc_maxind]<EOL>inveta_with_best_precision = gridresults['<STR_LIT>'][<EOL>inveta_precision_maxind<EOL>]<EOL>inveta_with_best_recall = gridresults['<STR_LIT>'][<EOL>inveta_recall_maxind<EOL>]<EOL>iqr_mcc_maxind = np.where(iqr_mcc == np.max(iqr_mcc))<EOL>iqr_precision_maxind = np.where(<EOL>iqr_precision == np.max(iqr_precision)<EOL>)<EOL>iqr_recall_maxind = (<EOL>np.where(iqr_recall == np.max(iqr_recall))<EOL>)<EOL>best_iqr_mcc = iqr_mcc[iqr_mcc_maxind]<EOL>best_iqr_precision = iqr_mcc[iqr_precision_maxind]<EOL>best_iqr_recall = iqr_mcc[iqr_recall_maxind]<EOL>iqr_with_best_mcc = gridresults['<STR_LIT>'][iqr_mcc_maxind]<EOL>iqr_with_best_precision = gridresults['<STR_LIT>'][<EOL>iqr_precision_maxind<EOL>]<EOL>iqr_with_best_recall = gridresults['<STR_LIT>'][<EOL>iqr_recall_maxind<EOL>]<EOL>plotres[magcol][magbinmedian] = {<EOL>'<STR_LIT>':gridresults['<STR_LIT>'],<EOL>'<STR_LIT>':stet_mcc,<EOL>'<STR_LIT>':stet_precision,<EOL>'<STR_LIT>':stet_recall,<EOL>'<STR_LIT>':stet_missed_inveta_found,<EOL>'<STR_LIT>':best_stet_mcc,<EOL>'<STR_LIT>':stet_with_best_mcc,<EOL>'<STR_LIT>':best_stet_precision,<EOL>'<STR_LIT>':stet_with_best_precision,<EOL>'<STR_LIT>':best_stet_recall,<EOL>'<STR_LIT>':stet_with_best_recall,<EOL>'<STR_LIT>':gridresults['<STR_LIT>'],<EOL>'<STR_LIT>':inveta_mcc,<EOL>'<STR_LIT>':inveta_precision,<EOL>'<STR_LIT>':inveta_recall,<EOL>'<STR_LIT>':inveta_missed_stet_found,<EOL>'<STR_LIT>':best_inveta_mcc,<EOL>'<STR_LIT>':inveta_with_best_mcc,<EOL>'<STR_LIT>':best_inveta_precision,<EOL>'<STR_LIT>':inveta_with_best_precision,<EOL>'<STR_LIT>':best_inveta_recall,<EOL>'<STR_LIT>':inveta_with_best_recall,<EOL>'<STR_LIT>':gridresults['<STR_LIT>'],<EOL>'<STR_LIT>':iqr_mcc,<EOL>'<STR_LIT>':iqr_precision,<EOL>'<STR_LIT>':iqr_recall,<EOL>'<STR_LIT>':iqr_missed_stet_found,<EOL>'<STR_LIT>':best_iqr_mcc,<EOL>'<STR_LIT>':iqr_with_best_mcc,<EOL>'<STR_LIT>':best_iqr_precision,<EOL>'<STR_LIT>':iqr_with_best_precision,<EOL>'<STR_LIT>':best_iqr_recall,<EOL>'<STR_LIT>':iqr_with_best_recall,<EOL>'<STR_LIT>':gridplotf<EOL>}<EOL>if stet_with_best_mcc.size > <NUM_LIT:1>:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(stet_with_best_mcc[<NUM_LIT:0>])<EOL><DEDENT>elif stet_with_best_mcc.size > <NUM_LIT:0>:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(stet_with_best_mcc[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(np.nan)<EOL><DEDENT>if inveta_with_best_mcc.size > <NUM_LIT:1>:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(inveta_with_best_mcc[<NUM_LIT:0>])<EOL><DEDENT>elif inveta_with_best_mcc.size > <NUM_LIT:0>:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(inveta_with_best_mcc[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(np.nan)<EOL><DEDENT>if iqr_with_best_mcc.size > <NUM_LIT:1>:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(iqr_with_best_mcc[<NUM_LIT:0>])<EOL><DEDENT>elif iqr_with_best_mcc.size > <NUM_LIT:0>:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(iqr_with_best_mcc[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>plotres[magcol]['<STR_LIT>'].append(np.nan)<EOL><DEDENT><DEDENT><DEDENT>plotrespicklef = os.path.join(simbasedir,<EOL>'<STR_LIT>')<EOL>with open(plotrespicklef, '<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(plotres, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>for magcol in gridresults['<STR_LIT>']:<EOL><INDENT>LOGINFO('<STR_LIT>' % magcol)<EOL>LOGINFO('<STR_LIT>')<EOL>for magbin, inveta, stet, iqr in zip(<EOL>plotres[magcol]['<STR_LIT>'],<EOL>plotres[magcol]['<STR_LIT>'],<EOL>plotres[magcol]['<STR_LIT>'],<EOL>plotres[magcol]['<STR_LIT>']):<EOL><INDENT>LOGINFO('<STR_LIT>' % (magbin,<EOL>inveta,<EOL>stet,<EOL>iqr))<EOL><DEDENT><DEDENT>return plotres<EOL>

This plots the gridsearch results from `variable_index_gridsearch_magbin`.

    Parameters
    ----------

    gridsearch_results : dict
        This is the dict produced by `variable_index_gridsearch_magbin` above.

    Returns
    -------

    dict
        The returned dict contains filenames of the recovery rate plots made for
        each variability index. These include plots of the precision, recall,
        and Matthews Correlation Coefficient over each magbin and a heatmap of
        these values over the grid points of the variability index stdev values
        arrays used.


pfpkl, simbasedir, period_tolerance = task<EOL>try:<EOL><INDENT>return periodicvar_recovery(pfpkl,<EOL>simbasedir,<EOL>period_tolerance=period_tolerance)<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' % repr(task))<EOL>return None<EOL><DEDENT>

This is a parallel worker for running period-recovery.

    Parameters
    ----------

    task : tuple
        This is used to pass args to the `periodicvar_recovery` function::

            task[0] = period-finding result pickle to work on
            task[1] = simbasedir
            task[2] = period_tolerance

    Returns
    -------

    dict
        This is the dict produced by the `periodicvar_recovery` function for the
        input period-finding result pickle.


<EOL>with open(os.path.join(simbasedir, '<STR_LIT>'),'<STR_LIT:rb>') as infd:<EOL><INDENT>siminfo = pickle.load(infd)<EOL><DEDENT>lcfpaths = siminfo['<STR_LIT>']<EOL>pfdir = os.path.join(simbasedir,'<STR_LIT>')<EOL>timecols = siminfo['<STR_LIT>']<EOL>magcols = siminfo['<STR_LIT>']<EOL>errcols = siminfo['<STR_LIT>']<EOL>fakelc_formatkey = '<STR_LIT>' % siminfo['<STR_LIT>']<EOL>lcproc.register_lcformat(<EOL>fakelc_formatkey,<EOL>'<STR_LIT>',<EOL>timecols,<EOL>magcols,<EOL>errcols,<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>magsarefluxes=siminfo['<STR_LIT>']<EOL>)<EOL>if liststartindex:<EOL><INDENT>lcfpaths = lcfpaths[liststartindex:]<EOL><DEDENT>if listmaxobjects:<EOL><INDENT>lcfpaths = lcfpaths[:listmaxobjects]<EOL><DEDENT>pfinfo = periodsearch.parallel_pf(lcfpaths,<EOL>pfdir,<EOL>lcformat=fakelc_formatkey,<EOL>pfmethods=pfmethods,<EOL>pfkwargs=pfkwargs,<EOL>getblssnr=getblssnr,<EOL>sigclip=sigclip,<EOL>nperiodworkers=nperiodworkers,<EOL>ncontrolworkers=ncontrolworkers)<EOL>with open(os.path.join(simbasedir,<EOL>'<STR_LIT>'),'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(pfinfo, outfd, pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>return os.path.join(simbasedir,'<STR_LIT>')<EOL>

This runs periodfinding using several period-finders on a collection of
    fake LCs.

    As a rough benchmark, 25000 fake LCs with 10000--50000 points per LC take
    about 26 days in total to run on an invocation of this function using
    GLS+PDM+BLS and 10 periodworkers and 4 controlworkers (so all 40 'cores') on
    a 2 x Xeon E5-2660v3 machine.

    Parameters
    ----------

    pfmethods : sequence of str
        This is used to specify which periodfinders to run. These must be in the
        `lcproc.periodsearch.PFMETHODS` dict.

    pfkwargs : sequence of dict
        This is used to provide optional kwargs to the period-finders.

    getblssnr : bool
        If this is True, will run BLS SNR calculations for each object and
        magcol. This takes a while to run, so it's disabled (False) by default.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    nperiodworkers : int
        This is the number of parallel period-finding worker processes to use.

    ncontrolworkers : int
        This is the number of parallel period-finding control workers to
        use. Each control worker will launch `nperiodworkers` worker processes.

    liststartindex : int
        The starting index of processing. This refers to the filename list
        generated by running `glob.glob` on the fake LCs in `simbasedir`.

    maxobjects : int
        The maximum number of objects to process in this run. Use this with
        `liststartindex` to effectively distribute working on a large list of
        input light curves over several sessions or machines.

    Returns
    -------

    str
        The path to the output summary pickle produced by
        `lcproc.periodsearch.parallel_pf`


<EOL>outdir = os.path.join(simbasedir,'<STR_LIT>')<EOL>if not os.path.exists(outdir):<EOL><INDENT>os.mkdir(outdir)<EOL><DEDENT>with open(os.path.join(simbasedir, '<STR_LIT>'),'<STR_LIT:rb>') as infd:<EOL><INDENT>siminfo = pickle.load(infd)<EOL><DEDENT>timecols = siminfo['<STR_LIT>']<EOL>magcols = siminfo['<STR_LIT>']<EOL>errcols = siminfo['<STR_LIT>']<EOL>magbinmedians = siminfo['<STR_LIT>'][magcols[<NUM_LIT:0>]]['<STR_LIT>']<EOL>stetson_grid = np.linspace(stetson_stdev_range[<NUM_LIT:0>],<EOL>stetson_stdev_range[<NUM_LIT:1>],<EOL>num=ngridpoints)<EOL>inveta_grid = np.linspace(inveta_stdev_range[<NUM_LIT:0>],<EOL>inveta_stdev_range[<NUM_LIT:1>],<EOL>num=ngridpoints)<EOL>iqr_grid = np.linspace(iqr_stdev_range[<NUM_LIT:0>],<EOL>iqr_stdev_range[<NUM_LIT:1>],<EOL>num=ngridpoints)<EOL>stet_inveta_iqr_grid = []<EOL>for stet in stetson_grid:<EOL><INDENT>for inveta in inveta_grid:<EOL><INDENT>for iqr in iqr_grid:<EOL><INDENT>grid_point = [stet, inveta, iqr]<EOL>stet_inveta_iqr_grid.append(grid_point)<EOL><DEDENT><DEDENT><DEDENT>grid_results = {'<STR_LIT>':stetson_grid,<EOL>'<STR_LIT>':inveta_grid,<EOL>'<STR_LIT>':iqr_grid,<EOL>'<STR_LIT>':stet_inveta_iqr_grid,<EOL>'<STR_LIT>':magbinmedians,<EOL>'<STR_LIT>':timecols,<EOL>'<STR_LIT>':magcols,<EOL>'<STR_LIT>':errcols,<EOL>'<STR_LIT>':os.path.abspath(simbasedir),<EOL>'<STR_LIT>':[]}<EOL>pool = mp.Pool(ngridworkers)<EOL>for magbinmedian in magbinmedians:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>' % magbinmedian)<EOL>tasks = [(simbasedir, gp, magbinmedian) for gp in stet_inveta_iqr_grid]<EOL>thisbin_results = pool.map(magbin_varind_gridsearch_worker, tasks)<EOL>grid_results['<STR_LIT>'].append(thisbin_results)<EOL><DEDENT>pool.close()<EOL>pool.join()<EOL>LOGINFO('<STR_LIT>')<EOL>with open(os.path.join(simbasedir,<EOL>'<STR_LIT>'),'<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(grid_results,outfd,pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>return grid_results<EOL>

This runs a variable index grid search per magbin.

    For each magbin, this does a grid search using the stetson and inveta ranges
    provided and tries to optimize the Matthews Correlation Coefficient (best
    value is +1.0), indicating the best possible separation of variables
    vs. nonvariables. The thresholds on these two variable indexes that produce
    the largest coeff for the collection of fake LCs will probably be the ones
    that work best for actual variable classification on the real LCs.

    https://en.wikipedia.org/wiki/Matthews_correlation_coefficient

    For each grid-point, calculates the true positives, false positives, true
    negatives, false negatives. Then gets the precision and recall, confusion
    matrix, and the ROC curve for variable vs. nonvariable.

    Once we've identified the best thresholds to use, we can then calculate
    variable object numbers:

    - as a function of magnitude
    - as a function of period
    - as a function of number of detections
    - as a function of amplitude of variability

    Writes everything back to `simbasedir/fakevar-recovery.pkl`. Use the
    plotting function below to make plots for the results.

    Parameters
    ----------

    simbasedir : str
        The directory where the fake LCs are located.

    stetson_stdev_range : sequence of 2 floats
        The min and max values of the Stetson J variability index to generate a
        grid over these to test for the values of this index that produce the
        'best' recovery rate for the injected variable stars.

    inveta_stdev_range : sequence of 2 floats
        The min and max values of the 1/eta variability index to generate a
        grid over these to test for the values of this index that produce the
        'best' recovery rate for the injected variable stars.

    iqr_stdev_range : sequence of 2 floats
        The min and max values of the IQR variability index to generate a
        grid over these to test for the values of this index that produce the
        'best' recovery rate for the injected variable stars.

    ngridpoints : int
        The number of grid points for each variability index grid. Remember that
        this function will be searching in 3D and will require lots of time to
        run if ngridpoints is too large.

        For the default number of grid points and 25000 simulated light curves,
        this takes about 3 days to run on a 40 (effective) core machine with 2 x
        Xeon E5-2650v3 CPUs.

    ngridworkers : int or None
        The number of parallel grid search workers that will be launched.

    Returns
    -------

    dict
        The returned dict contains a list of recovery stats for each magbin and
        each grid point in the variability index grids that were used. This dict
        can be passed to the plotting function below to plot the results.


bucket_item = url.replace('<STR_LIT>','<STR_LIT>')<EOL>bucket_item = bucket_item.split('<STR_LIT:/>')<EOL>bucket = bucket_item[<NUM_LIT:0>]<EOL>filekey = '<STR_LIT:/>'.join(bucket_item[<NUM_LIT:1>:])<EOL>return s3_get_file(bucket,<EOL>filekey,<EOL>bucket_item[-<NUM_LIT:1>],<EOL>altexts=altexts,<EOL>client=client,<EOL>raiseonfail=raiseonfail)<EOL>

This gets a file from an S3 bucket based on its s3:// URL.

    Parameters
    ----------

    url : str
        S3 URL to download. This should begin with 's3://'.

    altexts : None or list of str
        If not None, this is a list of alternate extensions to try for the file
        other than the one provided in `filename`. For example, to get anything
        that's an .sqlite where .sqlite.gz is expected, use altexts=[''] to
        strip the .gz.

    client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its operations. Alternatively, pass in an existing `boto3.Client`
        instance to re-use it here.

    raiseonfail : bool
        If True, will re-raise whatever Exception caused the operation to fail
        and break out immediately.

    Returns
    -------

    str
        Path to the downloaded filename or None if the download was
        unsuccessful. The file will be downloaded into the current working
        directory and will have a filename == basename of the file on S3.


def s3_delete_file(bucket, filename, client=None, raiseonfail=False):

if not client:<EOL><INDENT>client = boto3.client('<STR_LIT>')<EOL><DEDENT>try:<EOL><INDENT>resp = client.delete_object(Bucket=bucket, Key=filename)<EOL>if not resp:<EOL><INDENT>LOGERROR('<STR_LIT>' % (filename,<EOL>bucket))<EOL><DEDENT>else:<EOL><INDENT>return resp['<STR_LIT>']<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' % (filename,<EOL>bucket))<EOL>if raiseonfail:<EOL><INDENT>raise<EOL><DEDENT>return None<EOL><DEDENT>

This deletes a file from S3.

    Parameters
    ----------

    bucket : str
        The AWS S3 bucket to delete the file from.

    filename : str
        The full file name of the file to delete, including any prefixes.

    client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its operations. Alternatively, pass in an existing `boto3.Client`
        instance to re-use it here.

    raiseonfail : bool
        If True, will re-raise whatever Exception caused the operation to fail
        and break out immediately.

    Returns
    -------

    str or None
        If the file was successfully deleted, will return the delete-marker
        (https://docs.aws.amazon.com/AmazonS3/latest/dev/DeleteMarker.html). If
        it wasn't, returns None


c = paramiko.client.SSHClient()<EOL>c.load_system_host_keys()<EOL>c.set_missing_host_key_policy(paramiko.client.AutoAddPolicy)<EOL>privatekey = paramiko.RSAKey.from_private_key_file(keypem_file)<EOL>try:<EOL><INDENT>c.connect(ip_address,<EOL>pkey=privatekey,<EOL>username='<STR_LIT>')<EOL>return c<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(ip_address, keypem_file, username))<EOL>if raiseonfail:<EOL><INDENT>raise<EOL><DEDENT>return None<EOL><DEDENT>

This opens an SSH connection to the EC2 instance at `ip_address`.

    Parameters
    ----------

    ip_address : str
        IP address of the AWS EC2 instance to connect to.

    keypem_file : str
        The path to the keypair PEM file generated by AWS to allow SSH
        connections.

    username : str
        The username to use to login to the EC2 instance.

    raiseonfail : bool
        If True, will re-raise whatever Exception caused the operation to fail
        and break out immediately.

    Returns
    -------

    paramiko.SSHClient
        This has all the usual `paramiko` functionality:

        - Use `SSHClient.exec_command(command, environment=None)` to exec a
          shell command.

        - Use `SSHClient.open_sftp()` to get a `SFTPClient` for the server. Then
          call SFTPClient.get() and .put() to copy files from and to the server.


fleetconfig = copy.deepcopy(SPOT_FLEET_CONFIG)<EOL>fleetconfig['<STR_LIT>'] = spot_fleet_iam_role<EOL>fleetconfig['<STR_LIT>'] = allocation_strategy<EOL>fleetconfig['<STR_LIT>'] = target_capacity<EOL>fleetconfig['<STR_LIT>'] = str(spot_price)<EOL>fleetconfig['<STR_LIT>'] = (<EOL>datetime.utcnow() + timedelta(days=expires_days)<EOL>).strftime(<EOL>'<STR_LIT>'<EOL>)<EOL>if (isinstance(instance_user_data, str) and<EOL>os.path.exists(instance_user_data)):<EOL><INDENT>with open(instance_user_data,'<STR_LIT:rb>') as infd:<EOL><INDENT>udata = base64.b64encode(infd.read()).decode()<EOL><DEDENT><DEDENT>elif isinstance(instance_user_data, str):<EOL><INDENT>udata = base64.b64encode(instance_user_data.encode()).decode()<EOL><DEDENT>else:<EOL><INDENT>udata = (<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % datetime.utcnow().isoformat()<EOL>)<EOL>udata = base64.b64encode(udata.encode()).decode()<EOL><DEDENT>for ind, itype in enumerate(instance_types):<EOL><INDENT>thisinstance = SPOT_PERINSTANCE_CONFIG.copy()<EOL>thisinstance['<STR_LIT>'] = itype<EOL>thisinstance['<STR_LIT>'] = instance_ami<EOL>thisinstance['<STR_LIT>'] = subnet_id<EOL>thisinstance['<STR_LIT>'] = keypair_name<EOL>thisinstance['<STR_LIT>']['<STR_LIT>'] = iam_instance_profile_arn<EOL>thisinstance['<STR_LIT>'][<NUM_LIT:0>] = {'<STR_LIT>':security_groupid}<EOL>thisinstance['<STR_LIT>'] = udata<EOL>thisinstance['<STR_LIT>'] = instance_ebs_optimized<EOL>if isinstance(instance_weights, list):<EOL><INDENT>thisinstance['<STR_LIT>'] = instance_weights[ind]<EOL><DEDENT>fleetconfig['<STR_LIT>'].append(thisinstance)<EOL><DEDENT>if not client:<EOL><INDENT>client = boto3.client('<STR_LIT>')<EOL><DEDENT>try:<EOL><INDENT>resp = client.request_spot_fleet(<EOL>SpotFleetRequestConfig=fleetconfig,<EOL>)<EOL>if not resp:<EOL><INDENT>LOGERROR('<STR_LIT>')<EOL>return None<EOL><DEDENT>else:<EOL><INDENT>spot_fleet_reqid = resp['<STR_LIT>']<EOL>LOGINFO('<STR_LIT>' %<EOL>spot_fleet_reqid)<EOL>if not wait_until_up:<EOL><INDENT>return spot_fleet_reqid<EOL><DEDENT>else:<EOL><INDENT>ntries = <NUM_LIT:10><EOL>curr_try = <NUM_LIT:0><EOL>while curr_try < ntries:<EOL><INDENT>resp = client.describe_spot_fleet_requests(<EOL>SpotFleetRequestIds=[<EOL>spot_fleet_reqid<EOL>]<EOL>)<EOL>curr_state = resp.get('<STR_LIT>',[])<EOL>if len(curr_state) > <NUM_LIT:0>:<EOL><INDENT>curr_state = curr_state[<NUM_LIT:0>]['<STR_LIT>']<EOL>if curr_state == '<STR_LIT>':<EOL><INDENT>LOGINFO('<STR_LIT>' %<EOL>spot_fleet_reqid)<EOL>break<EOL><DEDENT><DEDENT>LOGINFO(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (curr_try, ntries)<EOL>)<EOL>curr_try = curr_try + <NUM_LIT:1><EOL>time.sleep(<NUM_LIT>)<EOL><DEDENT>return spot_fleet_reqid<EOL><DEDENT><DEDENT><DEDENT>except ClientError as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>')<EOL>if raiseonfail:<EOL><INDENT>raise<EOL><DEDENT>return None<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>')<EOL>if raiseonfail:<EOL><INDENT>raise<EOL><DEDENT>return None<EOL><DEDENT>

This makes an EC2 spot-fleet cluster.

    This requires a security group ID attached to a VPC config and subnet, a
    keypair generated beforehand, and an IAM role ARN for the instance. See:

    https://docs.aws.amazon.com/cli/latest/userguide/tutorial-ec2-ubuntu.html

    Use `user_data` to launch tasks on instance launch.

    Parameters
    ----------

    security_groupid : str
        The security group ID of the AWS VPC where the instances will be
        launched.

    subnet_id : str
        The subnet ID of the AWS VPC where the instances will be
        launched.

    keypair_name : str
        The name of the keypair to be used to allow SSH access to all instances
        launched here. This corresponds to an already downloaded AWS keypair PEM
        file.

    iam_instance_profile_arn : str
        The ARN string corresponding to the AWS instance profile that describes
        the permissions the launched instances have to access other AWS
        resources. Set this up in AWS IAM.

    spot_fleet_iam_role : str
        This is the name of AWS IAM role that allows the Spot Fleet Manager to
        scale up and down instances based on demand and instances failing,
        etc. Set this up in IAM.

    target_capacity : int
        The number of instances to target in the fleet request. The fleet
        manager service will attempt to maintain this number over the lifetime
        of the Spot Fleet Request.

    spot_price : float
        The bid price in USD for the instances. This is per hour. Keep this at
        about half the hourly on-demand price of the desired instances to make
        sure your instances aren't taken away by AWS when it needs capacity.

    expires_days : int
        The number of days this request is active for. All instances launched by
        this request will live at least this long and will be terminated
        automatically after.

    allocation_strategy : {'lowestPrice', 'diversified'}
        The allocation strategy used by the fleet manager.

    instance_types : list of str
        List of the instance type to launch. See the following URL for a list of
        IDs: https://aws.amazon.com/ec2/pricing/on-demand/

    instance_weights : list of float or None
        If `instance_types` is a list of different instance types, this is the
        relative weight applied towards launching each instance type. This can
        be used to launch a mix of instances in a defined ratio among their
        types. Doing this can make the spot fleet more resilient to AWS taking
        back the instances if it runs out of capacity.

    instance_ami : str
        The Amazon Machine Image ID that describes the OS the instances will use
        after launch. The default ID is Amazon Linux 2 in the US East region.

    instance_user_data : str or None
        This is either the path to a file on disk that contains a shell-script
        or a string containing a shell-script that will be executed by root
        right after the instance is launched. Use to automatically set up
        workers and queues. If None, will not execute anything at instance
        start up.

    instance_ebs_optimized : bool
        If True, will enable EBS optimization to speed up IO. This is usually
        True for all instances made available in the last couple of years.

    wait_until_up : bool
        If True, will not return from this function until the spot fleet request
        is acknowledged by AWS.

    client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its operations. Alternatively, pass in an existing `boto3.Client`
        instance to re-use it here.

    raiseonfail : bool
        If True, will re-raise whatever Exception caused the operation to fail
        and break out immediately.

    Returns
    -------

    str or None
        This is the spot fleet request ID if successful. Otherwise, returns
        None.


if not client:<EOL><INDENT>client = boto3.client('<STR_LIT>')<EOL><DEDENT>resp = client.cancel_spot_fleet_requests(<EOL>SpotFleetRequestIds=[spot_fleet_reqid],<EOL>TerminateInstances=True<EOL>)<EOL>return resp<EOL>

This deletes a spot-fleet cluster.

Parameters
----------

spot_fleet_reqid : str
    The fleet request ID returned by `make_spot_fleet_cluster`.

client : boto3.Client or None
    If None, this function will instantiate a new `boto3.Client` object to
    use in its operations. Alternatively, pass in an existing `boto3.Client`
    instance to re-use it here.

Returns
-------

Nothing.


if not client:<EOL><INDENT>client = boto3.client('<STR_LIT>')<EOL><DEDENT>if isinstance(user_data, str) and os.path.exists(user_data):<EOL><INDENT>with open(user_data,'<STR_LIT:r>') as infd:<EOL><INDENT>udata = infd.read()<EOL><DEDENT><DEDENT>elif isinstance(user_data, str):<EOL><INDENT>udata = user_data<EOL><DEDENT>else:<EOL><INDENT>udata = (<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % datetime.utcnow().isoformat()<EOL>)<EOL><DEDENT>try:<EOL><INDENT>resp = client.run_instances(<EOL>ImageId=ami,<EOL>InstanceType=instance,<EOL>SecurityGroupIds=[<EOL>security_groupid,<EOL>],<EOL>SubnetId=subnet_id,<EOL>UserData=udata,<EOL>IamInstanceProfile={'<STR_LIT>':iam_instance_profile_arn},<EOL>InstanceInitiatedShutdownBehavior='<STR_LIT>',<EOL>KeyName=keypair_name,<EOL>MaxCount=launch_instances,<EOL>MinCount=launch_instances,<EOL>EbsOptimized=ebs_optimized,<EOL>)<EOL>if not resp:<EOL><INDENT>LOGERROR('<STR_LIT>')<EOL>return None<EOL><DEDENT>else:<EOL><INDENT>instance_dict = {}<EOL>instance_list = resp.get('<STR_LIT>',[])<EOL>if len(instance_list) > <NUM_LIT:0>:<EOL><INDENT>for instance in instance_list:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% (instance['<STR_LIT>'],<EOL>instance['<STR_LIT>'],<EOL>instance['<STR_LIT>'].isoformat(),<EOL>instance['<STR_LIT>']['<STR_LIT:Name>']))<EOL>instance_dict[instance['<STR_LIT>']] = {<EOL>'<STR_LIT:type>':instance['<STR_LIT>'],<EOL>'<STR_LIT>':instance['<STR_LIT>'],<EOL>'<STR_LIT:state>':instance['<STR_LIT>']['<STR_LIT:Name>'],<EOL>'<STR_LIT:info>':instance<EOL>}<EOL><DEDENT><DEDENT>if wait_until_up:<EOL><INDENT>ready_instances = []<EOL>LOGINFO('<STR_LIT>')<EOL>ntries = <NUM_LIT:5><EOL>curr_try = <NUM_LIT:0><EOL>while ( (curr_try < ntries) or<EOL>( len(ready_instances) <<EOL>len(list(instance_dict.keys()))) ):<EOL><INDENT>resp = client.describe_instances(<EOL>InstanceIds=list(instance_dict.keys()),<EOL>)<EOL>if len(resp['<STR_LIT>']) > <NUM_LIT:0>:<EOL><INDENT>for resv in resp['<STR_LIT>']:<EOL><INDENT>if len(resv['<STR_LIT>']) > <NUM_LIT:0>:<EOL><INDENT>for instance in resv['<STR_LIT>']:<EOL><INDENT>if instance['<STR_LIT>']['<STR_LIT:Name>'] == '<STR_LIT>':<EOL><INDENT>ready_instances.append(<EOL>instance['<STR_LIT>']<EOL>)<EOL>instance_dict[<EOL>instance['<STR_LIT>']<EOL>]['<STR_LIT:state>'] = '<STR_LIT>'<EOL>instance_dict[<EOL>instance['<STR_LIT>']<EOL>]['<STR_LIT>'] = instance['<STR_LIT>']<EOL>instance_dict[<EOL>instance['<STR_LIT>']<EOL>]['<STR_LIT:info>'] = instance<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>curr_try = curr_try + <NUM_LIT:1><EOL>time.sleep(<NUM_LIT>)<EOL><DEDENT>if len(ready_instances) == len(list(instance_dict.keys())):<EOL><INDENT>LOGINFO('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>)<EOL><DEDENT><DEDENT>return instance_dict<EOL><DEDENT><DEDENT>except ClientError as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>')<EOL>if raiseonfail:<EOL><INDENT>raise<EOL><DEDENT>return None<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>')<EOL>if raiseonfail:<EOL><INDENT>raise<EOL><DEDENT>return None<EOL><DEDENT>

This makes new EC2 worker nodes.

    This requires a security group ID attached to a VPC config and subnet, a
    keypair generated beforehand, and an IAM role ARN for the instance. See:

    https://docs.aws.amazon.com/cli/latest/userguide/tutorial-ec2-ubuntu.html

    Use `user_data` to launch tasks on instance launch.

    Parameters
    ----------

    security_groupid : str
        The security group ID of the AWS VPC where the instances will be
        launched.

    subnet_id : str
        The subnet ID of the AWS VPC where the instances will be
        launched.

    keypair_name : str
        The name of the keypair to be used to allow SSH access to all instances
        launched here. This corresponds to an already downloaded AWS keypair PEM
        file.

    iam_instance_profile_arn : str
        The ARN string corresponding to the AWS instance profile that describes
        the permissions the launched instances have to access other AWS
        resources. Set this up in AWS IAM.

    launch_instances : int
        The number of instances to launch in this request.

    ami : str
        The Amazon Machine Image ID that describes the OS the instances will use
        after launch. The default ID is Amazon Linux 2 in the US East region.

    instance : str
        The instance type to launch. See the following URL for a list of IDs:
        https://aws.amazon.com/ec2/pricing/on-demand/

    ebs_optimized : bool
        If True, will enable EBS optimization to speed up IO. This is usually
        True for all instances made available in the last couple of years.

    user_data : str or None
        This is either the path to a file on disk that contains a shell-script
        or a string containing a shell-script that will be executed by root
        right after the instance is launched. Use to automatically set up
        workers and queues. If None, will not execute anything at instance
        start up.

    wait_until_up : bool
        If True, will not return from this function until all launched instances
        are verified as running by AWS.

    client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its operations. Alternatively, pass in an existing `boto3.Client`
        instance to re-use it here.

    raiseonfail : bool
        If True, will re-raise whatever Exception caused the operation to fail
        and break out immediately.

    Returns
    -------

    dict
        Returns launched instance info as a dict, keyed by instance ID.


def plot_periodbase_lsp(lspinfo, outfile=None, plotdpi=<NUM_LIT:100>):

<EOL>if isinstance(lspinfo,str) and os.path.exists(lspinfo):<EOL><INDENT>LOGINFO('<STR_LIT>' % lspinfo)<EOL>with open(lspinfo,'<STR_LIT:rb>') as infd:<EOL><INDENT>lspinfo = pickle.load(infd)<EOL><DEDENT><DEDENT>try:<EOL><INDENT>periods = lspinfo['<STR_LIT>']<EOL>lspvals = lspinfo['<STR_LIT>']<EOL>bestperiod = lspinfo['<STR_LIT>']<EOL>lspmethod = lspinfo['<STR_LIT>']<EOL>plt.plot(periods, lspvals)<EOL>plt.xscale('<STR_LIT>',basex=<NUM_LIT:10>)<EOL>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel(PLOTYLABELS[lspmethod])<EOL>plottitle = '<STR_LIT>' % (METHODSHORTLABELS[lspmethod],<EOL>bestperiod)<EOL>plt.title(plottitle)<EOL>for bestperiod, bestpeak in zip(lspinfo['<STR_LIT>'],<EOL>lspinfo['<STR_LIT>']):<EOL><INDENT>plt.annotate('<STR_LIT>' % bestperiod,<EOL>xy=(bestperiod, bestpeak), xycoords='<STR_LIT:data>',<EOL>xytext=(<NUM_LIT:0.0>,<NUM_LIT>), textcoords='<STR_LIT>',<EOL>arrowprops=dict(arrowstyle="<STR_LIT>"),fontsize='<STR_LIT>')<EOL><DEDENT>plt.grid(color='<STR_LIT>',<EOL>alpha=<NUM_LIT>,<EOL>zorder=<NUM_LIT:0>,<EOL>linewidth=<NUM_LIT:1.0>,<EOL>linestyle='<STR_LIT::>')<EOL>if outfile and isinstance(outfile, str):<EOL><INDENT>if outfile.endswith('<STR_LIT>'):<EOL><INDENT>plt.savefig(outfile,bbox_inches='<STR_LIT>',dpi=plotdpi)<EOL><DEDENT>else:<EOL><INDENT>plt.savefig(outfile,bbox_inches='<STR_LIT>')<EOL><DEDENT>plt.close()<EOL>return os.path.abspath(outfile)<EOL><DEDENT>elif dispok:<EOL><INDENT>plt.show()<EOL>plt.close()<EOL>return<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>')<EOL>outfile = '<STR_LIT>'<EOL>plt.savefig(outfile,bbox_inches='<STR_LIT>',dpi=plotdpi)<EOL>plt.close()<EOL>return os.path.abspath(outfile)<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>')<EOL>return<EOL><DEDENT>

Makes a plot of periodograms obtained from `periodbase` functions.

    This takes the output dict produced by any `astrobase.periodbase`
    period-finder function or a pickle filename containing such a dict and makes
    a periodogram plot.

    Parameters
    ----------

    lspinfo : dict or str
        If lspinfo is a dict, it must be a dict produced by an
        `astrobase.periodbase` period-finder function or a dict from your own
        period-finder function or routine that is of the form below with at
        least these keys::

            {'periods': np.array of all periods searched by the period-finder,
             'lspvals': np.array of periodogram power value for each period,
             'bestperiod': a float value that is the period with the highest
                           peak in the periodogram, i.e. the most-likely actual
                           period,
             'method': a three-letter code naming the period-finder used; must
                       be one of the keys in the `METHODLABELS` dict above,
             'nbestperiods': a list of the periods corresponding to periodogram
                             peaks (`nbestlspvals` below) to annotate on the
                             periodogram plot so they can be called out
                             visually,
             'nbestlspvals': a list of the power values associated with
                             periodogram peaks to annotate on the periodogram
                             plot so they can be called out visually; should be
                             the same length as `nbestperiods` above}

        If lspinfo is a str, then it must be a path to a pickle file that
        contains a dict of the form described above.

    outfile : str or None
        If this is a str, will write the periodogram plot to the file specified
        by this string. If this is None, will write to a file called
        'lsp-plot.png' in the current working directory.

    plotdpi : int
        Sets the resolution in DPI of the output periodogram plot PNG file.

    Returns
    -------

    str
        Absolute path to the periodogram plot file created.


<EOL>if wcsfrom is None:<EOL><INDENT>hdulist = pyfits.open(fitsfile)<EOL>img, hdr = hdulist[fitsext].data, hdulist[fitsext].header<EOL>hdulist.close()<EOL>frameshape = (hdr['<STR_LIT>'], hdr['<STR_LIT>'])<EOL>w = WCS(hdr)<EOL><DEDENT>elif os.path.exists(wcsfrom):<EOL><INDENT>hdulist = pyfits.open(fitsfile)<EOL>img, hdr = hdulist[fitsext].data, hdulist[fitsext].header<EOL>hdulist.close()<EOL>frameshape = (hdr['<STR_LIT>'], hdr['<STR_LIT>'])<EOL>w = WCS(wcsfrom)<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>' %<EOL>fitsfile)<EOL>return None<EOL><DEDENT>if findersize is None:<EOL><INDENT>fig = plt.figure(figsize=(frameshape[<NUM_LIT:0>]/<NUM_LIT>,<EOL>frameshape[<NUM_LIT:1>]/<NUM_LIT>))<EOL><DEDENT>else:<EOL><INDENT>fig = plt.figure(figsize=findersize)<EOL><DEDENT>if (overlay_zoomcontain and<EOL>overlay_ra is not None and<EOL>overlay_decl is not None):<EOL><INDENT>finder_coordlimits = [overlay_ra.min()-<NUM_LIT>/<NUM_LIT>,<EOL>overlay_ra.max()+<NUM_LIT>/<NUM_LIT>,<EOL>overlay_decl.min()-<NUM_LIT>/<NUM_LIT>,<EOL>overlay_decl.max()+<NUM_LIT>/<NUM_LIT>]<EOL><DEDENT>if finder_coordlimits and isinstance(finder_coordlimits, (list,tuple)):<EOL><INDENT>minra, maxra, mindecl, maxdecl = finder_coordlimits<EOL>cntra, cntdecl = (minra + maxra)/<NUM_LIT>, (mindecl + maxdecl)/<NUM_LIT><EOL>pixelcoords = w.all_world2pix([[minra, mindecl],<EOL>[maxra, maxdecl],<EOL>[cntra, cntdecl]],<NUM_LIT:1>)<EOL>x1, y1, x2, y2 = (int(pixelcoords[<NUM_LIT:0>,<NUM_LIT:0>]),<EOL>int(pixelcoords[<NUM_LIT:0>,<NUM_LIT:1>]),<EOL>int(pixelcoords[<NUM_LIT:1>,<NUM_LIT:0>]),<EOL>int(pixelcoords[<NUM_LIT:1>,<NUM_LIT:1>]))<EOL>xmin = x1 if x1 < x2 else x2<EOL>xmax = x2 if x2 > x1 else x1<EOL>ymin = y1 if y1 < y2 else y2<EOL>ymax = y2 if y2 > y1 else y1<EOL>whdr = w.to_header()<EOL>whdr['<STR_LIT>'] = (xmax - xmin)/<NUM_LIT:2><EOL>whdr['<STR_LIT>'] = (ymax - ymin)/<NUM_LIT:2><EOL>whdr['<STR_LIT>'] = cntra<EOL>whdr['<STR_LIT>'] = cntdecl<EOL>whdr['<STR_LIT>'] = xmax - xmin<EOL>whdr['<STR_LIT>'] = ymax - ymin<EOL>w = WCS(whdr)<EOL><DEDENT>else:<EOL><INDENT>xmin, xmax, ymin, ymax = <NUM_LIT:0>, hdr['<STR_LIT>'], <NUM_LIT:0>, hdr['<STR_LIT>']<EOL><DEDENT>fig.add_subplot(<NUM_LIT>,projection=w)<EOL>if scale is not None and stretch is not None:<EOL><INDENT>norm = ImageNormalize(img,<EOL>interval=scale,<EOL>stretch=stretch)<EOL>plt.imshow(img[ymin:ymax,xmin:xmax],<EOL>origin='<STR_LIT>',<EOL>cmap=colormap,<EOL>norm=norm)<EOL><DEDENT>else:<EOL><INDENT>plt.imshow(img[ymin:ymax,xmin:xmax],<EOL>origin='<STR_LIT>',<EOL>cmap=colormap)<EOL><DEDENT>if grid:<EOL><INDENT>plt.grid(color=gridcolor,ls='<STR_LIT>',lw=<NUM_LIT:1.0>)<EOL><DEDENT>if overlay_ra is not None and overlay_decl is not None:<EOL><INDENT>our_pltopts = dict(<EOL>transform=plt.gca().get_transform('<STR_LIT>'),<EOL>marker='<STR_LIT:o>',<EOL>markersize=<NUM_LIT>,<EOL>markerfacecolor='<STR_LIT:none>',<EOL>markeredgewidth=<NUM_LIT>,<EOL>markeredgecolor='<STR_LIT>',<EOL>rasterized=True,<EOL>linestyle='<STR_LIT:none>'<EOL>)<EOL>if overlay_pltopts is not None and isinstance(overlay_pltopts,<EOL>dict):<EOL><INDENT>our_pltopts.update(overlay_pltopts)<EOL><DEDENT>plt.gca().set_autoscale_on(False)<EOL>plt.gca().plot(overlay_ra, overlay_decl,<EOL>**our_pltopts)<EOL><DEDENT>plt.xlabel('<STR_LIT>')<EOL>plt.ylabel('<STR_LIT>')<EOL>xax = plt.gca().coords[<NUM_LIT:0>]<EOL>yax = plt.gca().coords[<NUM_LIT:1>]<EOL>yax.set_major_formatter('<STR_LIT>')<EOL>xax.set_major_formatter('<STR_LIT>')<EOL>plt.savefig(outfile, dpi=<NUM_LIT>)<EOL>plt.close('<STR_LIT:all>')<EOL>return outfile<EOL>

This makes a finder chart for a given FITS with an optional object
    position overlay.

    Parameters
    ----------

    fitsfile : str
        `fitsfile` is the FITS file to use to make the finder chart.

    outfile : str
        `outfile` is the name of the output file. This can be a png or pdf or
        whatever else matplotlib can write given a filename and extension.

    fitsext : int
        Sets the FITS extension in `fitsfile` to use to extract the image array
        from.

    wcsfrom : str or None
        If `wcsfrom` is None, the WCS to transform the RA/Dec to pixel x/y will
        be taken from the FITS header of `fitsfile`. If this is not None, it
        must be a FITS or similar file that contains a WCS header in its first
        extension.

    scale : astropy.visualization.Interval object
        `scale` sets the normalization for the FITS pixel values. This is an
        astropy.visualization Interval object.
        See http://docs.astropy.org/en/stable/visualization/normalization.html
        for details on `scale` and `stretch` objects.

    stretch : astropy.visualization.Stretch object
        `stretch` sets the stretch function for mapping FITS pixel values to
        output pixel values. This is an astropy.visualization Stretch object.
        See http://docs.astropy.org/en/stable/visualization/normalization.html
        for details on `scale` and `stretch` objects.

    colormap : matplotlib Colormap object
        `colormap` is a matplotlib color map object to use for the output image.

    findersize : None or tuple of two ints
        If `findersize` is None, the output image size will be set by the NAXIS1
        and NAXIS2 keywords in the input `fitsfile` FITS header. Otherwise,
        `findersize` must be a tuple with the intended x and y size of the image
        in inches (all output images will use a DPI = 100).

    finder_coordlimits : list of four floats or None
        If not None, `finder_coordlimits` sets x and y limits for the plot,
        effectively zooming it in if these are smaller than the dimensions of
        the FITS image. This should be a list of the form: [minra, maxra,
        mindecl, maxdecl] all in decimal degrees.

    overlay_ra, overlay_decl : np.array or None
        `overlay_ra` and `overlay_decl` are ndarrays containing the RA and Dec
        values to overplot on the image as an overlay. If these are both None,
        then no overlay will be plotted.

    overlay_pltopts : dict
        `overlay_pltopts` controls how the overlay points will be plotted. This
        a dict with standard matplotlib marker, etc. kwargs as key-val pairs,
        e.g. 'markersize', 'markerfacecolor', etc. The default options make red
        outline circles at the location of each object in the overlay.

    overlay_zoomcontain : bool
        `overlay_zoomcontain` controls if the finder chart will be zoomed to
        just contain the overlayed points. Everything outside the footprint of
        these points will be discarded.

    grid : bool
        `grid` sets if a grid will be made on the output image.

    gridcolor : str
        `gridcolor` sets the color of the grid lines. This is a usual matplotib
        color spec string.

    Returns
    -------

    str or None
        The filename of the generated output image if successful. None
        otherwise.


stampdict = get_stamp(ra, decl,<EOL>survey=survey,<EOL>scaling=scaling,<EOL>forcefetch=forcefetch,<EOL>cachedir=cachedir,<EOL>timeout=timeout,<EOL>retry_failed=retry_failed,<EOL>verbose=verbose)<EOL>if stampdict:<EOL><INDENT>stampfits = pyfits.open(stampdict['<STR_LIT>'])<EOL>header = stampfits[<NUM_LIT:0>].header<EOL>frame = stampfits[<NUM_LIT:0>].data<EOL>stampfits.close()<EOL>if flip:<EOL><INDENT>frame = np.flipud(frame)<EOL><DEDENT>if verbose:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>% (ra, decl))<EOL><DEDENT>if convolvewith:<EOL><INDENT>convolved = aconv.convolve(frame, convolvewith)<EOL>if savewcsheader:<EOL><INDENT>return convolved, header<EOL><DEDENT>else:<EOL><INDENT>return convolved<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if savewcsheader:<EOL><INDENT>return frame, header<EOL><DEDENT>else:<EOL><INDENT>return frame<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% (ra, decl, survey, scaling))<EOL>return None<EOL><DEDENT>

This downloads a DSS FITS stamp centered on the coordinates specified.

    This wraps the function :py:func:`astrobase.services.skyview.get_stamp`,
    which downloads Digitized Sky Survey stamps in FITS format from the NASA
    SkyView service:

    https://skyview.gsfc.nasa.gov/current/cgi/query.pl

    Also adds some useful operations on top of the FITS file returned.

    Parameters
    ----------

    ra,decl : float
        The center coordinates for the stamp in decimal degrees.

    survey : str
        The survey name to get the stamp from. This is one of the
        values in the 'SkyView Surveys' option boxes on the SkyView
        webpage. Currently, we've only tested using 'DSS2 Red' as the value for
        this kwarg, but the other ones should work in principle.

    scaling : str
        This is the pixel value scaling function to use.

    flip : bool
        Will flip the downloaded image top to bottom. This should usually be
        True because matplotlib and FITS have different image coord origin
        conventions. Alternatively, set this to False and use the
        `origin='lower'` in any call to `matplotlib.pyplot.imshow` when plotting
        this image.

    convolvewith : astropy.convolution Kernel object or None
        If `convolvewith` is an astropy.convolution Kernel object from:

        http://docs.astropy.org/en/stable/convolution/kernels.html

        then, this function will return the stamp convolved with that
        kernel. This can be useful to see effects of wide-field telescopes (like
        the HATNet and HATSouth lenses) degrading the nominal 1 arcsec/px of
        DSS, causing blending of targets and any variability.

    forcefetch : bool
        If True, will disregard any existing cached copies of the stamp already
        downloaded corresponding to the requested center coordinates and
        redownload the FITS from the SkyView service.

    cachedir : str
        This is the path to the astrobase cache directory. All downloaded FITS
        stamps are stored here as .fits.gz files so we can immediately respond
        with the cached copy when a request is made for a coordinate center
        that's already been downloaded.

    timeout : float
        Sets the timeout in seconds to wait for a response from the NASA SkyView
        service.

    retry_failed : bool
        If the initial request to SkyView fails, and this is True, will retry
        until it succeeds.

    savewcsheader : bool
        If this is True, also returns the WCS header of the downloaded FITS
        stamp in addition to the FITS image itself. Useful for projecting object
        coordinates onto image xy coordinates for visualization.

    verbose : bool
        If True, indicates progress.

    Returns
    -------

    tuple or array or None
        This returns based on the value of `savewcsheader`:

        - If `savewcsheader=True`, returns a tuple:
          (FITS stamp image as a numpy array, FITS header)
        - If `savewcsheader=False`, returns only the FITS stamp image as numpy
          array.
        - If the stamp retrieval fails, returns None.


if radians:<EOL><INDENT>wrapped = angle % (<NUM_LIT>*pi_value)<EOL>if wrapped < <NUM_LIT:0.0>:<EOL><INDENT>wrapped = <NUM_LIT>*pi_value + wrapped<EOL><DEDENT><DEDENT>else:<EOL><INDENT>wrapped = angle % <NUM_LIT><EOL>if wrapped < <NUM_LIT:0.0>:<EOL><INDENT>wrapped = <NUM_LIT> + wrapped<EOL><DEDENT><DEDENT>return wrapped<EOL>

Wraps the input angle to 360.0 degrees.

    Parameters
    ----------

    angle : float
        The angle to wrap around 360.0 deg.

    radians : bool
        If True, will assume that the input is in radians. The output will then
        also be in radians.

    Returns
    -------

    float
        Wrapped angle. If radians is True: input is assumed to be in radians,
        output is also in radians.


if decimal_value < <NUM_LIT:0>:<EOL><INDENT>negative = True<EOL>dec_val = fabs(decimal_value)<EOL><DEDENT>else:<EOL><INDENT>negative = False<EOL>dec_val = decimal_value<EOL><DEDENT>degrees = trunc(dec_val)<EOL>minutes_deg = dec_val - degrees<EOL>minutes_mm = minutes_deg * <NUM_LIT><EOL>minutes_out = trunc(minutes_mm)<EOL>seconds = (minutes_mm - minutes_out)*<NUM_LIT><EOL>if negative:<EOL><INDENT>degrees = degrees<EOL>return '<STR_LIT:->', degrees, minutes_out, seconds<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT:+>', degrees, minutes_out, seconds<EOL><DEDENT>

Converts from decimal degrees (for declination coords) to DD:MM:SS.

    Parameters
    ----------

    decimal_value : float
        A decimal value to convert to degrees, minutes, seconds sexagesimal
        format.

    Returns
    -------

    tuple
        A four element tuple is returned: (sign, HH, MM, SS.ssss...)


if '<STR_LIT::>' in hms_string:<EOL><INDENT>separator = '<STR_LIT::>'<EOL><DEDENT>else:<EOL><INDENT>separator = '<STR_LIT:U+0020>'<EOL><DEDENT>hh, mm, ss = hms_string.split(separator)<EOL>return int(hh), int(mm), float(ss)<EOL>

Converts a string of the form HH:MM:SS or HH MM SS to a tuple of the form
    (HH, MM, SS).

    Parameters
    ----------

    hms_string : str
        A RA coordinate string of the form 'HH:MM:SS.sss' or 'HH MM SS.sss'.

    Returns
    -------

    tuple
        A three element tuple is returned (HH, MM, SS.ssss...)


def xieta_from_radecl(inra, indecl,<EOL>incenterra, incenterdecl,<EOL>deg=True):

if deg:<EOL><INDENT>ra = np.radians(inra)<EOL>decl = np.radians(indecl)<EOL>centerra = np.radians(incenterra)<EOL>centerdecl = np.radians(incenterdecl)<EOL><DEDENT>else:<EOL><INDENT>ra = inra<EOL>decl = indecl<EOL>centerra = incenterra<EOL>centerdecl = incenterdecl<EOL><DEDENT>cdecc = np.cos(centerdecl)<EOL>sdecc = np.sin(centerdecl)<EOL>crac = np.cos(centerra)<EOL>srac = np.sin(centerra)<EOL>uu = np.cos(decl)*np.cos(ra)<EOL>vv = np.cos(decl)*np.sin(ra)<EOL>ww = np.sin(decl)<EOL>uun = uu*cdecc*crac + vv*cdecc*srac + ww*sdecc<EOL>vvn = -uu*srac + vv*crac<EOL>wwn = -uu*sdecc*crac - vv*sdecc*srac + ww*cdecc<EOL>denom = vvn*vvn + wwn*wwn<EOL>aunn = np.zeros_like(uun)<EOL>aunn[uun >= <NUM_LIT:1.0>] = <NUM_LIT:0.0><EOL>aunn[uun < <NUM_LIT:1.0>] = np.arccos(uun)<EOL>xi, eta = np.zeros_like(aunn), np.zeros_like(aunn)<EOL>xi[(aunn <= <NUM_LIT:0.0>) | (denom <= <NUM_LIT:0.0>)] = <NUM_LIT:0.0><EOL>eta[(aunn <= <NUM_LIT:0.0>) | (denom <= <NUM_LIT:0.0>)] = <NUM_LIT:0.0><EOL>sdenom = np.sqrt(denom)<EOL>xi[(aunn > <NUM_LIT:0.0>) | (denom > <NUM_LIT:0.0>)] = aunn*vvn/sdenom<EOL>eta[(aunn > <NUM_LIT:0.0>) | (denom > <NUM_LIT:0.0>)] = aunn*wwn/sdenom<EOL>if deg:<EOL><INDENT>return np.degrees(xi), np.degrees(eta)<EOL><DEDENT>else:<EOL><INDENT>return xi, eta<EOL><DEDENT>

This returns the image-plane projected xi-eta coords for inra, indecl.

    Parameters
    ----------

    inra,indecl : array-like
        The equatorial coordinates to get the xi, eta coordinates for in decimal
        degrees or radians.

    incenterra,incenterdecl : float
        The center coordinate values to use to calculate the plane-projected
        coordinates around.

    deg : bool
        If this is True, the input angles are assumed to be in degrees and the
        output is in degrees as well.

    Returns
    -------

    tuple of np.arrays
        This is the (`xi`, `eta`) coordinate pairs corresponding to the
        image-plane projected coordinates for each pair of input equatorial
        coordinates in (`inra`, `indecl`).


cosdecl = np.cos(np.radians(declcenter))<EOL>sindecl = np.sin(np.radians(declcenter))<EOL>cosra = np.cos(np.radians(racenter))<EOL>sinra = np.sin(np.radians(racenter))<EOL>xyzdist = <NUM_LIT> * np.sin(np.radians(searchradiusdeg)/<NUM_LIT>)<EOL>kdtindices = kdtree.query_ball_point([cosra*cosdecl,<EOL>sinra*cosdecl,<EOL>sindecl],<EOL>xyzdist,<EOL>n_jobs=conesearchworkers)<EOL>return kdtindices<EOL>

This does a cone-search around (`racenter`, `declcenter`) in `kdtree`.

    Parameters
    ----------

    kdtree : scipy.spatial.CKDTree
        This is a kdtree object generated by the `make_kdtree` function.

    racenter,declcenter : float or array-like
        This is the center coordinate to run the cone-search around in decimal
        degrees. If this is an np.array, will search for all coordinate pairs in
        the array.

    searchradiusdeg : float
        The search radius to use for the cone-search in decimal degrees.

    conesearchworkers : int
        The number of parallel workers to launch for the cone-search.

    Returns
    -------

    list or np.array of lists
        If (`racenter`, `declcenter`) is a single coordinate, this will return a
        list of the indices of the matching objects in the kdtree. If
        (`racenter`, `declcenter`) are array-likes, this will return an object
        array containing lists of matching object indices for each coordinate
        searched.


def total_proper_motion(pmra, pmdecl, decl):

pm = np.sqrt( pmdecl*pmdecl + pmra*pmra*np.cos(np.radians(decl)) *<EOL>np.cos(np.radians(decl)) )<EOL>return pm<EOL>

This calculates the total proper motion of an object.

    Parameters
    ----------

    pmra : float or array-like
        The proper motion(s) in right ascension, measured in mas/yr.

    pmdecl : float or array-like
        The proper motion(s) in declination, measured in mas/yr.

    decl : float or array-like
        The declination of the object(s) in decimal degrees.

    Returns
    -------

    float or array-like
        The total proper motion(s) of the object(s) in mas/yr.


def xmatch_neighbors(ra1, dec1,<EOL>ra2, dec2,<EOL>match_radius=<NUM_LIT>,<EOL>includeself=False,<EOL>sortresults=True):

dist = great_circle_dist(ra1,dec1,ra2,dec2)<EOL>if includeself:<EOL><INDENT>match_dist_ind = np.where(dist < match_radius)<EOL><DEDENT>else:<EOL><INDENT>match_dist_ind = np.where((dist < match_radius) & (dist > <NUM_LIT:0.1>))<EOL><DEDENT>if len(match_dist_ind) > <NUM_LIT:0>:<EOL><INDENT>match_dists = dist[match_dist_ind]<EOL>dist_sort_ind = np.argsort(match_dists)<EOL>if sortresults:<EOL><INDENT>match_dist_ind = (match_dist_ind[<NUM_LIT:0>])[dist_sort_ind]<EOL><DEDENT>min_dist = np.min(match_dists)<EOL>return (True,min_dist,match_dist_ind,match_dists[dist_sort_ind])<EOL><DEDENT>else:<EOL><INDENT>return (False,)<EOL><DEDENT>

Finds the closest objects in (`ra2`, `dec2`) to scalar coordinate pair
    (`ra1`, `dec1`) and returns the indices of the objects that match.

    This is a quick matcher that uses the `great_circle_dist` function to find
    the closest object in (`ra2`, `dec2`) within `match_radius` arcseconds to
    (`ra1`, `dec1`). (`ra1`, `dec1`) must be a scalar pair, while
    (`ra2`, `dec2`) must be array-likes of the same lengths.

    Parameters
    ----------

    ra1,dec1 : float
        Coordinate of the object to find matches to. In decimal degrees.

    ra2,dec2 : array-like
        The coordinates that will be searched for matches. In decimal degrees.

    match_radius : float
        The match radius in arcseconds to use for the match.

    includeself : bool
        If this is True, the object itself will be included in the match
        results.

    sortresults : bool
        If this is True, the match indices will be sorted by distance.

    Returns
    -------

    tuple
        A tuple like the following is returned::

            (True -> matches found or False -> no matches found,
             minimum distance between target and list,
             np.array of indices where list of coordinates is
             closer than `match_radius` arcseconds from the target,
             np.array of distances in arcseconds)


<EOL>cosdecl = np.cos(np.radians(decl))<EOL>sindecl = np.sin(np.radians(decl))<EOL>cosra = np.cos(np.radians(ra))<EOL>sinra = np.sin(np.radians(ra))<EOL>xyz = np.column_stack((cosra*cosdecl,sinra*cosdecl, sindecl))<EOL>kdt = sps.cKDTree(xyz,copy_data=True)<EOL>return kdt<EOL>

This makes a `scipy.spatial.CKDTree` on (`ra`, `decl`).

    Parameters
    ----------

    ra,decl : array-like
        The right ascension and declination coordinate pairs in decimal degrees.

    Returns
    -------

    `scipy.spatial.CKDTree`
        The cKDTRee object generated by this function is returned and can be
        used to run various spatial queries.


<EOL>if not os.path.exists(outdir):<EOL><INDENT>os.makedirs(outdir)<EOL><DEDENT>if maxobjects:<EOL><INDENT>lclist = lclist[:maxobjects]<EOL><DEDENT>with open(lc_catalog_pickle, '<STR_LIT:rb>') as infd:<EOL><INDENT>kdt_dict = pickle.load(infd)<EOL><DEDENT>kdt = kdt_dict['<STR_LIT>']<EOL>objlist = kdt_dict['<STR_LIT>']['<STR_LIT>']<EOL>objlcfl = kdt_dict['<STR_LIT>']['<STR_LIT>']<EOL>tasks = [(x, outdir, kdt, objlist, objlcfl,<EOL>neighbor_radius_arcsec,<EOL>deredden, custom_bandpasses,<EOL>lcformat, lcformatdir) for x in lclist]<EOL>for task in tqdm(tasks):<EOL><INDENT>result = _starfeatures_worker(task)<EOL><DEDENT>return result<EOL>

This drives the `get_starfeatures` function for a collection of LCs.

    Parameters
    ----------

    lclist : list of str
        The list of light curve file names to process.

    outdir : str
        The output directory where the results will be placed.

    lc_catalog_pickle : str
        The path to a catalog containing at a dict with least:

        - an object ID array accessible with `dict['objects']['objectid']`

        - an LC filename array accessible with `dict['objects']['lcfname']`

        - a `scipy.spatial.KDTree` or `cKDTree` object to use for finding
          neighbors for each object accessible with `dict['kdtree']`

        A catalog pickle of the form needed can be produced using
        :py:func:`astrobase.lcproc.catalogs.make_lclist` or
        :py:func:`astrobase.lcproc.catalogs.filter_lclist`.

    neighbor_radius_arcsec : float
        This indicates the radius in arcsec to search for neighbors for this
        object using the light curve catalog's `kdtree`, `objlist`, `lcflist`,
        and in GAIA.

    maxobjects : int
        The number of objects to process from `lclist`.

    deredden : bool
        This controls if the colors and any color classifications will be
        dereddened using 2MASS DUST.

    custom_bandpasses : dict or None
        This is a dict used to define any custom bandpasses in the
        `in_objectinfo` dict you want to make this function aware of and
        generate colors for. Use the format below for this dict::

            {
            '<bandpass_key_1>':{'dustkey':'<twomass_dust_key_1>',
                                'label':'<band_label_1>'
                                'colors':[['<bandkey1>-<bandkey2>',
                                           '<BAND1> - <BAND2>'],
                                          ['<bandkey3>-<bandkey4>',
                                           '<BAND3> - <BAND4>']]},
            .
            ...
            .
            '<bandpass_key_N>':{'dustkey':'<twomass_dust_key_N>',
                                'label':'<band_label_N>'
                                'colors':[['<bandkey1>-<bandkey2>',
                                           '<BAND1> - <BAND2>'],
                                          ['<bandkey3>-<bandkey4>',
                                           '<BAND3> - <BAND4>']]},
            }

        Where:

        `bandpass_key` is a key to use to refer to this bandpass in the
        `objectinfo` dict, e.g. 'sdssg' for SDSS g band

        `twomass_dust_key` is the key to use in the 2MASS DUST result table for
        reddening per band-pass. For example, given the following DUST result
        table (using http://irsa.ipac.caltech.edu/applications/DUST/)::

            |Filter_name|LamEff |A_over_E_B_V_SandF|A_SandF|A_over_E_B_V_SFD|A_SFD|
            |char       |float  |float             |float  |float           |float|
            |           |microns|                  |mags   |                |mags |
             CTIO U       0.3734              4.107   0.209            4.968 0.253
             CTIO B       0.4309              3.641   0.186            4.325 0.221
             CTIO V       0.5517              2.682   0.137            3.240 0.165
            .
            .
            ...

        The `twomass_dust_key` for 'vmag' would be 'CTIO V'. If you want to
        skip DUST lookup and want to pass in a specific reddening magnitude
        for your bandpass, use a float for the value of
        `twomass_dust_key`. If you want to skip DUST lookup entirely for
        this bandpass, use None for the value of `twomass_dust_key`.

        `band_label` is the label to use for this bandpass, e.g. 'W1' for
        WISE-1 band, 'u' for SDSS u, etc.

        The 'colors' list contains color definitions for all colors you want
        to generate using this bandpass. this list contains elements of the
        form::

            ['<bandkey1>-<bandkey2>','<BAND1> - <BAND2>']

        where the the first item is the bandpass keys making up this color,
        and the second item is the label for this color to be used by the
        frontends. An example::

            ['sdssu-sdssg','u - g']

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    Returns
    -------

    list of str
        A list of all star features pickles produced.


try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>try:<EOL><INDENT>lcdict = readerfunc(lcfile)<EOL>if ( (isinstance(lcdict, (list, tuple))) and<EOL>(isinstance(lcdict[<NUM_LIT:0>], dict)) ):<EOL><INDENT>lcdict = lcdict[<NUM_LIT:0>]<EOL><DEDENT>resultdict = {'<STR_LIT>':lcdict['<STR_LIT>'],<EOL>'<STR_LIT:info>':lcdict['<STR_LIT>'],<EOL>'<STR_LIT>':os.path.basename(lcfile)}<EOL>coordfeat = starfeatures.coord_features(lcdict['<STR_LIT>'])<EOL>colorfeat = starfeatures.color_features(<EOL>lcdict['<STR_LIT>'],<EOL>deredden=deredden,<EOL>custom_bandpasses=custom_bandpasses<EOL>)<EOL>colorclass = starfeatures.color_classification(colorfeat,<EOL>coordfeat)<EOL>nbrfeat = starfeatures.neighbor_gaia_features(lcdict['<STR_LIT>'],<EOL>kdtree,<EOL>neighbor_radius_arcsec)<EOL>if nbrfeat['<STR_LIT>'].size > <NUM_LIT:0>:<EOL><INDENT>nbrfeat['<STR_LIT>'] = objlist[nbrfeat['<STR_LIT>']]<EOL>nbrfeat['<STR_LIT>'] = objlist[<EOL>nbrfeat['<STR_LIT>']<EOL>]<EOL>nbrfeat['<STR_LIT>'] = lcflist[<EOL>nbrfeat['<STR_LIT>']<EOL>]<EOL><DEDENT>else:<EOL><INDENT>nbrfeat['<STR_LIT>'] = np.array([])<EOL>nbrfeat['<STR_LIT>'] = np.array([])<EOL>nbrfeat['<STR_LIT>'] = np.array([])<EOL><DEDENT>resultdict.update(coordfeat)<EOL>resultdict.update(colorfeat)<EOL>resultdict.update(colorclass)<EOL>resultdict.update(nbrfeat)<EOL>outfile = os.path.join(outdir,<EOL>'<STR_LIT>' %<EOL>squeeze(resultdict['<STR_LIT>']).replace('<STR_LIT:U+0020>','<STR_LIT:->'))<EOL>with open(outfile, '<STR_LIT:wb>') as outfd:<EOL><INDENT>pickle.dump(resultdict, outfd, protocol=<NUM_LIT:4>)<EOL><DEDENT>return outfile<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>' %<EOL>(os.path.basename(lcfile), e))<EOL>return None<EOL><DEDENT>

This runs the functions from :py:func:`astrobase.varclass.starfeatures`
    on a single light curve file.

    Parameters
    ----------

    lcfile : str
        This is the LC file to extract star features for.

    outdir : str
        This is the directory to write the output pickle to.

    kdtree: scipy.spatial.cKDTree
        This is a `scipy.spatial.KDTree` or `cKDTree` used to calculate neighbor
        proximity features. This is for the light curve catalog this object is
        in.

    objlist : np.array
        This is a Numpy array of object IDs in the same order as the
        `kdtree.data` np.array. This is for the light curve catalog this object
        is in.

    lcflist : np.array
        This is a Numpy array of light curve filenames in the same order as
        `kdtree.data`. This is for the light curve catalog this object is in.

    neighbor_radius_arcsec : float
        This indicates the radius in arcsec to search for neighbors for this
        object using the light curve catalog's `kdtree`, `objlist`, `lcflist`,
        and in GAIA.

    deredden : bool
        This controls if the colors and any color classifications will be
        dereddened using 2MASS DUST.

    custom_bandpasses : dict or None
        This is a dict used to define any custom bandpasses in the
        `in_objectinfo` dict you want to make this function aware of and
        generate colors for. Use the format below for this dict::

            {
            '<bandpass_key_1>':{'dustkey':'<twomass_dust_key_1>',
                                'label':'<band_label_1>'
                                'colors':[['<bandkey1>-<bandkey2>',
                                           '<BAND1> - <BAND2>'],
                                          ['<bandkey3>-<bandkey4>',
                                           '<BAND3> - <BAND4>']]},
            .
            ...
            .
            '<bandpass_key_N>':{'dustkey':'<twomass_dust_key_N>',
                                'label':'<band_label_N>'
                                'colors':[['<bandkey1>-<bandkey2>',
                                           '<BAND1> - <BAND2>'],
                                          ['<bandkey3>-<bandkey4>',
                                           '<BAND3> - <BAND4>']]},
            }

        Where:

        `bandpass_key` is a key to use to refer to this bandpass in the
        `objectinfo` dict, e.g. 'sdssg' for SDSS g band

        `twomass_dust_key` is the key to use in the 2MASS DUST result table for
        reddening per band-pass. For example, given the following DUST result
        table (using http://irsa.ipac.caltech.edu/applications/DUST/)::

            |Filter_name|LamEff |A_over_E_B_V_SandF|A_SandF|A_over_E_B_V_SFD|A_SFD|
            |char       |float  |float             |float  |float           |float|
            |           |microns|                  |mags   |                |mags |
             CTIO U       0.3734              4.107   0.209            4.968 0.253
             CTIO B       0.4309              3.641   0.186            4.325 0.221
             CTIO V       0.5517              2.682   0.137            3.240 0.165
            .
            .
            ...

        The `twomass_dust_key` for 'vmag' would be 'CTIO V'. If you want to
        skip DUST lookup and want to pass in a specific reddening magnitude
        for your bandpass, use a float for the value of
        `twomass_dust_key`. If you want to skip DUST lookup entirely for
        this bandpass, use None for the value of `twomass_dust_key`.

        `band_label` is the label to use for this bandpass, e.g. 'W1' for
        WISE-1 band, 'u' for SDSS u, etc.

        The 'colors' list contains color definitions for all colors you want
        to generate using this bandpass. this list contains elements of the
        form::

            ['<bandkey1>-<bandkey2>','<BAND1> - <BAND2>']

        where the the first item is the bandpass keys making up this color,
        and the second item is the label for this color to be used by the
        frontends. An example::

            ['sdssu-sdssg','u - g']

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    Returns
    -------

    str
        Path to the output pickle containing all of the star features for this
        object.


try:<EOL><INDENT>(lcfile, outdir, kdtree, objlist,<EOL>lcflist, neighbor_radius_arcsec,<EOL>deredden, custom_bandpasses, lcformat, lcformatdir) = task<EOL>return get_starfeatures(lcfile, outdir,<EOL>kdtree, objlist, lcflist,<EOL>neighbor_radius_arcsec,<EOL>deredden=deredden,<EOL>custom_bandpasses=custom_bandpasses,<EOL>lcformat=lcformat,<EOL>lcformatdir=lcformatdir)<EOL><DEDENT>except Exception as e:<EOL><INDENT>return None<EOL><DEDENT>


try:<EOL><INDENT>formatinfo = get_lcformat(lcformat,<EOL>use_lcformat_dir=lcformatdir)<EOL>if formatinfo:<EOL><INDENT>(dfileglob, readerfunc,<EOL>dtimecols, dmagcols, derrcols,<EOL>magsarefluxes, normfunc) = formatinfo<EOL><DEDENT>else:<EOL><INDENT>LOGERROR("<STR_LIT>")<EOL>return None<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION("<STR_LIT>")<EOL>return None<EOL><DEDENT>if not fileglob:<EOL><INDENT>fileglob = dfileglob<EOL><DEDENT>LOGINFO('<STR_LIT>' % (lcformat, lcdir))<EOL>if recursive is False:<EOL><INDENT>matching = glob.glob(os.path.join(lcdir, fileglob))<EOL><DEDENT>else:<EOL><INDENT>if sys.version_info[:<NUM_LIT:2>] > (<NUM_LIT:3>,<NUM_LIT:4>):<EOL><INDENT>matching = glob.glob(os.path.join(lcdir,<EOL>'<STR_LIT>',<EOL>fileglob),recursive=True)<EOL><DEDENT>else:<EOL><INDENT>walker = os.walk(lcdir)<EOL>matching = []<EOL>for root, dirs, _files in walker:<EOL><INDENT>for sdir in dirs:<EOL><INDENT>searchpath = os.path.join(root,<EOL>sdir,<EOL>fileglob)<EOL>foundfiles = glob.glob(searchpath)<EOL>if foundfiles:<EOL><INDENT>matching.extend(foundfiles)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if matching and len(matching) > <NUM_LIT:0>:<EOL><INDENT>LOGINFO('<STR_LIT>' %<EOL>len(matching))<EOL>return parallel_starfeatures(matching,<EOL>outdir,<EOL>lc_catalog_pickle,<EOL>neighbor_radius_arcsec,<EOL>deredden=deredden,<EOL>custom_bandpasses=custom_bandpasses,<EOL>maxobjects=maxobjects,<EOL>lcformat=lcformat,<EOL>lcformatdir=lcformatdir,<EOL>nworkers=nworkers)<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>' % (lcformat,<EOL>lcdir))<EOL>return None<EOL><DEDENT>

This runs parallel star feature extraction for a directory of LCs.

    Parameters
    ----------

    lcdir : list of str
        The directory to search for light curves.

    outdir : str
        The output directory where the results will be placed.

    lc_catalog_pickle : str
        The path to a catalog containing at a dict with least:

        - an object ID array accessible with `dict['objects']['objectid']`

        - an LC filename array accessible with `dict['objects']['lcfname']`

        - a `scipy.spatial.KDTree` or `cKDTree` object to use for finding
          neighbors for each object accessible with `dict['kdtree']`

        A catalog pickle of the form needed can be produced using
        :py:func:`astrobase.lcproc.catalogs.make_lclist` or
        :py:func:`astrobase.lcproc.catalogs.filter_lclist`.

    neighbor_radius_arcsec : float
        This indicates the radius in arcsec to search for neighbors for this
        object using the light curve catalog's `kdtree`, `objlist`, `lcflist`,
        and in GAIA.

    fileglob : str
        The UNIX file glob to use to search for the light curves in `lcdir`. If
        None, the default value for the light curve format specified will be
        used.

    maxobjects : int
        The number of objects to process from `lclist`.

    deredden : bool
        This controls if the colors and any color classifications will be
        dereddened using 2MASS DUST.

    custom_bandpasses : dict or None
        This is a dict used to define any custom bandpasses in the
        `in_objectinfo` dict you want to make this function aware of and
        generate colors for. Use the format below for this dict::

            {
            '<bandpass_key_1>':{'dustkey':'<twomass_dust_key_1>',
                                'label':'<band_label_1>'
                                'colors':[['<bandkey1>-<bandkey2>',
                                           '<BAND1> - <BAND2>'],
                                          ['<bandkey3>-<bandkey4>',
                                           '<BAND3> - <BAND4>']]},
            .
            ...
            .
            '<bandpass_key_N>':{'dustkey':'<twomass_dust_key_N>',
                                'label':'<band_label_N>'
                                'colors':[['<bandkey1>-<bandkey2>',
                                           '<BAND1> - <BAND2>'],
                                          ['<bandkey3>-<bandkey4>',
                                           '<BAND3> - <BAND4>']]},
            }

        Where:

        `bandpass_key` is a key to use to refer to this bandpass in the
        `objectinfo` dict, e.g. 'sdssg' for SDSS g band

        `twomass_dust_key` is the key to use in the 2MASS DUST result table for
        reddening per band-pass. For example, given the following DUST result
        table (using http://irsa.ipac.caltech.edu/applications/DUST/)::

            |Filter_name|LamEff |A_over_E_B_V_SandF|A_SandF|A_over_E_B_V_SFD|A_SFD|
            |char       |float  |float             |float  |float           |float|
            |           |microns|                  |mags   |                |mags |
             CTIO U       0.3734              4.107   0.209            4.968 0.253
             CTIO B       0.4309              3.641   0.186            4.325 0.221
             CTIO V       0.5517              2.682   0.137            3.240 0.165
            .
            .
            ...

        The `twomass_dust_key` for 'vmag' would be 'CTIO V'. If you want to
        skip DUST lookup and want to pass in a specific reddening magnitude
        for your bandpass, use a float for the value of
        `twomass_dust_key`. If you want to skip DUST lookup entirely for
        this bandpass, use None for the value of `twomass_dust_key`.

        `band_label` is the label to use for this bandpass, e.g. 'W1' for
        WISE-1 band, 'u' for SDSS u, etc.

        The 'colors' list contains color definitions for all colors you want
        to generate using this bandpass. this list contains elements of the
        form::

            ['<bandkey1>-<bandkey2>','<BAND1> - <BAND2>']

        where the the first item is the bandpass keys making up this color,
        and the second item is the label for this color to be used by the
        frontends. An example::

            ['sdssu-sdssg','u - g']

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    nworkers : int
        The number of parallel workers to launch.

    Returns
    -------

    dict
        A dict with key:val pairs of the input light curve filename and the
        output star features pickle for each LC processed.


<EOL>if not os.path.exists(outdir):<EOL><INDENT>os.makedirs(outdir)<EOL><DEDENT>if maxobjects:<EOL><INDENT>pfpkl_list = pfpkl_list[:maxobjects]<EOL><DEDENT>LOGINFO('<STR_LIT>' % len(pfpkl_list))<EOL>if starfeaturesdir and os.path.exists(starfeaturesdir):<EOL><INDENT>starfeatures_list = []<EOL>LOGINFO('<STR_LIT>')<EOL>for pfpkl in pfpkl_list:<EOL><INDENT>sfpkl1 = os.path.basename(pfpkl).replace('<STR_LIT>',<EOL>'<STR_LIT>')<EOL>sfpkl2 = sfpkl1.replace('<STR_LIT>','<STR_LIT>')<EOL>sfpath1 = os.path.join(starfeaturesdir, sfpkl1)<EOL>sfpath2 = os.path.join(starfeaturesdir, sfpkl2)<EOL>if os.path.exists(sfpath1):<EOL><INDENT>starfeatures_list.append(sfpkl1)<EOL><DEDENT>elif os.path.exists(sfpath2):<EOL><INDENT>starfeatures_list.append(sfpkl2)<EOL><DEDENT>else:<EOL><INDENT>starfeatures_list.append(None)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>starfeatures_list = [None for x in pfpkl_list]<EOL><DEDENT>kwargs = {'<STR_LIT>':fourierorder,<EOL>'<STR_LIT>':transitparams,<EOL>'<STR_LIT>':ebparams,<EOL>'<STR_LIT>':pdiff_threshold,<EOL>'<STR_LIT>':sidereal_threshold,<EOL>'<STR_LIT>':sampling_peak_multiplier,<EOL>'<STR_LIT>':sampling_startp,<EOL>'<STR_LIT>':sampling_endp,<EOL>'<STR_LIT>':timecols,<EOL>'<STR_LIT>':magcols,<EOL>'<STR_LIT>':errcols,<EOL>'<STR_LIT>':lcformat,<EOL>'<STR_LIT>':lcformat,<EOL>'<STR_LIT>':sigclip,<EOL>'<STR_LIT>':verbose}<EOL>tasks = [(x, lcbasedir, outdir, y, kwargs) for (x,y) in<EOL>zip(pfpkl_list, starfeatures_list)]<EOL>LOGINFO('<STR_LIT>')<EOL>with ProcessPoolExecutor(max_workers=nworkers) as executor:<EOL><INDENT>resultfutures = executor.map(_periodicfeatures_worker, tasks)<EOL><DEDENT>results = [x for x in resultfutures]<EOL>resdict = {os.path.basename(x):y for (x,y) in zip(pfpkl_list, results)}<EOL>return resdict<EOL>

This runs periodic feature generation in parallel for all periodfinding
    pickles in the input list.

    Parameters
    ----------

    pfpkl_list : list of str
        The list of period-finding pickles to use.

    lcbasedir : str
        The base directory where the associated light curves are located.

    outdir : str
        The directory where the results will be written.

    starfeaturesdir : str or None
        The directory containing the `starfeatures-<objectid>.pkl` files for
        each object to use calculate neighbor proximity light curve features.

    fourierorder : int
        The Fourier order to use to generate sinusoidal function and fit that to
        the phased light curve.

    transitparams : list of floats
        The transit depth, duration, and ingress duration to use to generate a
        trapezoid planet transit model fit to the phased light curve. The period
        used is the one provided in `period`, while the epoch is automatically
        obtained from a spline fit to the phased light curve.

    ebparams : list of floats
        The primary eclipse depth, eclipse duration, the primary-secondary depth
        ratio, and the phase of the secondary eclipse to use to generate an
        eclipsing binary model fit to the phased light curve. The period used is
        the one provided in `period`, while the epoch is automatically obtained
        from a spline fit to the phased light curve.

    pdiff_threshold : float
        This is the max difference between periods to consider them the same.

    sidereal_threshold : float
        This is the max difference between any of the 'best' periods and the
        sidereal day periods to consider them the same.

    sampling_peak_multiplier : float
        This is the minimum multiplicative factor of a 'best' period's
        normalized periodogram peak over the sampling periodogram peak at the
        same period required to accept the 'best' period as possibly real.

    sampling_startp, sampling_endp : float
        If the `pgramlist` doesn't have a time-sampling Lomb-Scargle
        periodogram, it will be obtained automatically. Use these kwargs to
        control the minimum and maximum period interval to be searched when
        generating this periodogram.

    timecols : list of str or None
        The timecol keys to use from the lcdict in calculating the features.

    magcols : list of str or None
        The magcol keys to use from the lcdict in calculating the features.

    errcols : list of str or None
        The errcol keys to use from the lcdict in calculating the features.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    verbose : bool
        If True, will indicate progress while working.

    maxobjects : int
        The total number of objects to process from `pfpkl_list`.

    nworkers : int
        The number of parallel workers to launch to process the input.

    Returns
    -------

    dict
        A dict containing key: val pairs of the input period-finder result and
        the output periodic feature result pickles for each input pickle is
        returned.