if not fname:<EOL><INDENT>fname = self.getname()<EOL><DEDENT>assert fname != None, '<STR_LIT>'<EOL>if not fname.endswith('<STR_LIT>'):<EOL><INDENT>fname += '<STR_LIT>'<EOL><DEDENT>try:<EOL><INDENT>urlretrieve(self.url, fname)<EOL><DEDENT>except Exception:<EOL><INDENT>raise IOError('<STR_LIT>'%fname)<EOL><DEDENT>return fname<EOL>

Download the chart from the URL into a filename as a PNG

The filename defaults to the chart title (chtt) if any


assert args[<NUM_LIT:0>].lower() in '<STR_LIT>', '<STR_LIT>'%level<EOL>self['<STR_LIT>'] = '<STR_LIT>'%args<EOL>return self<EOL>

Just used in QRCode for the moment
args are error_correction,margin_size
APIPARAM: chld


self._scale =  ['<STR_LIT:U+002C>'.join(map(smart_str, args))]<EOL>return self<EOL>

Scales the data down to the given size
args must be of the form::
    <data set 1 minimum value>,
    <data set 1 maximum value>,
    <data set n minimum value>,
    <data set n maximum value>
will only work with text encoding!
APIPARAM: chds


self.data['<STR_LIT>'].append(<EOL>str('<STR_LIT>'%(index, '<STR_LIT:|>'.join(map(str,args)) )).replace('<STR_LIT:None>','<STR_LIT>')<EOL>)<EOL>return self.parent<EOL>

Label each axes one at a time
args are of the form <label 1>,...,<label n>
APIPARAM: chxl


assert encoding in ('<STR_LIT>','<STR_LIT>','<STR_LIT>'),'<STR_LIT>'%encoding<EOL>self['<STR_LIT>'] = encoding<EOL>return self<EOL>

Output encoding to use for QRCode encoding
Must be one of 'Shift_JIS','UTF-8', or 'ISO-8859-1'
APIPARAM: choe


for opt,values in self.data.items():<EOL><INDENT>if opt == '<STR_LIT>':<EOL><INDENT>self['<STR_LIT>'] = '<STR_LIT:|>'.join(values)<EOL><DEDENT>else:<EOL><INDENT>self['<STR_LIT>'%opt[<NUM_LIT:0>]] = '<STR_LIT:|>'.join(values)<EOL><DEDENT><DEDENT>return self<EOL>


grids =  map(str,map(float,args))<EOL>self['<STR_LIT>'] = '<STR_LIT:U+002C>'.join(grids).replace('<STR_LIT:None>','<STR_LIT>')<EOL>return self<EOL>

Apply a grid to your chart
args are of the form::
    <x axis step size>,
    <y axis step size>,
    <length of line segment>,
    <length of blank segment>
    <x offset>,
    <y offset>
APIPARAM: chg


args = color_args(args, <NUM_LIT:0>)<EOL>self.data['<STR_LIT>'].append(<EOL>'<STR_LIT:U+002C>'.join([str(index)]+list(map(str,args)))<EOL>)<EOL>return self.parent<EOL>

Add style to your axis, one at a time
args are of the form::
    <axis color>,
    <font size>,
    <alignment>,
    <drawing control>,
    <tick mark color>
APIPARAM: chxs


self['<STR_LIT>'] = self.check_type(str(type))<EOL>return self<EOL>

Set the chart type, either Google API type or regular name
APIPARAM: cht


if type in TYPES:<EOL><INDENT>return type<EOL><DEDENT>tdict = dict(zip(TYPES,TYPES))<EOL>tdict.update({<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT:bar>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT:p>',<EOL>'<STR_LIT>': '<STR_LIT:v>',<EOL>'<STR_LIT>': '<STR_LIT:s>',<EOL>'<STR_LIT>': '<STR_LIT:r>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>})<EOL>assert type in tdict, '<STR_LIT>'%type<EOL>return tdict[type]<EOL>

Check to see if the type is either in TYPES or fits type name

        Returns proper type


def dataset(self, data, series='<STR_LIT>'):

self._dataset = data<EOL>self._series = series<EOL>return self<EOL>

Update the chart's dataset, can be two dimensional or contain string data


def _chart_support(self, *args, **kwargs):

print(args,kwargs)<EOL>return '<STR_LIT>'<EOL>


K = nx.complete_graph(k)<EOL>g1 = nx.cartesian_product(nx.create_empty_copy(graph), K)<EOL>g2 = nx.cartesian_product(graph, nx.create_empty_copy(K))<EOL>return nx.compose(g1, g2)<EOL>

the QUBO for k-coloring a graph A is as follows:

variables:
x_{v,c} = 1 if vertex v of A gets color c; x_{v,c} = 0 otherwise

constraints:
1)  each v in A gets exactly one color.
   This constraint is enforced by including the term (\sum_c x_{v,c} - 1)^2 in the QUBO,
   which is minimized when \sum_c x_{v,c} = 1.

2) If u and v in A are adjacent, then they get different colors.
   This constraint is enforced by including terms x_{v,c} x_{u,c} in the QUBO,
   which is minimzed when at most one of u and v get color c.

Total QUBO:
Q(x) = \sum_v (\sum_c x_{v,c} - 1)^2  + \sum_{u ~ v} \sum_c x_{v,c} x_{u,c}

The graph of interactions for this QUBO consists of cliques of size k (with vertices {x_{v,c} for c = 0,...,k-1})
plus k disjoint copies of the graph A (one for each color).


return {<EOL>"<STR_LIT:id>": course_id,<EOL>"<STR_LIT:url>": "<STR_LIT>".format(course_id),<EOL>"<STR_LIT:name>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:type>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>".format(course_id),<EOL>"<STR_LIT>": "<STR_LIT>"<EOL>}<EOL>


def mock_api_response(self, status, body):

httpretty.register_uri(<EOL>httpretty.POST,<EOL>'<STR_LIT>',<EOL>status=status,<EOL>body=json.dumps(body),<EOL>content_type='<STR_LIT:application/json>'<EOL>)<EOL>


return self.code<EOL>


return MockSailthruError(self.error, self.code)<EOL>


def assert_get_sailthru_client_raises(self, exc_class, config):

overrides = {<EOL>self.SITE_CODE: {<EOL>'<STR_LIT>': config<EOL>}<EOL>}<EOL>with mock.patch.dict(self.SITE_OVERRIDES_MODULE, overrides):<EOL><INDENT>with self.assertRaises(exc_class):<EOL><INDENT>get_sailthru_client(self.SITE_CODE)<EOL><DEDENT><DEDENT>

Asserts an error is raised by a call to get_sailthru_client.


def _update_unenrolled_list(sailthru_client, email, course_url, unenroll):

try:<EOL><INDENT>sailthru_response = sailthru_client.api_get("<STR_LIT:user>", {"<STR_LIT:id>": email, "<STR_LIT>": {"<STR_LIT>": <NUM_LIT:1>}})<EOL>if not sailthru_response.is_ok():<EOL><INDENT>error = sailthru_response.get_error()<EOL>logger.error("<STR_LIT>", error.get_message())<EOL>return not can_retry_sailthru_request(error)<EOL><DEDENT>response_json = sailthru_response.json<EOL>unenroll_list = []<EOL>if response_json and "<STR_LIT>" in response_json and response_json["<STR_LIT>"]and "<STR_LIT>" in response_json["<STR_LIT>"]:<EOL><INDENT>unenroll_list = response_json["<STR_LIT>"]["<STR_LIT>"]<EOL><DEDENT>changed = False<EOL>if unenroll:<EOL><INDENT>if course_url not in unenroll_list:<EOL><INDENT>unenroll_list.append(course_url)<EOL>changed = True<EOL><DEDENT><DEDENT>elif course_url in unenroll_list:<EOL><INDENT>unenroll_list.remove(course_url)<EOL>changed = True<EOL><DEDENT>if changed:<EOL><INDENT>sailthru_response = sailthru_client.api_post(<EOL>'<STR_LIT:user>', {'<STR_LIT:id>': email, '<STR_LIT:key>': '<STR_LIT:email>', '<STR_LIT>': {'<STR_LIT>': unenroll_list}})<EOL>if not sailthru_response.is_ok():<EOL><INDENT>error = sailthru_response.get_error()<EOL>logger.error("<STR_LIT>", error.get_message())<EOL>return not can_retry_sailthru_request(error)<EOL><DEDENT><DEDENT>return True<EOL><DEDENT>except SailthruClientError as exc:<EOL><INDENT>logger.exception("<STR_LIT>", email, text_type(exc))<EOL>return False<EOL><DEDENT>

Maintain a list of courses the user has unenrolled from in the Sailthru user record

    Arguments:
        sailthru_client (object): SailthruClient
        email (str): user's email address
        course_url (str): LMS url for course info page.
        unenroll (boolean): True if unenrolling, False if enrolling

    Returns:
        False if retryable error, else True


def _send_offer_assignment_notification_email(config, user_email, subject, email_body, site_code, task):

Handles sending offer assignment notification emails and retrying failed emails when appropriate.


def _get_course_content(course_id, course_url, sailthru_client, site_code, config):

<EOL>cache_key = "<STR_LIT>".format(site_code, course_url)<EOL>response = cache.get(cache_key)<EOL>if not response:<EOL><INDENT>try:<EOL><INDENT>sailthru_response = sailthru_client.api_get("<STR_LIT:content>", {"<STR_LIT:id>": course_url})<EOL>if not sailthru_response.is_ok():<EOL><INDENT>response = {}<EOL><DEDENT>else:<EOL><INDENT>response = sailthru_response.json<EOL>cache.set(cache_key, response, config.get('<STR_LIT>'))<EOL><DEDENT><DEDENT>except SailthruClientError:<EOL><INDENT>response = {}<EOL><DEDENT>if not response:<EOL><INDENT>logger.error('<STR_LIT>'<EOL>'<STR_LIT>')<EOL>response = _get_course_content_from_ecommerce(course_id, site_code=site_code)<EOL>if response:<EOL><INDENT>cache.set(cache_key, response, config.get('<STR_LIT>'))<EOL><DEDENT><DEDENT><DEDENT>return response<EOL>

Get course information using the Sailthru content api or from cache.

    If there is an error, just return with an empty response.

    Arguments:
        course_id (str): course key of the course
        course_url (str): LMS url for course info page.
        sailthru_client (object): SailthruClient
        site_code (str): site code
        config (dict): config options

    Returns:
        course information from Sailthru


api = get_ecommerce_client(site_code=site_code)<EOL>try:<EOL><INDENT>api_response = api.courses(course_id).get()<EOL><DEDENT>except Exception:  <EOL><INDENT>logger.exception(<EOL>'<STR_LIT>',<EOL>course_id,<EOL>exc_info=True<EOL>)<EOL>return {}<EOL><DEDENT>return {<EOL>'<STR_LIT:title>': api_response.get('<STR_LIT:name>'),<EOL>'<STR_LIT>': api_response.get('<STR_LIT>')<EOL>}<EOL>

Get course information using the Ecommerce course api.

In case of error returns empty response.
Arguments:
    course_id (str): course key of the course
    site_code (str): site code

Returns:
    course information from Ecommerce


<EOL>config = get_sailthru_configuration(site_code)<EOL>if not config.get('<STR_LIT>'):<EOL><INDENT>msg = '<STR_LIT>'.format(site_code)<EOL>log.debug(msg)<EOL>raise SailthruNotEnabled(msg)<EOL><DEDENT>key = config.get('<STR_LIT>')<EOL>secret = config.get('<STR_LIT>')<EOL>if not (key and secret):<EOL><INDENT>msg = '<STR_LIT>'.format(site_code)<EOL>log.error(msg)<EOL>raise ConfigurationError(msg)<EOL><DEDENT>return SailthruClient(key, secret)<EOL>

Returns a Sailthru client for the specified site.

Args:
    site_code (str): Site for which the client should be configured.

Returns:
    SailthruClient

Raises:
    SailthruNotEnabled: If Sailthru is not enabled for the specified site.
    ConfigurationError: If either the Sailthru API key or secret are not set for the site.


config = get_configuration('<STR_LIT>', site_code=site_code)<EOL>return config<EOL>

Returns the Sailthru configuration for the specified site.


def _retry_order(self, exception, max_fulfillment_retries, order_number):

retries = self.request.retries<EOL>if retries == max_fulfillment_retries:<EOL><INDENT>logger.exception('<STR_LIT>', order_number)<EOL><DEDENT>else:<EOL><INDENT>logger.warning('<STR_LIT>', order_number)<EOL><DEDENT>countdown = <NUM_LIT:2> ** retries<EOL>raise self.retry(exc=exception, countdown=countdown, max_retries=max_fulfillment_retries)<EOL>

Retry with exponential backoff until fulfillment
succeeds or the retry limit is reached. If the retry limit is exceeded,
the exception is re-raised.


filename = os.environ.get(variable)<EOL>if filename is None:<EOL><INDENT>msg = '<STR_LIT>'.format(variable)<EOL>raise EnvironmentError(msg)<EOL><DEDENT>return filename<EOL>

Get the name of the file containing configuration overrides
from the provided environment variable.


<EOL>if local_loglevel not in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>local_loglevel = '<STR_LIT>'<EOL><DEDENT>hostname = platform.node().split('<STR_LIT:.>')[<NUM_LIT:0>]<EOL>syslog_format = (<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>).format(<EOL>service_variant=service_variant,<EOL>logging_env=logging_env, hostname=hostname<EOL>)<EOL>if debug:<EOL><INDENT>handlers = ['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>handlers = ['<STR_LIT>']<EOL><DEDENT>logger_config = {<EOL>'<STR_LIT:version>': <NUM_LIT:1>,<EOL>'<STR_LIT>': False,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>'<STR_LIT>',<EOL>},<EOL>'<STR_LIT>': {'<STR_LIT>': syslog_format},<EOL>'<STR_LIT>': {'<STR_LIT>': '<STR_LIT>'},<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': '<STR_LIT>' if debug else '<STR_LIT>',<EOL>'<STR_LIT:class>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': sys.stdout,<EOL>},<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': handlers,<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': True<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': handlers,<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': False<EOL>},<EOL>}<EOL>}<EOL>if dev_env:<EOL><INDENT>edx_file_loc = os.path.join(log_dir, edx_filename)<EOL>logger_config['<STR_LIT>'].update({<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:class>': '<STR_LIT>',<EOL>'<STR_LIT>': local_loglevel,<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT:filename>': edx_file_loc,<EOL>'<STR_LIT>': <NUM_LIT> * <NUM_LIT> * <NUM_LIT:2>,<EOL>'<STR_LIT>': <NUM_LIT:5>,<EOL>},<EOL>})<EOL><DEDENT>else:<EOL><INDENT>logger_config['<STR_LIT>'].update({<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': local_loglevel,<EOL>'<STR_LIT:class>': '<STR_LIT>',<EOL>'<STR_LIT:address>': '<STR_LIT>' if sys.platform == '<STR_LIT>' else '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': SysLogHandler.LOG_LOCAL0,<EOL>},<EOL>})<EOL><DEDENT>return logger_config<EOL>

Returns a dictionary containing logging configuration.

If dev_env is True, logging will not be done via local rsyslogd.
Instead, application logs will be dropped into log_dir. 'edx_filename'
is ignored unless dev_env is True.


payload = {<EOL>'<STR_LIT:list>': (None, gene_list),<EOL>'<STR_LIT:description>': (None, self.descriptions)<EOL>}<EOL>response = requests.post(url, files=payload)<EOL>if not response.ok:<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT>sleep(<NUM_LIT:1>)<EOL>job_id = json.loads(response.text)<EOL>return job_id<EOL>


<EOL>if os.path.isfile(self.background):<EOL><INDENT>with open(self.background) as b:<EOL><INDENT>bg2 = b.readlines() <EOL><DEDENT>bg = [g.strip() for g in bg2]  <EOL>return set(bg)<EOL><DEDENT>DB_FILE = resource_filename("<STR_LIT>", "<STR_LIT>".format(self.background))<EOL>filename = os.path.join(DEFAULT_CACHE_PATH, "<STR_LIT>".format(self.background))  <EOL>if os.path.exists(filename):<EOL><INDENT>df = pd.read_csv(filename,sep="<STR_LIT:\t>")<EOL><DEDENT>elif os.path.exists(DB_FILE):<EOL><INDENT>df = pd.read_csv(DB_FILE,sep="<STR_LIT:\t>")<EOL><DEDENT>else:<EOL><INDENT>self._logger.warning("<STR_LIT>"%self.background)<EOL>bm = Biomart()<EOL>df = bm.query(dataset=self.background)<EOL>df.dropna(subset=['<STR_LIT>'], inplace=True)<EOL><DEDENT>self._logger.info("<STR_LIT>")<EOL>df.dropna(subset=['<STR_LIT>'], inplace=True)     <EOL>if self._isezid:<EOL><INDENT>bg = df['<STR_LIT>'].astype(int)<EOL><DEDENT>else:<EOL><INDENT>bg = df['<STR_LIT>']<EOL><DEDENT>return set(bg)<EOL>


enrichr_library = self.get_libraries()<EOL>if isinstance(self.gene_sets, list):<EOL><INDENT>gss = self.gene_sets<EOL><DEDENT>elif isinstance(self.gene_sets, str):<EOL><INDENT>gss = [ g.strip() for g in self.gene_sets.strip().split("<STR_LIT:U+002C>") ]<EOL><DEDENT>elif isinstance(self.gene_sets, dict):<EOL><INDENT>gss = [self.gene_sets]<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>gss_exist = [] <EOL>for g in gss:<EOL><INDENT>if isinstance(g, dict): <EOL><INDENT>gss_exist.append(g)<EOL>continue<EOL><DEDENT>if isinstance(g, str): <EOL><INDENT>if g in enrichr_library: <EOL><INDENT>gss_exist.append(g)<EOL>continue<EOL><DEDENT>if g.lower().endswith("<STR_LIT>") and os.path.exists(g):<EOL><INDENT>self._logger.info("<STR_LIT>"%g)<EOL>with open(g) as genesets:<EOL><INDENT>g_dict = { line.strip().split("<STR_LIT:\t>")[<NUM_LIT:0>]: line.strip().split("<STR_LIT:\t>")[<NUM_LIT:2>:]<EOL>for line in genesets.readlines() }<EOL><DEDENT>gss_exist.append(g_dict)<EOL><DEDENT><DEDENT><DEDENT>return gss_exist<EOL>


def gsea_compute(data, gmt, n, weighted_score_type, permutation_type,<EOL>method, pheno_pos, pheno_neg, classes, ascending,<EOL>processes=<NUM_LIT:1>, seed=None, single=False, scale=False):

w = weighted_score_type<EOL>subsets = sorted(gmt.keys())<EOL>es = []<EOL>RES=[]<EOL>hit_ind=[]<EOL>esnull = [ [] for a in range(len(subsets)) ]<EOL>logging.debug("<STR_LIT>")<EOL>if permutation_type == "<STR_LIT>":<EOL><INDENT>logging.debug("<STR_LIT>")<EOL>rs = np.random.RandomState(seed)<EOL>genes_mat, cor_mat = ranking_metric_tensor(exprs=data, method=method,<EOL>permutation_num=n,<EOL>pos=pheno_pos, neg=pheno_neg,<EOL>classes=classes,<EOL>ascending=ascending, rs=rs)<EOL>logging.debug("<STR_LIT>")<EOL>es, esnull, hit_ind, RES = enrichment_score_tensor(gene_mat=genes_mat,<EOL>cor_mat=cor_mat,<EOL>gene_sets=gmt,<EOL>weighted_score_type=w,<EOL>nperm=n, rs=rs,<EOL>single=False, scale=False,)<EOL><DEDENT>else:<EOL><INDENT>gl, cor_vec = data.index.values, data.values<EOL>logging.debug("<STR_LIT>")<EOL>temp_esnu=[]<EOL>pool_esnu = Pool(processes=processes)<EOL>for subset in subsets:<EOL><INDENT>rs = np.random.RandomState(seed)<EOL>temp_esnu.append(pool_esnu.apply_async(enrichment_score,<EOL>args=(gl, cor_vec, gmt.get(subset), w,<EOL>n, rs, single, scale)))<EOL><DEDENT>pool_esnu.close()<EOL>pool_esnu.join()<EOL>for si, temp in enumerate(temp_esnu):<EOL><INDENT>e, enu, hit, rune = temp.get()<EOL>esnull[si] = enu<EOL>es.append(e)<EOL>RES.append(rune)<EOL>hit_ind.append(hit)<EOL><DEDENT><DEDENT>return gsea_significance(es, esnull), hit_ind, RES, subsets<EOL>

compute enrichment scores and enrichment nulls.

        :param data: preprocessed expression dataframe or a pre-ranked file if prerank=True.
        :param dict gmt: all gene sets in .gmt file. need to call load_gmt() to get results.
        :param int n: permutation number. default: 1000.
        :param str method: ranking_metric method. see above.
        :param str pheno_pos: one of labels of phenotype's names.
        :param str pheno_neg: one of labels of phenotype's names.
        :param list classes: a list of phenotype labels, to specify which column of dataframe belongs to what category of phenotype.
        :param float weighted_score_type: default:1
        :param bool ascending: sorting order of rankings. Default: False.
        :param seed: random seed. Default: np.random.RandomState()
        :param bool scale: if true, scale es by gene number.

        :return: a tuple contains::

                | zipped results of es, nes, pval, fdr.
                | nested list of hit indices of input gene_list.
                | nested list of ranked enrichment score of each input gene_sets.
                | list of enriched terms


def gsea_compute_tensor(data, gmt, n, weighted_score_type, permutation_type,<EOL>method, pheno_pos, pheno_neg, classes, ascending,<EOL>processes=<NUM_LIT:1>, seed=None, single=False, scale=False):

w = weighted_score_type<EOL>subsets = sorted(gmt.keys())<EOL>rs = np.random.RandomState(seed)<EOL>genes_mat, cor_mat = data.index.values, data.values<EOL>base = <NUM_LIT:5> if data.shape[<NUM_LIT:0>] >= <NUM_LIT> else <NUM_LIT:10><EOL>block = ceil(len(subsets) / base)<EOL>if permutation_type == "<STR_LIT>":<EOL><INDENT>logging.debug("<STR_LIT>")<EOL>genes_ind = []<EOL>cor_mat = []<EOL>temp_rnk = []<EOL>pool_rnk = Pool(processes=processes)<EOL>i=<NUM_LIT:1><EOL>while i <= block:<EOL><INDENT>rs = np.random.RandomState(seed)<EOL>temp_rnk.append(pool_rnk.apply_async(ranking_metric_tensor,<EOL>args=(data, method, base, pheno_pos, pheno_neg, classes,<EOL>ascending, rs)))<EOL>i +=<NUM_LIT:1><EOL><DEDENT>pool_rnk.close()<EOL>pool_rnk.join()<EOL>for k, temp in enumerate(temp_rnk):<EOL><INDENT>gi, cor = temp.get()<EOL>if k+<NUM_LIT:1> == block:<EOL><INDENT>genes_ind.append(gi)<EOL>cor_mat.append(cor)<EOL><DEDENT>else:<EOL><INDENT>genes_ind.append(gi[:-<NUM_LIT:1>])<EOL>cor_mat.append(cor[:-<NUM_LIT:1>])<EOL><DEDENT><DEDENT>genes_ind, cor_mat = np.vstack(genes_ind), np.vstack(cor_mat)<EOL>genes_mat = (data.index.values, genes_ind)<EOL><DEDENT>logging.debug("<STR_LIT>")<EOL>es = []<EOL>RES = []<EOL>hit_ind = []<EOL>esnull = []<EOL>temp_esnu = []<EOL>pool_esnu = Pool(processes=processes)<EOL>i, m = <NUM_LIT:1>, <NUM_LIT:0><EOL>while i <= block:<EOL><INDENT>rs = np.random.RandomState(seed)<EOL>gmtrim = {k: gmt.get(k) for k in subsets[m:base * i]}<EOL>temp_esnu.append(pool_esnu.apply_async(enrichment_score_tensor,<EOL>args=(genes_mat, cor_mat,<EOL>gmtrim, w, n, rs,<EOL>single, scale)))<EOL>m = base * i<EOL>i += <NUM_LIT:1><EOL><DEDENT>pool_esnu.close()<EOL>pool_esnu.join()<EOL>for si, temp in enumerate(temp_esnu):<EOL><INDENT>e, enu, hit, rune = temp.get()<EOL>esnull.append(enu)<EOL>es.append(e)<EOL>RES.append(rune)<EOL>hit_ind += hit<EOL><DEDENT>es, esnull, RES = np.hstack(es), np.vstack(esnull), np.vstack(RES)<EOL>return gsea_significance(es, esnull), hit_ind, RES, subsets<EOL>


def ranking_metric_tensor(exprs, method, permutation_num, pos, neg, classes,<EOL>ascending, rs=np.random.RandomState()):

<EOL>G, S = exprs.shape<EOL>expr_mat = exprs.values.T<EOL>perm_cor_tensor = np.tile(expr_mat, (permutation_num+<NUM_LIT:1>,<NUM_LIT:1>,<NUM_LIT:1>))<EOL>for arr in perm_cor_tensor[:-<NUM_LIT:1>]: rs.shuffle(arr)<EOL>classes = np.array(classes)<EOL>pos = classes == pos<EOL>neg = classes == neg<EOL>pos_cor_mean = perm_cor_tensor[:,pos,:].mean(axis=<NUM_LIT:1>)<EOL>neg_cor_mean = perm_cor_tensor[:,neg,:].mean(axis=<NUM_LIT:1>)<EOL>pos_cor_std = perm_cor_tensor[:,pos,:].std(axis=<NUM_LIT:1>, ddof=<NUM_LIT:1>)<EOL>neg_cor_std = perm_cor_tensor[:,neg,:].std(axis=<NUM_LIT:1>, ddof=<NUM_LIT:1>)<EOL>if method == '<STR_LIT>':<EOL><INDENT>cor_mat = (pos_cor_mean - neg_cor_mean)/(pos_cor_std + neg_cor_std)<EOL><DEDENT>elif method == '<STR_LIT>':<EOL><INDENT>denom = <NUM_LIT:1.0>/G<EOL>cor_mat = (pos_cor_mean - neg_cor_mean)/ np.sqrt(denom*pos_cor_std**<NUM_LIT:2> + denom*neg_cor_std**<NUM_LIT:2>)<EOL><DEDENT>elif method == '<STR_LIT>':<EOL><INDENT>cor_mat = pos_cor_mean / neg_cor_mean<EOL><DEDENT>elif method == '<STR_LIT>':<EOL><INDENT>cor_mat  = pos_cor_mean - neg_cor_mean<EOL><DEDENT>elif method == '<STR_LIT>':<EOL><INDENT>cor_mat  =  np.log2(pos_cor_mean / neg_cor_mean)<EOL><DEDENT>else:<EOL><INDENT>logging.error("<STR_LIT>")<EOL>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>cor_mat_ind = cor_mat.argsort()<EOL>cor_mat.sort()<EOL>if ascending: return cor_mat_ind, cor_mat<EOL>return cor_mat_ind[:, ::-<NUM_LIT:1>], cor_mat[:, ::-<NUM_LIT:1>]<EOL>

Build shuffled ranking matrix when permutation_type eq to phenotype.

       :param exprs:   gene_expression DataFrame, gene_name indexed.
       :param str method:  calculate correlation or ranking. methods including:
                           1. 'signal_to_noise'.
                           2. 't_test'.
                           3. 'ratio_of_classes' (also referred to as fold change).
                           4. 'diff_of_classes'.
                           5. 'log2_ratio_of_classes'.
       :param int permuation_num: how many times of classes is being shuffled
       :param str pos: one of labels of phenotype's names.
       :param str neg: one of labels of phenotype's names.
       :param list classes:  a list of phenotype labels, to specify which column of
                             dataframe belongs to what class of phenotype.
       :param bool ascending:  bool. Sort ascending vs. descending.

       :return:
                returns two 2d ndarray with shape (nperm, gene_num).

                | cor_mat_indices: the indices of sorted and permutated (exclude last row) ranking matrix.
                | cor_mat: sorted and permutated (exclude last row) ranking matrix.


if isinstance(cls, list) :<EOL><INDENT>classes = cls<EOL>sample_name= unique(classes)<EOL><DEDENT>elif isinstance(cls, str) :<EOL><INDENT>with open(cls) as c:<EOL><INDENT>file = c.readlines()<EOL><DEDENT>classes = file[<NUM_LIT:2>].strip('<STR_LIT:\n>').split("<STR_LIT:U+0020>")<EOL>sample_name = file[<NUM_LIT:1>].lstrip("<STR_LIT>").strip('<STR_LIT:\n>').split("<STR_LIT:U+0020>")<EOL><DEDENT>else:<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT>return sample_name[<NUM_LIT:0>], sample_name[<NUM_LIT:1>], classes<EOL>

Extract class(phenotype) name from .cls file.

    :param cls: the a class list instance or .cls file which is identical to GSEA input .
    :return: phenotype name and a list of class vector.


<EOL>if database not in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>sys.stderr.write("""<STR_LIT>""")<EOL>return <EOL><DEDENT>if database in ['<STR_LIT>', '<STR_LIT>']: database='<STR_LIT>'<EOL>lib_url='<STR_LIT>'%database<EOL>libs_json = json.loads(requests.get(lib_url).text)<EOL>libs = [lib['<STR_LIT>'] for lib in libs_json['<STR_LIT>']]<EOL>return sorted(libs)<EOL>

return enrichr active enrichr library name.
    :param str database: Select one from { 'Human', 'Mouse', 'Yeast', 'Fly', 'Fish', 'Worm' }


def heatmap(df, z_score=None, title='<STR_LIT>', figsize=(<NUM_LIT:5>,<NUM_LIT:5>), cmap='<STR_LIT>', <EOL>xticklabels=True, yticklabels=True, ofname=None, **kwargs):

df = zscore(df, axis=z_score)<EOL>df = df.iloc[::-<NUM_LIT:1>]<EOL>ny, nx = df.shape<EOL>xticks = np.arange(<NUM_LIT:0>, nx, <NUM_LIT:1>) + <NUM_LIT><EOL>yticks = np.arange(<NUM_LIT:0>, ny, <NUM_LIT:1>) + <NUM_LIT><EOL>if hasattr(sys, '<STR_LIT>') and (ofname is None): <EOL><INDENT>fig = plt.figure(figsize=figsize)<EOL><DEDENT>else:<EOL><INDENT>fig = Figure(figsize=figsize)<EOL>canvas = FigureCanvas(fig)<EOL><DEDENT>ax = fig.add_subplot(<NUM_LIT>)<EOL>vmin = np.percentile(df.min(), <NUM_LIT:2>)<EOL>vmax =  np.percentile(df.max(), <NUM_LIT>)<EOL>matrix = ax.pcolormesh(df.values, cmap=cmap, vmin=vmin, vmax=vmax)<EOL>ax.set_ylim([<NUM_LIT:0>,len(df)])<EOL>ax.set(xticks=xticks, yticks=yticks)<EOL>ax.set_xticklabels(df.columns.values if xticklabels else '<STR_LIT>', fontsize=<NUM_LIT>, rotation=<NUM_LIT>)<EOL>ax.set_yticklabels(df.index.values if yticklabels else '<STR_LIT>',  fontsize=<NUM_LIT>)<EOL>ax.set_title("<STR_LIT>"%title, fontsize=<NUM_LIT:20>)<EOL>ax.tick_params(axis='<STR_LIT>', which='<STR_LIT>', bottom=False, top=False,<EOL>right=False, left=False)<EOL>cbar = colorbar(matrix)<EOL>cbar.ax.tick_params(axis='<STR_LIT>', which='<STR_LIT>', bottom=False, top=False,<EOL>right=False, left=False)<EOL>for side in ["<STR_LIT>", "<STR_LIT:right>", "<STR_LIT:left>", "<STR_LIT>"]:<EOL><INDENT>ax.spines[side].set_visible(False)<EOL>cbar.ax.spines[side].set_visible(False)<EOL><DEDENT>if ofname is not None: <EOL><INDENT>fig.savefig(ofname, bbox_inches='<STR_LIT>', dpi=<NUM_LIT>)<EOL><DEDENT>return<EOL>

Visualize the dataframe.

    :param df: DataFrame from expression table.
    :param z_score: z_score axis{0, 1}. If None, don't normalize data.
    :param title: gene set name.
    :param outdir: path to save heatmap.
    :param figsize: heatmap figsize.
    :param cmap: matplotlib colormap.
    :param ofname: output file name. If None, don't save figure


def dotplot(df, column='<STR_LIT>', title='<STR_LIT>', cutoff=<NUM_LIT>, top_term=<NUM_LIT:10>, <EOL>sizes=None, norm=None, legend=True, figsize=(<NUM_LIT:6>, <NUM_LIT>), <EOL>cmap='<STR_LIT>', ofname=None, **kwargs):

colname = column    <EOL>if colname in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>can_be_coerced = df[colname].map(isfloat)<EOL>if np.sum(~can_be_coerced) > <NUM_LIT:0>:<EOL><INDENT>raise ValueError('<STR_LIT>'%colname)<EOL><DEDENT>else:<EOL><INDENT>df.loc[:, colname] = df[colname].map(float)<EOL><DEDENT>df = df[df[colname] <= cutoff]<EOL>if len(df) < <NUM_LIT:1>: <EOL><INDENT>msg = "<STR_LIT>"%cutoff<EOL>return msg<EOL><DEDENT>df = df.assign(logAP=lambda x: - x[colname].apply(np.log10))<EOL>colname='<STR_LIT>'<EOL><DEDENT>df = df.sort_values(by=colname).iloc[-top_term:,:]<EOL>temp = df['<STR_LIT>'].str.split("<STR_LIT:/>", expand=True).astype(int)<EOL>df = df.assign(Hits=temp.iloc[:,<NUM_LIT:0>], Background=temp.iloc[:,<NUM_LIT:1>])<EOL>df = df.assign(Hits_ratio=lambda x:x.Hits / x.Background)<EOL>x = df.loc[:, colname].values<EOL>combined_score = df['<STR_LIT>'].round().astype('<STR_LIT:int>')<EOL>y = [i for i in range(<NUM_LIT:0>,len(df))]<EOL>ylabels = df['<STR_LIT>'].values<EOL>levels = numbers = np.sort(df.Hits.unique())<EOL>if norm is None:<EOL><INDENT>norm = Normalize()<EOL><DEDENT>elif isinstance(norm, tuple):<EOL><INDENT>norm = Normalize(*norm)<EOL><DEDENT>elif not isinstance(norm, Normalize):<EOL><INDENT>err = ("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>raise ValueError(err)<EOL><DEDENT>min_width, max_width = np.r_[<NUM_LIT:20>, <NUM_LIT:100>] * plt.rcParams["<STR_LIT>"]<EOL>norm.clip = True<EOL>if not norm.scaled():<EOL><INDENT>norm(np.asarray(numbers))<EOL><DEDENT>size_limits = norm.vmin, norm.vmax<EOL>scl = norm(numbers)<EOL>widths = np.asarray(min_width + scl * (max_width - min_width))<EOL>if scl.mask.any():<EOL><INDENT>widths[scl.mask] = <NUM_LIT:0><EOL><DEDENT>sizes = dict(zip(levels, widths))<EOL>df['<STR_LIT>'] = df.Hits.map(sizes)<EOL>area = df['<STR_LIT>'].values<EOL>if hasattr(sys, '<STR_LIT>') and (ofname is None):<EOL><INDENT>fig, ax = plt.subplots(figsize=figsize)<EOL><DEDENT>else:<EOL><INDENT>fig = Figure(figsize=figsize)<EOL>canvas = FigureCanvas(fig)<EOL>ax = fig.add_subplot(<NUM_LIT>)<EOL><DEDENT>vmin = np.percentile(combined_score.min(), <NUM_LIT:2>)<EOL>vmax =  np.percentile(combined_score.max(), <NUM_LIT>)<EOL>sc = ax.scatter(x=x, y=y, s=area, edgecolors='<STR_LIT>', c=combined_score,<EOL>cmap=cmap, vmin=vmin, vmax=vmax)<EOL>if column in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>xlabel = "<STR_LIT>"%column<EOL><DEDENT>else:<EOL><INDENT>xlabel = column <EOL><DEDENT>ax.set_xlabel(xlabel, fontsize=<NUM_LIT>, fontweight='<STR_LIT>')<EOL>ax.yaxis.set_major_locator(plt.FixedLocator(y))<EOL>ax.yaxis.set_major_formatter(plt.FixedFormatter(ylabels))<EOL>ax.set_yticklabels(ylabels, fontsize=<NUM_LIT:16>)<EOL>ax.grid()<EOL>cax=fig.add_axes([<NUM_LIT>,<NUM_LIT>,<NUM_LIT>,<NUM_LIT>])<EOL>cbar = fig.colorbar(sc, cax=cax,)<EOL>cbar.ax.tick_params(right=True)<EOL>cbar.ax.set_title('<STR_LIT>',loc='<STR_LIT:left>', fontsize=<NUM_LIT:12>)<EOL>if len(df) >= <NUM_LIT:3>:<EOL><INDENT>idx = [area.argmax(), np.abs(area - area.mean()).argmin(), area.argmin()]<EOL>idx = unique(idx)<EOL><DEDENT>else:<EOL><INDENT>idx = df.index.values<EOL><DEDENT>label = df.iloc[idx, df.columns.get_loc('<STR_LIT>')]<EOL>if legend:<EOL><INDENT>handles, _ = ax.get_legend_handles_labels()<EOL>legend_markers = []<EOL>for ix in idx: <EOL><INDENT>legend_markers.append(ax.scatter([],[], s=area[ix], c='<STR_LIT:b>'))<EOL><DEDENT>ax.legend(legend_markers, label, title='<STR_LIT>')<EOL><DEDENT>ax.set_title(title, fontsize=<NUM_LIT:20>, fontweight='<STR_LIT>')<EOL>if ofname is not None: <EOL><INDENT>fig.savefig(ofname, bbox_inches='<STR_LIT>', dpi=<NUM_LIT>)<EOL>return<EOL><DEDENT>return ax<EOL>

Visualize enrichr results.

    :param df: GSEApy DataFrame results.
    :param column: which column of DataFrame to show. Default: Adjusted P-value
    :param title: figure title
    :param cutoff: p-adjust cut-off.
    :param top_term: number of enriched terms to show.
    :param ascending: bool, the order of y axis.
    :param sizes: tuple, (min, max) scatter size. Not functional for now
    :param norm: maplotlib.colors.Normalize object.
    :param legend: bool, whether to show legend.
    :param figsize: tuple, figure size.
    :param cmap: matplotlib colormap
    :param ofname: output file name. If None, don't save figure


for loc, spine in ax.spines.items():<EOL><INDENT>if loc in spines:<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>spine.set_color('<STR_LIT:none>')  <EOL><DEDENT><DEDENT>if '<STR_LIT:left>' in spines:<EOL><INDENT>ax.yaxis.set_ticks_position('<STR_LIT:left>')<EOL><DEDENT>else:<EOL><INDENT>ax.yaxis.set_ticks([])<EOL><DEDENT>if '<STR_LIT>' in spines:<EOL><INDENT>ax.xaxis.set_ticks_position('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>ax.xaxis.set_ticks([])<EOL><DEDENT>

function for removing spines and ticks.

    :param ax: axes object
    :param spines: a list of spines names to keep. e.g [left, right, top, bottom]
                    if spines = []. remove all spines and ticks.


if axis is None:<EOL><INDENT>return data2d<EOL><DEDENT>assert axis in [<NUM_LIT:0>,<NUM_LIT:1>]<EOL>z_scored = data2d.apply(lambda x: (x-x.mean())/x.std(ddof=<NUM_LIT:1>), <EOL>axis=operator.xor(<NUM_LIT:1>, axis))<EOL>return z_scored<EOL>

Standardize the mean and variance of the data axis Parameters.

    :param data2d: DataFrame to normalize.
    :param axis: int, Which axis to normalize across. If 0, normalize across rows,
                  if 1, normalize across columns. If None, don't change data

    :Returns: Normalized DataFrame. Normalized data with a mean of 0 and variance of 1
              across the specified axis.


def runSamplesPermu(self, df, gmt=None):

assert self.min_size <= self.max_size<EOL>mkdirs(self.outdir)<EOL>self.resultsOnSamples = OrderedDict()<EOL>outdir = self.outdir<EOL>for name, ser in df.iteritems():<EOL><INDENT>self.outdir = os.path.join(outdir, str(name))<EOL>self._logger.info("<STR_LIT>" % name)<EOL>mkdirs(self.outdir)<EOL>dat2 = ser.sort_values(ascending=self.ascending)<EOL>gsea_results, hit_ind,rank_ES, subsets = gsea_compute(data=dat2, n=self.permutation_num, gmt=gmt,<EOL>weighted_score_type=self.weighted_score_type,<EOL>permutation_type='<STR_LIT>', method=None,<EOL>pheno_pos='<STR_LIT>', pheno_neg='<STR_LIT>',<EOL>classes=None, ascending=self.ascending,<EOL>processes=self._processes,<EOL>seed=self.seed, single=True, scale=self.scale)<EOL>res_zip = zip(subsets, list(gsea_results), hit_ind, rank_ES)<EOL>self._save_results(zipdata=res_zip, outdir=self.outdir, module=self.module,<EOL>gmt=gmt, rank_metric=dat2, permutation_type="<STR_LIT>")<EOL>self.resultsOnSamples[name] = self.res2d.es<EOL>if self._noplot: continue<EOL>self._logger.info("<STR_LIT>" % name)<EOL>self._plotting(rank_metric=dat2, results=self.results,<EOL>graph_num=self.graph_num, outdir=self.outdir,<EOL>figsize=self.figsize, format=self.format)<EOL><DEDENT>self._save(outdir)<EOL>return<EOL>

Single Sample GSEA workflow with permutation procedure


self._logger.info("<STR_LIT>")<EOL>data = self.load_data()<EOL>normdat = self.norm_samples(data)<EOL>gmt = self.load_gmt(gene_list=normdat.index.values, gmt=self.gene_sets)<EOL>self._logger.info("<STR_LIT>"% len(gmt))<EOL>self._set_cores()<EOL>self._logger.info("<STR_LIT>")<EOL>if self.permutation_num == <NUM_LIT:0> :<EOL><INDENT>self.runSamples(df=normdat, gmt=gmt)<EOL><DEDENT>else:<EOL><INDENT>self._logger.warning("<STR_LIT>")<EOL>self.runSamplesPermu(df=normdat, gmt=gmt)<EOL><DEDENT>if self._outdir is None:<EOL><INDENT>self._tmpdir.cleanup()<EOL><DEDENT>


if gmt.lower().endswith("<STR_LIT>"):<EOL><INDENT>with open(gmt) as genesets:<EOL><INDENT>genesets_dict = { line.strip().split("<STR_LIT:\t>")[<NUM_LIT:0>]: line.strip().split("<STR_LIT:\t>")[<NUM_LIT:2>:]<EOL>for line in genesets.readlines()}<EOL><DEDENT>return genesets_dict<EOL><DEDENT>elif gmt in DEFAULT_LIBRARY:<EOL><INDENT>pass<EOL><DEDENT>elif gmt in self.get_libraries():<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>self._logger.error("<STR_LIT>"%gmt)<EOL>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>tmpname = "<STR_LIT>" + gmt + "<STR_LIT>"<EOL>tempath = os.path.join(DEFAULT_CACHE_PATH, tmpname)<EOL>if os.path.isfile(tempath):<EOL><INDENT>self._logger.info("<STR_LIT>"%DEFAULT_CACHE_PATH)<EOL>return self.parse_gmt(tempath)<EOL><DEDENT>else:<EOL><INDENT>return self._download_libraries(gmt)<EOL><DEDENT>


def _heatmat(self, df, classes, pheno_pos, pheno_neg):

width = len(classes) if len(classes) >= <NUM_LIT:6> else  <NUM_LIT:5><EOL>cls_booA =list(map(lambda x: True if x == pheno_pos else False, classes))<EOL>cls_booB =list(map(lambda x: True if x == pheno_neg else False, classes))<EOL>datA = df.loc[:, cls_booA]<EOL>datB = df.loc[:, cls_booB]<EOL>datAB=pd.concat([datA,datB], axis=<NUM_LIT:1>)<EOL>self._width = width<EOL>self.heatmat = datAB<EOL>return<EOL>


def prerank(rnk, gene_sets, outdir='<STR_LIT>', pheno_pos='<STR_LIT>', pheno_neg='<STR_LIT>',<EOL>min_size=<NUM_LIT:15>, max_size=<NUM_LIT>, permutation_num=<NUM_LIT:1000>, weighted_score_type=<NUM_LIT:1>,<EOL>ascending=False, processes=<NUM_LIT:1>, figsize=(<NUM_LIT>,<NUM_LIT:6>), format='<STR_LIT>',<EOL>graph_num=<NUM_LIT:20>, no_plot=False, seed=None, verbose=False):

pre = Prerank(rnk, gene_sets, outdir, pheno_pos, pheno_neg,<EOL>min_size, max_size, permutation_num, weighted_score_type,<EOL>ascending, processes, figsize, format, graph_num, no_plot, seed, verbose)<EOL>pre.run()<EOL>return pre<EOL>

Run Gene Set Enrichment Analysis with pre-ranked correlation defined by user.

    :param rnk: pre-ranked correlation table or pandas DataFrame. Same input with ``GSEA`` .rnk file.
    :param gene_sets: Enrichr Library name or .gmt gene sets file or dict of gene sets. Same input with GSEA.
    :param outdir: results output directory.
    :param int permutation_num: Number of permutations for significance computation. Default: 1000.
    :param int min_size: Minimum allowed number of genes from gene set also the data set. Default: 15.
    :param int max_size: Maximum allowed number of genes from gene set also the data set. Defaults: 500.
    :param str weighted_score_type: Refer to :func:`algorithm.enrichment_score`. Default:1.
    :param bool ascending: Sorting order of rankings. Default: False.
    :param int processes: Number of Processes you are going to use. Default: 1.
    :param list figsize: Matplotlib figsize, accept a tuple or list, e.g. [width,height]. Default: [6.5,6].
    :param str format: Matplotlib figure format. Default: 'pdf'.
    :param int graph_num: Plot graphs for top sets of each phenotype.
    :param bool no_plot: If equals to True, no figure will be drawn. Default: False.
    :param seed: Random seed. expect an integer. Default:None.
    :param bool verbose: Bool, increase output verbosity, print out progress of your job, Default: False.

    :return: Return a Prerank obj. All results store to  a dictionary, obj.results,
             where contains::

                 | {es: enrichment score,
                 |  nes: normalized enrichment score,
                 |  p: P-value,
                 |  fdr: FDR,
                 |  size: gene set size,
                 |  matched_size: genes matched to the data,
                 |  genes: gene names from the data set
                 |  ledge_genes: leading edge genes}


argparser_replot = subparsers.add_parser("<STR_LIT>", help="<STR_LIT>")<EOL>group_replot = argparser_replot.add_argument_group("<STR_LIT>")<EOL>group_replot.add_argument("<STR_LIT>", "<STR_LIT>", action="<STR_LIT:store>", dest="<STR_LIT>", required=True, metavar='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>add_output_option(group_replot)<EOL>group_replot.add_argument("<STR_LIT>", "<STR_LIT>", action='<STR_LIT:store>', dest='<STR_LIT>', default=<NUM_LIT:1.0>, type=float, metavar='<STR_LIT:float>',<EOL>help='<STR_LIT>',)<EOL>return<EOL>


parser.add_argument("<STR_LIT>", "<STR_LIT>", dest="<STR_LIT>", type=str, default='<STR_LIT>',<EOL>metavar='<STR_LIT>', action="<STR_LIT:store>",<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", "<STR_LIT>", dest="<STR_LIT>", type=str, metavar='<STR_LIT>', action="<STR_LIT:store>",<EOL>choices=("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"), default="<STR_LIT>",<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", "<STR_LIT>", action='<STR_LIT:store>', nargs=<NUM_LIT:2>, dest='<STR_LIT>',<EOL>metavar=('<STR_LIT:width>', '<STR_LIT>'),type=float, default=(<NUM_LIT>, <NUM_LIT:6>),<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", dest = "<STR_LIT>", action="<STR_LIT:store>", type=int, default=<NUM_LIT:20>, metavar='<STR_LIT:int>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", action='<STR_LIT:store_true>', dest='<STR_LIT>', default=False,<EOL>help="<STR_LIT>"+"<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", "<STR_LIT>", action="<STR_LIT:store_true>", default=False, dest='<STR_LIT>',<EOL>help="<STR_LIT>", )<EOL>


argparser_enrichr = subparsers.add_parser("<STR_LIT>", help="<STR_LIT>")<EOL>enrichr_opt = argparser_enrichr.add_argument_group("<STR_LIT>")<EOL>enrichr_opt.add_argument("<STR_LIT>", "<STR_LIT>", action="<STR_LIT:store>", dest="<STR_LIT>", type=str, required=True, metavar='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>enrichr_opt.add_argument("<STR_LIT>", "<STR_LIT>", action="<STR_LIT:store>", dest="<STR_LIT>", type=str, required=True, metavar='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>enrichr_opt.add_argument("<STR_LIT>", "<STR_LIT>", action="<STR_LIT:store>", dest="<STR_LIT>", type=str, default='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>enrichr_opt.add_argument("<STR_LIT>", "<STR_LIT>", action="<STR_LIT:store>", dest="<STR_LIT>", type=str, default='<STR_LIT>', metavar='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>enrichr_opt.add_argument("<STR_LIT>", "<STR_LIT>", action="<STR_LIT:store>", dest="<STR_LIT>", metavar='<STR_LIT:float>', type=float, default=<NUM_LIT>,<EOL>help="<STR_LIT>")<EOL>enrichr_opt.add_argument("<STR_LIT>", "<STR_LIT>", action="<STR_LIT:store>", dest="<STR_LIT>", default='<STR_LIT>', metavar='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>enrichr_opt.add_argument("<STR_LIT>", "<STR_LIT>", dest="<STR_LIT>", action="<STR_LIT:store>", type=int, default=<NUM_LIT:10>, metavar='<STR_LIT:int>',<EOL>help="<STR_LIT>")<EOL>enrichr_output = argparser_enrichr.add_argument_group("<STR_LIT>")<EOL>add_output_option(enrichr_output)<EOL>return<EOL>

Add function 'enrichr' argument parsers.


argparser_gsea = subparsers.add_parser("<STR_LIT>", help="<STR_LIT>")<EOL>group_input = argparser_gsea.add_argument_group("<STR_LIT>")<EOL>group_input.add_argument("<STR_LIT>", "<STR_LIT>", dest="<STR_LIT:data>", action="<STR_LIT:store>", type=str, required=True,<EOL>help="<STR_LIT>")<EOL>group_input.add_argument("<STR_LIT:-c>", "<STR_LIT>", dest="<STR_LIT>", action="<STR_LIT:store>", type=str, required=True,<EOL>help="<STR_LIT>")<EOL>group_input.add_argument("<STR_LIT>", "<STR_LIT>", dest="<STR_LIT>", action="<STR_LIT:store>", type=str, required=True,<EOL>help="<STR_LIT>")<EOL>group_input.add_argument("<STR_LIT>", "<STR_LIT>", action="<STR_LIT:store>", dest="<STR_LIT:type>", type=str, metavar='<STR_LIT>',<EOL>choices=("<STR_LIT>", "<STR_LIT>"), default="<STR_LIT>",<EOL>help="<STR_LIT>")<EOL>group_output = argparser_gsea.add_argument_group("<STR_LIT>")<EOL>add_output_option(group_output)<EOL>group_opt = argparser_gsea.add_argument_group("<STR_LIT>")<EOL>group_opt.add_argument("<STR_LIT>", "<STR_LIT>", dest = "<STR_LIT:n>", action="<STR_LIT:store>", type=int, default=<NUM_LIT:1000>, metavar='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>group_opt.add_argument("<STR_LIT>",  dest="<STR_LIT>", action="<STR_LIT:store>", type=int, default=<NUM_LIT:15>, metavar='<STR_LIT:int>',<EOL>help="<STR_LIT>")<EOL>group_opt.add_argument("<STR_LIT>", dest = "<STR_LIT>", action="<STR_LIT:store>", type=int, default=<NUM_LIT>, metavar='<STR_LIT:int>',<EOL>help="<STR_LIT>")<EOL>group_opt.add_argument("<STR_LIT>", "<STR_LIT>", action='<STR_LIT:store>', dest='<STR_LIT>', default=<NUM_LIT:1.0>, type=float, metavar='<STR_LIT:float>',<EOL>help='<STR_LIT>',)<EOL>group_opt.add_argument("<STR_LIT>", "<STR_LIT>", action="<STR_LIT:store>", dest="<STR_LIT>", type=str, metavar='<STR_LIT>',<EOL>choices=("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"),<EOL>default="<STR_LIT>",<EOL>help="<STR_LIT>")<EOL>group_opt.add_argument("<STR_LIT>", "<STR_LIT>", action='<STR_LIT:store_true>', dest='<STR_LIT>', default=False,<EOL>help='<STR_LIT>')<EOL>group_opt.add_argument("<STR_LIT>", "<STR_LIT>", dest = "<STR_LIT>", action="<STR_LIT:store>", type=int, default=None, metavar='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>group_opt.add_argument("<STR_LIT>", "<STR_LIT>", dest = "<STR_LIT>", action="<STR_LIT:store>", type=int, default=<NUM_LIT:1>, metavar='<STR_LIT>',<EOL>help="<STR_LIT>")<EOL>return<EOL>

Add main function 'gsea' argument parsers.


s = requests.Session()<EOL>retries = Retry(total=num, backoff_factor=<NUM_LIT:0.1>,<EOL>status_forcelist=[<NUM_LIT>, <NUM_LIT>, <NUM_LIT>, <NUM_LIT>])<EOL>s.mount('<STR_LIT>', HTTPAdapter(max_retries=retries))<EOL>return s<EOL>

retry connection.

        define max tries num
        if the backoff_factor is 0.1, then sleep() will sleep for
        [0.1s, 0.2s, 0.4s, ...] between retries.
        It will also force a retry if the status code returned is 500, 502, 503 or 504.


seen = set()<EOL>seen_add = seen.add<EOL>"""<STR_LIT>"""<EOL>return [x for x in seq if x not in seen and not seen_add(x)]<EOL>

Remove duplicates from a list in Python while preserving order.

    :param seq: a python list object.
    :return: a list without duplicates while preserving order.


usage = "<STR_LIT>"<EOL>description = "<STR_LIT>"<EOL>argparser = argparse.ArgumentParser(<EOL>usage=usage, description=description)<EOL>argparser.add_argument(<EOL>'<STR_LIT>', type=argparse.FileType('<STR_LIT:r>'),<EOL>help="<STR_LIT>")<EOL>argparser.add_argument(<EOL>'<STR_LIT>', nargs='<STR_LIT:?>', type=argparse.FileType('<STR_LIT:w>'),<EOL>default=sys.stdout, help="<STR_LIT>"<EOL>"<STR_LIT>")<EOL>argparser.add_argument(<EOL>'<STR_LIT>', nargs="<STR_LIT:?>", const=str, help="<STR_LIT>")<EOL>args = argparser.parse_args()<EOL>return args<EOL>

Parses command line args using argparse library


return self.meter.one_minute_rate<EOL>


return self.histogram.min<EOL>

Returns the minimum amount of time spent in the operation.


return self.histogram.mean<EOL>

Returns the mean time spent in the operation.


return self.meter.fifteen_minute_rate<EOL>

Returns the fifteen-minute average rate.


return self.histogram.max<EOL>

Returns the maximum amount of time spent in the operation.


return self.m5_rate.rate<EOL>


return self.m1_rate.rate<EOL>


if self.counter.value == <NUM_LIT:0>:<EOL><INDENT>return <NUM_LIT:0.0><EOL><DEDENT>else:<EOL><INDENT>elapsed = time() - self.start_time<EOL>return self.counter.value / elapsed<EOL><DEDENT>

Returns the mean rate of the events since the start of the process.


if self.counter.value > <NUM_LIT:0>:<EOL><INDENT>return self.variance ** <NUM_LIT><EOL><DEDENT>return <NUM_LIT:0.0><EOL>


last = self.last.get_and_set(value)<EOL>if last <= value:<EOL><INDENT>value = value - last<EOL><DEDENT>super(Derive, self).mark(value)<EOL>

Record an event with the derive.

        :param value: counter value to record


def format_metric_string(self, name, value, m_type):

<EOL>template = '<STR_LIT>'<EOL>if self.prefix:<EOL><INDENT>name = "<STR_LIT>".format(prefix=self.prefix, m_name=name)<EOL><DEDENT>return template.format(name=name, value=value, m_type=m_type)<EOL>

Compose a statsd compatible string for a metric's measurement.


<EOL>self.log.debug("<STR_LIT>")<EOL>self.connected.clear()<EOL>self.reconnect_required.set()<EOL>if self.socket:<EOL><INDENT>self.socket.close()<EOL><DEDENT>

Issues a reconnection by setting the reconnect_required event.

        :return:


<EOL>self.log.debug("<STR_LIT>",<EOL>data)<EOL>self.pass_to_client('<STR_LIT:data>', data, ts)<EOL>

Handles data messages by passing them up to the client.

        :param data:
        :param ts:
        :return:


def __init__(self, *args, url=None, timeout=None, sslopt=None,<EOL>http_proxy_host=None, http_proxy_port=None, http_proxy_auth=None, http_no_proxy=None,<EOL>reconnect_interval=None, log_level=None, **kwargs):

<EOL>self.q = Queue()<EOL>self.socket = None<EOL>self.url = url if url else '<STR_LIT>'<EOL>self.sslopt = sslopt if sslopt else {}<EOL>self.http_proxy_host = http_proxy_host<EOL>self.http_proxy_port = http_proxy_port<EOL>self.http_proxy_auth = http_proxy_auth<EOL>self.http_no_proxy = http_no_proxy<EOL>self.channel_configs = OrderedDict()<EOL>self.connected = Event()<EOL>self.disconnect_called = Event()<EOL>self.reconnect_required = Event()<EOL>self.reconnect_interval = reconnect_interval if reconnect_interval else <NUM_LIT:10><EOL>self.paused = Event()<EOL>self.ping_timer = None<EOL>self.ping_interval = <NUM_LIT><EOL>self.connection_timer = None<EOL>self.connection_timeout = timeout if timeout else <NUM_LIT:10><EOL>self.pong_timer = None<EOL>self.pong_received = False<EOL>self.pong_timeout = <NUM_LIT:30><EOL>self.log = logging.getLogger(self.__module__)<EOL>if log_level == logging.DEBUG:<EOL><INDENT>websocket.enableTrace(True)<EOL><DEDENT>self.log.setLevel(level=log_level if log_level else logging.INFO)<EOL>Thread.__init__(self)<EOL>self.daemon = True<EOL>self.bitfinex_config = None<EOL>

Initialize a WebSocketConnection Instance.

        :param data_q: Queue(), connection to the Client Class
        :param args: args for Thread.__init__()
        :param url: websocket address, defaults to v2 websocket.
        :param http_proxy_host: proxy host name.
        :param http_proxy_port: http proxy port. If not set, set to 80.
        :param http_proxy_auth: http proxy auth information.
                                tuple of username and password.
        :param http_no_proxy: host names, which doesn't use proxy.
        :param timeout: timeout for connection; defaults to 10s
        :param reconnect_interval: interval at which to try reconnecting;
                                   defaults to 10s.
        :param log_level: logging level for the connection Logger. Defaults to
                          logging.INFO.
        :param kwargs: kwargs for Thread.__ini__()


<EOL>self.log.debug("<STR_LIT>")<EOL>self.pong_received = True<EOL>

Handle a pong response.

        :return:


def raise_exception():<EOL><INDENT>"""<STR_LIT>"""<EOL>self.log.error("<STR_LIT>", data['<STR_LIT:code>'], info_message[data['<STR_LIT:code>']])<EOL>raise ValueError("<STR_LIT>" % (data['<STR_LIT:code>'], info_message[data['<STR_LIT:code>']]))<EOL><DEDENT>if '<STR_LIT:code>' not in data and '<STR_LIT:version>' in data:<EOL><INDENT>self.log.info('<STR_LIT>', data['<STR_LIT:version>'])<EOL>return<EOL><DEDENT>info_message = {<NUM_LIT>: '<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>'<EOL>'<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>'<EOL>'<STR_LIT>',<EOL><NUM_LIT>: '<STR_LIT>'<EOL>'<STR_LIT>'}<EOL>codes = {<NUM_LIT>: self.reconnect, <NUM_LIT>: self._pause,<EOL><NUM_LIT>: self._unpause}<EOL>if '<STR_LIT:version>' in data:<EOL><INDENT>self.log.info("<STR_LIT>", data['<STR_LIT:version>'])<EOL>return<EOL><DEDENT>try:<EOL><INDENT>self.log.info(info_message[data['<STR_LIT:code>']])<EOL>codes[data['<STR_LIT:code>']]()<EOL><DEDENT>except KeyError as e:<EOL><INDENT>self.log.exception(e)<EOL>self.log.error("<STR_LIT>", data['<STR_LIT:code>'])<EOL>raise<EOL><DEDENT>

Handle INFO messages from the API and issues relevant actions.

:param data:
:param ts:


def send(self, api_key=None, secret=None, list_data=None, auth=False, **kwargs):

if auth:<EOL><INDENT>nonce = str(int(time.time() * <NUM_LIT>))<EOL>auth_string = '<STR_LIT>' + nonce<EOL>auth_sig = hmac.new(secret.encode(), auth_string.encode(),<EOL>hashlib.sha384).hexdigest()<EOL>payload = {'<STR_LIT>': '<STR_LIT>', '<STR_LIT>': api_key, '<STR_LIT>': auth_sig,<EOL>'<STR_LIT>': auth_string, '<STR_LIT>': nonce}<EOL>payload = json.dumps(payload)<EOL><DEDENT>elif list_data:<EOL><INDENT>payload = json.dumps(list_data)<EOL><DEDENT>else:<EOL><INDENT>payload = json.dumps(kwargs)<EOL><DEDENT>self.log.debug("<STR_LIT>", payload)<EOL>try:<EOL><INDENT>self.socket.send(payload)<EOL><DEDENT>except websocket.WebSocketConnectionClosedException:<EOL><INDENT>self.log.error("<STR_LIT>", kwargs)<EOL><DEDENT>

Sends the given Payload to the API via the websocket connection.

        :param kwargs: payload paarameters as key=value pairs
        :return:


self.log.debug("<STR_LIT>")<EOL>self._stop_timers()<EOL>self.ping_timer = Timer(self.ping_interval, self.send_ping)<EOL>self.ping_timer.start()<EOL>self.connection_timer = Timer(self.connection_timeout,<EOL>self._connection_timed_out)<EOL>self.connection_timer.start()<EOL>

Resets and starts timers for API data and connection.

        :return:


self.log.debug("<STR_LIT>")<EOL>self.paused.set()<EOL>

Pauses the connection.

        :return:


<EOL>if self.bitfinex_config:<EOL><INDENT>self.send(**self.bitfinex_config)<EOL><DEDENT>q_list = []<EOL>while True:<EOL><INDENT>try:<EOL><INDENT>identifier, q = self.channel_configs.popitem(last=True if soft else False)<EOL><DEDENT>except KeyError:<EOL><INDENT>break<EOL><DEDENT>q_list.append((identifier, q.copy()))<EOL>if identifier == '<STR_LIT>':<EOL><INDENT>self.send(**q, auth=True)<EOL>continue<EOL><DEDENT>if soft:<EOL><INDENT>q['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>self.send(**q)<EOL><DEDENT>if soft:<EOL><INDENT>for identifier, q in reversed(q_list):<EOL><INDENT>self.channel_configs[identifier] = q<EOL>self.send(**q)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for identifier, q in q_list:<EOL><INDENT>self.channel_configs[identifier] = q<EOL><DEDENT><DEDENT>

Resubscribes to all channels found in self.channel_configs.

        :param soft: if True, unsubscribes first.
        :return: None


def pass_to_client(self, event, data, *args):

self.q.put((event, data, *args))<EOL>

Passes data up to the client via a Queue().

        :param event:
        :param data:
        :param args:
        :return:


@is_connected<EOL><INDENT>def subscribe_to_ticker(self, pair, **kwargs):<DEDENT>

identifier = ('<STR_LIT>', pair)<EOL>self._subscribe('<STR_LIT>', identifier, symbol=pair, **kwargs)<EOL>

Subscribe to the passed pair's ticker channel.

        :param pair: str, Symbol pair to request data for
        :param kwargs:
        :return:


@is_connected<EOL><INDENT>def cancel_order(self, multi=False, **order_identifiers):<DEDENT>

if multi:<EOL><INDENT>self._send_auth_command('<STR_LIT>', order_identifiers)<EOL><DEDENT>else:<EOL><INDENT>self._send_auth_command('<STR_LIT>', order_identifiers)<EOL><DEDENT>

Cancel one or multiple orders via Websocket.

        :param multi: bool, whether order_settings contains settings for one, or
                      multiples orders
        :param order_identifiers: Identifiers for the order(s) you with to cancel
        :return:


key = ('<STR_LIT>', pair)<EOL>return self.queue_processor.books[key]<EOL>

Return a queue containing all received book data.

        :param pair:
        :return: Queue()


@is_connected<EOL><INDENT>def unsubscribe_from_trades(self, pair, **kwargs):<DEDENT>

identifier = ('<STR_LIT>', pair)<EOL>self._unsubscribe('<STR_LIT>', identifier, symbol=pair, **kwargs)<EOL>

Unsubscribe to the passed pair's trades channel.

        :param pair: str, Symbol pair to request data for
        :param kwargs:
        :return:


self._send_auth_command('<STR_LIT>', operations)<EOL>

Execute multiple, order-related operations via Websocket.

        :param operations: operations to send to the websocket
        :return:


@is_connected<EOL><INDENT>def subscribe_to_raw_order_book(self, pair, prec=None, **kwargs):<DEDENT>

identifier = ('<STR_LIT>', pair)<EOL>prec = '<STR_LIT>' if prec is None else prec<EOL>self._subscribe('<STR_LIT>', identifier, pair=pair, prec=prec, **kwargs)<EOL>

Subscribe to the passed pair's raw order book channel.

        :param pair: str, Symbol pair to request data for
        :param prec:
        :param kwargs:
        :return:


key = ('<STR_LIT>', pair)<EOL>return self.queue_processor.raw_books[key]<EOL>

Return a queue containing all received raw book data.

        :param pair:
        :return: Queue()


def _handle_auth(self, dtype, data, ts):

<EOL>if dtype == '<STR_LIT>':<EOL><INDENT>raise NotImplementedError<EOL><DEDENT>channel_id = data.pop('<STR_LIT>')<EOL>user_id = data.pop('<STR_LIT>')<EOL>identifier = ('<STR_LIT>', user_id)<EOL>self.channel_handlers[identifier] = channel_id<EOL>self.channel_directory[identifier] = channel_id<EOL>self.channel_directory[channel_id] = identifier<EOL>

Handles authentication responses.

        :param dtype:
        :param data:
        :param ts:
        :return:


<EOL>chan_id, channel_short_name, *data = data<EOL>entry = (channel_short_name, data, ts)<EOL>self.account.put(entry)<EOL>

Handles Account related data.

        translation table for channel names:
            Data Channels
            os      -   Orders
            hos     -   Historical Orders
            ps      -   Positions
            hts     -   Trades (snapshot)
            te      -   Trade Event
            tu      -   Trade Update
            ws      -   Wallets
            bu      -   Balance Info
            miu     -   Margin Info
            fiu     -   Funding Info
            fos     -   Offers
            hfos    -   Historical Offers
            fcs     -   Credits
            hfcs    -   Historical Credits
            fls     -   Loans
            hfls    -   Historical Loans
            htfs    -   Funding Trades
            n       -   Notifications (WIP)

        :param dtype:
        :param data:
        :param ts:
        :return:


def __init__(self, data_q, log_level=None,<EOL>*args, **kwargs):

super(QueueProcessor, self).__init__(*args, **kwargs)<EOL>self.q = data_q<EOL>self._response_handlers = {'<STR_LIT>': self._handle_unsubscribed,<EOL>'<STR_LIT>': self._handle_subscribed,<EOL>'<STR_LIT>': self._handle_conf,<EOL>'<STR_LIT>': self._handle_auth,<EOL>'<STR_LIT>': self._handle_auth}<EOL>self._data_handlers = {'<STR_LIT>': self._handle_ticker,<EOL>'<STR_LIT>': self._handle_book,<EOL>'<STR_LIT>': self._handle_raw_book,<EOL>'<STR_LIT>': self._handle_candles,<EOL>'<STR_LIT>': self._handle_trades}<EOL>self._registry = {}<EOL>self.channel_directory = {}<EOL>self.channel_handlers = {}<EOL>self.last_update = {}<EOL>self.tickers = defaultdict(Queue)<EOL>self.books = defaultdict(Queue)<EOL>self.raw_books = defaultdict(Queue)<EOL>self.trades = defaultdict(Queue)<EOL>self.candles = defaultdict(Queue)<EOL>self.account = Queue()<EOL>self._stopped = Event()<EOL>self.log = logging.getLogger(self.__module__)<EOL>self.log.setLevel(level=logging.INFO if not log_level else log_level)<EOL>

Initialze a QueueProcessor instance.

        :param data_q: Queue()
        :param log_level: logging level
        :param args: Thread *args
        :param kwargs: Thread **kwargs


def _handle_unsubscribed(self, dtype, data, ts):

self.log.debug("<STR_LIT>", dtype, data, ts)<EOL>channel_id = data.pop('<STR_LIT>')<EOL>chan_identifier = self.channel_directory.pop(channel_id)<EOL>self.channel_directory.pop(chan_identifier)<EOL>self.channel_handlers.pop(channel_id)<EOL>self.last_update.pop(channel_id)<EOL>self.log.info("<STR_LIT>", chan_identifier)<EOL>

Handles responses to unsubscribe() commands.

        Removes a channel id from the client.

        :param dtype:
        :param data:
        :param ts:
        :return:


def _handle_trades(self, dtype, data, ts):

self.log.debug("<STR_LIT>", dtype, data, ts)<EOL>channel_id, *data = data<EOL>channel_identifier = self.channel_directory[channel_id]<EOL>entry = (data, ts)<EOL>self.trades[channel_identifier].put(entry)<EOL>

Files trades in self._trades[chan_id].

        :param dtype:
        :param data:
        :param ts:
        :return:


while not self._stopped.is_set():<EOL><INDENT>try:<EOL><INDENT>message = self.q.get(timeout=<NUM_LIT:0.1>)<EOL><DEDENT>except Empty:<EOL><INDENT>continue<EOL><DEDENT>dtype, data, ts = message<EOL>if dtype in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>self._response_handlers[dtype](dtype, data, ts)<EOL><DEDENT>except KeyError:<EOL><INDENT>self.log.error("<STR_LIT>"<EOL>"<STR_LIT>", dtype, message)<EOL><DEDENT><DEDENT>elif dtype == '<STR_LIT:data>':<EOL><INDENT>try:<EOL><INDENT>channel_id = data[<NUM_LIT:0>]<EOL>if channel_id != <NUM_LIT:0>:<EOL><INDENT>channel_type, *_ = self.channel_directory[channel_id]<EOL>self._data_handlers[channel_type](channel_type, data, ts)<EOL>self.update_timestamps(channel_id, ts)<EOL><DEDENT>else:<EOL><INDENT>self._handle_account(data=data, ts=ts)<EOL><DEDENT><DEDENT>except KeyError:<EOL><INDENT>self.log.error("<STR_LIT>",<EOL>message)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.log.error("<STR_LIT>", message)<EOL>continue<EOL><DEDENT><DEDENT>


from sregistry.main import get_client<EOL>cli = get_client(quiet=args.quiet)<EOL>for query in args.query:<EOL><INDENT>if query in ['<STR_LIT>','<STR_LIT:*>']:<EOL><INDENT>query = None<EOL><DEDENT>cli.ls(query=query)<EOL><DEDENT>

the list command corresponds with listing images for an external
       resource. This is different from listing images that are local to the
       database, which should be done with "images"


from sregistry.main import Client as cli<EOL>cli.list_builders()<EOL>sys.exit(<NUM_LIT:0>)<EOL>

list running instances for a user, including all builders and report
       instance names and statuses.


from sregistry.main import get_client<EOL>cli = get_client(init=False)<EOL>if len(args.commands) > <NUM_LIT:0>:<EOL><INDENT>template_name = args.commands.pop(<NUM_LIT:0>)<EOL><DEDENT>cli.list_templates(template_name)<EOL>sys.exit(<NUM_LIT:0>)<EOL>

list a specific template (if a name is provided) or all templates
       available.

       Parameters
       ==========
       args: the argparse object to look for a template name
       template_name: if not set, show all


from sregistry.main import get_client<EOL>cli = get_client(quiet=args.quiet)<EOL>for query in args.query:<EOL><INDENT>if query in ['<STR_LIT>','<STR_LIT:*>']:<EOL><INDENT>query = None<EOL><DEDENT>cli.images(query=query)<EOL><DEDENT>

the images entrypoint is intended to list images locally in the user
       database, optionally taking one or more query string to subset the
       search


def add(backend, variable, value, force=False):

print('<STR_LIT>')<EOL>settings = read_client_secrets()<EOL>prefix = '<STR_LIT>' %backend.upper()<EOL>if not variable.startswith(prefix):<EOL><INDENT>variable = '<STR_LIT>' %(prefix, variable)<EOL><DEDENT>variable = variable.upper()<EOL>bot.info("<STR_LIT>" %(variable, value))<EOL>if backend in settings:<EOL><INDENT>if variable in settings[backend] and force is False:<EOL><INDENT>previous = settings[backend][variable]<EOL>bot.error('<STR_LIT>' %(variable, previous))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>if backend not in settings:<EOL><INDENT>settings[backend] = {}<EOL><DEDENT>settings[backend][variable] = value<EOL>update_secrets(settings)<EOL>


settings = read_client_secrets()<EOL>if backend is not None:<EOL><INDENT>settings['<STR_LIT>'] = backend<EOL>update_secrets(settings)<EOL>print('<STR_LIT>' %backend)<EOL><DEDENT>

activate a backend by adding it to the .sregistry configuration file.


DISABLE_CACHE = convert2boolean(getenv("<STR_LIT>",<EOL>default=False))<EOL>if DISABLE_CACHE:<EOL><INDENT>SINGULARITY_CACHE = tempfile.mkdtemp()<EOL><DEDENT>else:<EOL><INDENT>userhome = pwd.getpwuid(os.getuid())[<NUM_LIT:5>]<EOL>_cache = os.path.join(userhome, "<STR_LIT>")<EOL>SINGULARITY_CACHE = getenv("<STR_LIT>", default=_cache)<EOL><DEDENT>cache_base = clean_path(SINGULARITY_CACHE)<EOL>if subfolder is not None:<EOL><INDENT>cache_base = "<STR_LIT>" % (cache_base, subfolder)<EOL><DEDENT>mkdir_p(cache_base)<EOL>if not quiet:<EOL><INDENT>bot.debug("<STR_LIT>" % cache_base)<EOL><DEDENT>return cache_base<EOL>

get_cache will return the user's cache for singularity.
    :param subfolder: a subfolder in the cache base to retrieve, specifically


return os.path.abspath(os.path.dirname(os.path.dirname(__file__)))<EOL>

get_installdir returns the installation directory of the application


if sudo is True:<EOL><INDENT>cmd = ['<STR_LIT>'] + cmd<EOL><DEDENT>try:<EOL><INDENT>output = Popen(cmd, stderr=STDOUT, stdout=PIPE)<EOL><DEDENT>except FileNotFoundError:<EOL><INDENT>cmd.pop(<NUM_LIT:0>)<EOL>output = Popen(cmd, stderr=STDOUT, stdout=PIPE)<EOL><DEDENT>t = output.communicate()[<NUM_LIT:0>],output.returncode<EOL>output = {'<STR_LIT:message>':t[<NUM_LIT:0>],<EOL>'<STR_LIT>':t[<NUM_LIT:1>]}<EOL>if isinstance(output['<STR_LIT:message>'], bytes):<EOL><INDENT>output['<STR_LIT:message>'] = output['<STR_LIT:message>'].decode('<STR_LIT:utf-8>')<EOL><DEDENT>return output<EOL>

run_command uses subprocess to send a command to the terminal.

    Parameters
    ==========
    cmd: the command to send, should be a list for subprocess
    error_message: the error message to give to user if fails,
    if none specified, will alert that command failed.


if special_characters is None:<EOL><INDENT>special_characters = []<EOL><DEDENT>return '<STR_LIT>'.join(e.lower()<EOL>for e in name if e.isalnum() or e in special_characters)<EOL>

format_container_name will take a name supplied by the user,
    remove all special characters (except for those defined by "special-characters"
    and return the new image name.


hasher = hashlib.md5()<EOL>with open(image_path, "<STR_LIT:rb>") as f:<EOL><INDENT>for chunk in iter(lambda: f.read(<NUM_LIT>), b"<STR_LIT>"):<EOL><INDENT>hasher.update(chunk)<EOL><DEDENT><DEDENT>return hasher.hexdigest()<EOL>

return an md5 hash of the file based on a criteria level. This
       is intended to give the file a reasonable version.

       Parameters
       ==========
       image_path: full path to the singularity image


from .terminal import ( run_command, which )<EOL>result = which('<STR_LIT>')<EOL>if result['<STR_LIT>'] != <NUM_LIT:0>:<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>script = result['<STR_LIT:message>'] <EOL>command = ['<STR_LIT>' ,script, '<STR_LIT>', archive, '<STR_LIT>', output_folder]<EOL>if not bot.is_quiet():<EOL><INDENT>print("<STR_LIT>" % archive)<EOL><DEDENT>return run_command(command)<EOL>

use blob2oci to handle whiteout files for extraction. Credit for this
       script goes to docker2oci by Olivier Freyermouth, and see script
       folder for license.

       Parameters
       ==========
       archive: the archive to extract
       output_folder the output folder (sandbox) to extract to


def write_json(json_obj, filename, mode="<STR_LIT:w>", print_pretty=True):

with open(filename, mode) as filey:<EOL><INDENT>if print_pretty:<EOL><INDENT>filey.writelines(print_json(json_obj))<EOL><DEDENT>else:<EOL><INDENT>filey.writelines(json.dumps(json_obj))<EOL><DEDENT><DEDENT>return filename<EOL>

write_json will (optionally,pretty print) a json object to file

       Parameters
       ==========
       json_obj: the dict to print to json
       filename: the output file to write to
       pretty_print: if True, will use nicer formatting


with open(filename, mode) as filey:<EOL><INDENT>data = json.load(filey)<EOL><DEDENT>return data<EOL>

read_json reads in a json file and returns
       the data structure as dict.


return pwd.getpwuid(os.getuid())[<NUM_LIT:5>]<EOL>

get the user home based on the effective uid


return json.dumps(<EOL>json_obj,<EOL>indent=<NUM_LIT:4>,<EOL>separators=(<EOL>'<STR_LIT:U+002C>',<EOL>'<STR_LIT>'))<EOL>

just dump the json in a "pretty print" format


try:<EOL><INDENT>os.makedirs(path)<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno == errno.EEXIST and os.path.isdir(path):<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>bot.error("<STR_LIT>" % path)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>

mkdir_p attempts to get the same functionality as mkdir -p
    :param path: the path to create.


tag = re.sub('<STR_LIT>','<STR_LIT>', path)<EOL>if tag in ['<STR_LIT>', None]:<EOL><INDENT>tag = '<STR_LIT>'<EOL><DEDENT>return tag<EOL>

get a recipe tag (the extension of a Singularity file). The extension
       determines the tag. If no extension is found, latest is used.

       Parameters
       ==========
       path: the path to the recipe file (e.g. /opt/Singularity.tag)


def parse_header(recipe, header="<STR_LIT>", remove_header=True):

parsed_header = None<EOL>fromline = [x for x in recipe.split('<STR_LIT:\n>') if "<STR_LIT>" %header in x.lower()]<EOL>if len(fromline) == <NUM_LIT:0>:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>if len(fromline) > <NUM_LIT:0>:<EOL><INDENT>fromline = fromline[<NUM_LIT:0>]<EOL>parsed_header = fromline.strip()<EOL><DEDENT>if remove_header is True:<EOL><INDENT>parsed_header = fromline.split('<STR_LIT::>', <NUM_LIT:1>)[-<NUM_LIT:1>].strip()<EOL><DEDENT>return parsed_header<EOL>

take a recipe, and return the complete header, line. If
       remove_header is True, only return the value.

       Parameters
       ==========
       recipe: the recipe file
       headers: the header key to find and parse
       remove_header: if true, remove the key


container = self.rm(image_name, delete=True)<EOL>if container is not None:<EOL><INDENT>bot.info("<STR_LIT>" % container)<EOL><DEDENT>

Remove an image from the database and filesystem.


container = self.get(image_name)<EOL>if container is not None:<EOL><INDENT>name = container.uri or container.get_uri()<EOL>image = container.image<EOL>self.session.delete(container)<EOL>self.session.commit()<EOL>if image is not None:<EOL><INDENT>if os.path.exists(image) and delete is True:<EOL><INDENT>os.remove(container.image)<EOL><DEDENT>return image<EOL><DEDENT>bot.info("<STR_LIT>" % name)<EOL><DEDENT>

Remove an image from the database, akin to untagging the image. This
    does not delete the file from the cache, unless delete is set to True
    (as called by rmi).


from sregistry.database.models import (<EOL>Container,<EOL>Collection<EOL>)<EOL>if image_path is not None:<EOL><INDENT>if not os.path.exists(image_path) and save is True:<EOL><INDENT>bot.error('<STR_LIT>' %image_path)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT><DEDENT>if image_uri is None:<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>names = parse_image_name( remove_uri(image_uri) )<EOL>bot.debug('<STR_LIT>' % names['<STR_LIT>'])    <EOL>metadata = self.get_metadata(image_path, names=names)<EOL>collection = self.get_or_create_collection(names['<STR_LIT>'])<EOL>version = names.get('<STR_LIT:version>')<EOL>if version == None:<EOL><INDENT>if image_path != None:<EOL><INDENT>version = get_image_hash(image_path)<EOL><DEDENT>else:<EOL><INDENT>version = '<STR_LIT>'  <EOL><DEDENT>names = parse_image_name( remove_uri(image_uri), version=version )<EOL><DEDENT>if save is True and image_path is not None:<EOL><INDENT>if image_name is None:      <EOL><INDENT>image_name = self._get_storage_name(names)<EOL><DEDENT>if copy is True:<EOL><INDENT>copyfile(image_path, image_name)<EOL><DEDENT>else:<EOL><INDENT>shutil.move(image_path, image_name)<EOL><DEDENT>image_path = image_name<EOL><DEDENT>if url is None and "<STR_LIT:url>" in metadata:<EOL><INDENT>url = metadata['<STR_LIT:url>']<EOL><DEDENT>container = self.get_container(name=names['<STR_LIT:image>'],<EOL>collection_id=collection.id, <EOL>tag=names['<STR_LIT>'],<EOL>version=version)<EOL>if container is None:<EOL><INDENT>action = "<STR_LIT>"<EOL>container = Container(metrics=json.dumps(metadata),<EOL>name=names['<STR_LIT:image>'],<EOL>image=image_path,<EOL>client=self.client_name,<EOL>tag=names['<STR_LIT>'],<EOL>version=version,<EOL>url=url,<EOL>uri=names['<STR_LIT>'],<EOL>collection_id=collection.id)<EOL>self.session.add(container)<EOL>collection.containers.append(container)<EOL><DEDENT>else:<EOL><INDENT>action="<STR_LIT>"<EOL>metrics=json.loads(container.metrics)<EOL>metrics.update(metadata)<EOL>container.url= url<EOL>container.client=self.client_name<EOL>if image_path is not None:<EOL><INDENT>container.image=image_path<EOL><DEDENT>container.metrics=json.dumps(metrics)<EOL><DEDENT>self.session.commit()<EOL>bot.info("<STR_LIT>" % (action,names['<STR_LIT>']))<EOL>return container<EOL>

get or create a container, including the collection to add it to.
    This function can be used from a file on the local system, or via a URL
    that has been downloaded. Either way, if one of url, version, or image_file
    is not provided, the model is created without it. If a version is not
    provided but a file path is, then the file hash is used.

    Parameters
    ==========
    image_path: full path to image file
    image_name: if defined, the user wants a custom name (and not based on uri)
    metadata: any extra metadata to keep for the image (dict)
    save: if True, move the image to the cache if it's not there
    copy: If True, copy the image instead of moving it.

    image_name: a uri that gets parsed into a names object that looks like:

    {'collection': 'vsoch',
     'image': 'hello-world',
     'storage': 'vsoch/hello-world-latest.img',
     'tag': 'latest',
     'version': '12345'
     'uri': 'vsoch/hello-world:latest@12345'}

    After running add, the user will take some image in a working
    directory, add it to the database, and have it available for search
    and use under SREGISTRY_STORAGE/<collection>/<container>

    If the container was retrieved from a webby place, it should have version
    If no version is found, the file hash is used.


<EOL>self.database = '<STR_LIT>'<EOL>

initialize the database, meaning we just set the name to be dummy


<EOL>self.database = '<STR_LIT>' % db_path<EOL>self.storage = SREGISTRY_STORAGE<EOL>bot.debug("<STR_LIT>" % self.database)<EOL>self.engine = create_engine(self.database, convert_unicode=True)<EOL>self.session = scoped_session(sessionmaker(autocommit=False,<EOL>autoflush=False,<EOL>bind=self.engine))<EOL>Base.query = self.session.query_property()<EOL>Base.metadata.create_all(bind=self.engine)<EOL>self.Base = Base<EOL>

initialize the database, with the default database path or custom of

       the format sqlite:////scif/data/expfactory.db

    The custom path can be set with the environment variable SREGISTRY_DATABASE
    when a user creates the client, we must initialize this db
    the database should use the .singularity cache folder to cache
    layers and images, and .singularity/sregistry.db as a database


return "<STR_LIT>"<EOL>


if self.colorize:<EOL><INDENT>if level in self.colors:<EOL><INDENT>text = "<STR_LIT>" % (self.colors[level],<EOL>text,<EOL>self.colors["<STR_LIT>"])<EOL><DEDENT><DEDENT>return text<EOL>

addColor to the prompt (usually prefix) if terminal
        supports, and specified to do so


if isinstance(message, bytes):<EOL><INDENT>message = message.decode('<STR_LIT:utf-8>')<EOL><DEDENT>stream.write(message)<EOL>

write will write a message to a stream,
        first checking the encoding


if messageLevel <= self.level:<EOL><INDENT>return True<EOL><DEDENT>return False<EOL>

check if a messageLevel is enabled to emit a level


def emit(self, level, message, prefix=None, color=None):

if color is None:<EOL><INDENT>color = level<EOL><DEDENT>if prefix is not None:<EOL><INDENT>prefix = self.addColor(color, "<STR_LIT>" % (prefix))<EOL><DEDENT>else:<EOL><INDENT>prefix = "<STR_LIT>"<EOL>message = self.addColor(color, message)<EOL><DEDENT>message = "<STR_LIT>" % (prefix, message)<EOL>if not message.endswith('<STR_LIT:\n>'):<EOL><INDENT>message = "<STR_LIT>" % message<EOL><DEDENT>if self.level == QUIET:<EOL><INDENT>pass<EOL><DEDENT>elif self.isEnabledFor(level):<EOL><INDENT>if self.emitError(level):<EOL><INDENT>self.write(self.errorStream, message)<EOL><DEDENT>else:<EOL><INDENT>self.write(self.outputStream, message)<EOL><DEDENT><DEDENT>self.history.append(message)<EOL>

emit is the main function to print the message
        optionally with a prefix
        :param level: the level of the message
        :param message: the message to print
        :param prefix: a prefix for the message


if len(select_from) <= <NUM_LIT:0>:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>return choice(select_from)<EOL>

select an element from a list using random.choice

            Parameters
            ==========
            should be a list of things to select from


def generate(self, delim='<STR_LIT:->', length=<NUM_LIT:4>, chars='<STR_LIT>'):

descriptor = self._select(self._descriptors)<EOL>noun = self._select(self._nouns)<EOL>numbers = '<STR_LIT>'.join((self._select(chars) for _ in range(length)))<EOL>return delim.join([descriptor, noun, numbers])<EOL>

Generate a robot name. Inspiration from Haikunator, but much more
         poorly implemented ;)

Parameters
==========
delim: Delimiter
length: TokenLength
chars: TokenChars


bot.info('<STR_LIT>' % self._bucket_name)<EOL>

add the bucket name to be printed to the user at appropriate times


content = None<EOL>logfile = self._bucket.get_blob(logname)<EOL>print(logname)<EOL>if logfile:<EOL><INDENT>bot.info('<STR_LIT>' %logname)<EOL>content = requests.get(logfile.media_link).text<EOL>print(content)<EOL><DEDENT>return content<EOL>

helper function to retrieve a particular log, and print.

       Parameters
       ==========
       name: the name of the log to retrieve


bot.debug("<STR_LIT>" % name)<EOL>for file_object in files:<EOL><INDENT>if isinstance(file_object, dict):<EOL><INDENT>if "<STR_LIT>" in file_object:<EOL><INDENT>if file_object['<STR_LIT>'] == "<STR_LIT>":<EOL><INDENT>object_name = "<STR_LIT:/>".join(file_object['<STR_LIT:id>'].split('<STR_LIT:/>')[:-<NUM_LIT:1>])<EOL>object_name = re.sub('<STR_LIT>' %self._bucket['<STR_LIT:name>'],'<STR_LIT>', object_name,<NUM_LIT:1>)<EOL>delete_object(service=self._bucket_service,<EOL>bucket_name=bucket['<STR_LIT:name>'],<EOL>object_name=object_name)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

delete an image from Google Storage.

       Parameters
       ==========
       name: the name of the file (or image) to delete


def pull(self, images, file_name=None, save=True, force=False, **kwargs):

if not isinstance(images,list):<EOL><INDENT>images = [images]<EOL><DEDENT>bot.debug('<STR_LIT>' %len(images))<EOL>finished = []<EOL>for image in images:<EOL><INDENT>q = parse_image_name( remove_uri(image), <EOL>default_collection='<STR_LIT>' )<EOL>image_file = self._pull(file_name=file_name, <EOL>save=save, <EOL>force=force, <EOL>names=q,<EOL>kwargs=kwargs)<EOL>finished.append(image_file)<EOL><DEDENT>if len(finished) == <NUM_LIT:1>:<EOL><INDENT>finished = finished[<NUM_LIT:0>]<EOL><DEDENT>return finished<EOL>

pull an image from a docker hub. This is a (less than ideal) workaround
       that actually does the following:

       - creates a sandbox folder
       - adds docker layers, metadata folder, and custom metadata to it
       - converts to a squashfs image with build

    the docker manifests are stored with registry metadata.

    Parameters
    ==========
    images: refers to the uri given by the user to pull in the format
    <collection>/<namespace>. You should have an API that is able to
    retrieve a container based on parsing this uri.
    file_name: the user's requested name for the file. It can
               optionally be None if the user wants a default.
    save: if True, you should save the container to the database
          using self.add()

    Returns
    =======
    finished: a single container path, or list of paths


def __init__(self, secrets=None, **kwargs):

self.reverseLayers = False<EOL>self._reset_headers()<EOL>self._update_secrets()<EOL>self._set_base()<EOL>self._update_headers()<EOL>super(ApiConnection, self).__init__(**kwargs)<EOL>

to work with nvidia container registry. If you need help with
           their API, the best reference is:

           https://docs.aws.amazon.com/AmazonECR/latest/APIReference/ecr-api.pdf


bot.debug('<STR_LIT>')<EOL>try:<EOL><INDENT>from awscli.clidriver import create_clidriver<EOL><DEDENT>except:<EOL><INDENT>bot.exit('<STR_LIT>')<EOL><DEDENT>driver = create_clidriver()<EOL>self.aws = driver.session.create_client('<STR_LIT>')<EOL>

update secrets will take a secrets credential file
           either located at .sregistry or the environment variable
           SREGISTRY_CLIENT_SECRETS and update the current client
           secrets as well as the associated API base. For the case of
           using Docker Hub, if we find a .docker secrets file, we update
           from there.


image = None<EOL>repo = self.aws.describe_images(repositoryName=repo_name)<EOL>if '<STR_LIT>' in repo:<EOL><INDENT>for contender in repo.get('<STR_LIT>'):<EOL><INDENT>if tag in contender['<STR_LIT>']:<EOL><INDENT>image = contender<EOL>break<EOL><DEDENT><DEDENT><DEDENT>if image is None:<EOL><INDENT>bot.exit('<STR_LIT>' %(repo_name, digest))<EOL><DEDENT>digest = image['<STR_LIT>']<EOL>digests = self.aws.batch_get_image(repositoryName=repo_name, <EOL>imageIds=[{"<STR_LIT>": digest,<EOL>"<STR_LIT>": tag}])<EOL>self.manifest = json.loads(digests['<STR_LIT>'][<NUM_LIT:0>]['<STR_LIT>'])<EOL>return self.manifest<EOL>

return the image manifest via the aws client, saved in self.manifest


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>return self.manifest['<STR_LIT>']<EOL>

return a list of layers from a manifest.
The function is intended to work with both version
1 and 2 of the schema. All layers (including redundant)
are returned. By default, we try version 2 first,
then fall back to version 1.

For version 1 manifests: extraction is reversed

Parameters
==========
manifest: the manifest to read_layers from


def call(url, func, data=None, headers=None, <EOL>return_json=True, stream=False, <EOL>retry=True):

if DISABLE_SSL_CHECK is True:<EOL><INDENT>bot.warning('<STR_LIT>')<EOL><DEDENT>if data is not None:<EOL><INDENT>if not isinstance(data,dict):<EOL><INDENT>data = json.dumps(data)<EOL><DEDENT><DEDENT>response = func(url=url,<EOL>headers=headers,<EOL>data=data,<EOL>verify=not DISABLE_SSL_CHECK,<EOL>stream=stream)<EOL>if response.status_code in [<NUM_LIT>, <NUM_LIT>]:<EOL><INDENT>bot.error("<STR_LIT>" %(response.reason,<EOL>response.status_code))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if response.status_code == <NUM_LIT>:<EOL><INDENT>bot.error("<STR_LIT>" %(response.reason,<EOL>response.status_code))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if response.status_code == <NUM_LIT>:<EOL><INDENT>if retry is True:<EOL><INDENT>headers = update_token(response, headers)<EOL>return call(url, func, data=data,<EOL>headers=headers,<EOL>return_json=return_json,<EOL>stream=stream, retry=False)<EOL><DEDENT>bot.error("<STR_LIT>" %(response.reason,<EOL>response.status_code))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>elif response.status_code == <NUM_LIT:200>:<EOL><INDENT>if return_json:<EOL><INDENT>try:<EOL><INDENT>response = response.json()<EOL><DEDENT>except ValueError:<EOL><INDENT>bot.error("<STR_LIT>")<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT><DEDENT><DEDENT>return response<EOL>

call will issue the call, and issue a refresh token
    given a 401 response, and if the client has a _update_token function

    Parameters
    ==========
    func: the function (eg, post, get) to call
    url: the url to send file to
    headers: headers for the request
    data: additional data to add to the request
    return_json: return json if successful


def download_task(url, headers, destination, download_type='<STR_LIT>'):

<EOL>bot.verbose("<STR_LIT>" % (download_type, url))<EOL>file_name = "<STR_LIT>" % (destination,<EOL>next(tempfile._get_candidate_names()))<EOL>tar_download = download(url, file_name, headers=headers)<EOL>try:<EOL><INDENT>shutil.move(tar_download, destination)<EOL><DEDENT>except Exception:<EOL><INDENT>msg = "<STR_LIT>" % tar_download<EOL>msg += "<STR_LIT>"<EOL>bot.error(msg)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>return destination<EOL>

download an image layer (.tar.gz) to a specified download folder.
       This task is done by using local versions of the same download functions
       that are used for the client.
       core stream/download functions of the parent client.

       Parameters
       ==========
       image_id: the shasum id of the layer, already determined to not exist
       repo_name: the image name (library/ubuntu) to retrieve
       download_folder: download to this folder. If not set, uses temp.


def stream(url, headers, stream_to=None, retry=True):

bot.debug("<STR_LIT>" %url)<EOL>if DISABLE_SSL_CHECK is True:<EOL><INDENT>bot.warning('<STR_LIT>')<EOL><DEDENT>response = requests.get(url,         <EOL>headers=headers,<EOL>verify=not DISABLE_SSL_CHECK,<EOL>stream=True)<EOL>if response.status_code == <NUM_LIT> and retry is True:<EOL><INDENT>headers = update_token(response, headers)<EOL>return stream(url, headers, stream_to, retry=False)<EOL><DEDENT>if response.status_code == <NUM_LIT:200>:<EOL><INDENT>content_size = None<EOL>if '<STR_LIT>' in response.headers:<EOL><INDENT>progress = <NUM_LIT:0><EOL>content_size = int(response.headers['<STR_LIT>'])<EOL>bot.show_progress(progress,content_size,length=<NUM_LIT>)<EOL><DEDENT>chunk_size = <NUM_LIT:1> << <NUM_LIT:20><EOL>with open(stream_to,'<STR_LIT:wb>') as filey:<EOL><INDENT>for chunk in response.iter_content(chunk_size=chunk_size):<EOL><INDENT>filey.write(chunk)<EOL>if content_size is not None:<EOL><INDENT>progress+=chunk_size<EOL>bot.show_progress(iteration=progress,<EOL>total=content_size,<EOL>length=<NUM_LIT>,<EOL>carriage_return=False)<EOL><DEDENT><DEDENT><DEDENT>sys.stdout.write('<STR_LIT:\n>')<EOL>return stream_to <EOL><DEDENT>bot.error("<STR_LIT>" %(response.status_code))<EOL>sys.exit(<NUM_LIT:1>)<EOL>

stream is a get that will stream to file_name. Since this is a worker
       task, it differs from the client provided version in that it requires
       headers.


<EOL>progress = <NUM_LIT:1><EOL>total = len(tasks)<EOL>if len(tasks) == <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>if func2 is not None:<EOL><INDENT>total = total * <NUM_LIT:2><EOL><DEDENT>finished = []<EOL>level1 = []<EOL>results = []<EOL>try:<EOL><INDENT>prefix = "<STR_LIT>" % (progress, total)<EOL>bot.show_progress(<NUM_LIT:0>, total, length=<NUM_LIT>, prefix=prefix)<EOL>pool = multiprocessing.Pool(self.workers, init_worker)<EOL>self.start()<EOL>for task in tasks:<EOL><INDENT>result = pool.apply_async(multi_wrapper,<EOL>multi_package(func, [task]))<EOL>results.append(result)<EOL>level1.append(result._job)<EOL><DEDENT>while len(results) > <NUM_LIT:0>:<EOL><INDENT>result = results.pop()<EOL>result.wait()<EOL>bot.show_progress(progress, total, length=<NUM_LIT>, prefix=prefix)<EOL>progress += <NUM_LIT:1><EOL>prefix = "<STR_LIT>" % (progress, total)<EOL>if func2 is not None and result._job in level1:<EOL><INDENT>result = pool.apply_async(multi_wrapper,<EOL>multi_package(func2,<EOL>[(result.get(),)]))<EOL>results.append(result)<EOL><DEDENT>else:<EOL><INDENT>finished.append(result.get())<EOL><DEDENT><DEDENT>self.end()<EOL>pool.close()<EOL>pool.join()<EOL><DEDENT>except (KeyboardInterrupt, SystemExit):<EOL><INDENT>bot.error("<STR_LIT>")<EOL>pool.terminate()<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>except Exception as e:<EOL><INDENT>bot.error(e)<EOL><DEDENT>return finished<EOL>

run will send a list of tasks,
        a tuple with arguments, through a function.
        the arguments should be ordered correctly.
        :param func: the function to run with multiprocessing.pool
        :param tasks: a list of tasks, each a tuple
                      of arguments to process
        :param func2: filter function to run result
                      from func through (optional)


<EOL>self.base = self._get_and_update_setting('<STR_LIT>', self.base)<EOL>self._id = self._required_get_and_update('<STR_LIT>')<EOL>self._key = self._required_get_and_update('<STR_LIT>')<EOL>self._signature = self._get_and_update_setting('<STR_LIT>')<EOL>if self._signature == '<STR_LIT>':<EOL><INDENT>self._signature = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>self._signature = '<STR_LIT>'<EOL><DEDENT>self.get_bucket_name()<EOL>self.get_resource()<EOL>self.get_bucket()<EOL>

update secrets will update/get the base for the server, along
           with the bucket name, defaulting to sregistry.


for attr in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>if not hasattr(self, attr):<EOL><INDENT>bot.exit('<STR_LIT>' %(attr))<EOL><DEDENT><DEDENT>self.bucket = None<EOL>for bucket in self.s3.buckets.all():<EOL><INDENT>if bucket.name == self.bucket_name:<EOL><INDENT>self.bucket = bucket<EOL><DEDENT><DEDENT>if self.bucket is None:<EOL><INDENT>self.bucket = self.s3.create_bucket(Bucket=self.bucket_name)<EOL>bot.info('<STR_LIT>' % self.bucket.name )<EOL><DEDENT>return self.bucket<EOL>

given a bucket name and a client that is initialized, get or
           create the bucket.


try:<EOL><INDENT>message = json.loads(response._content.decode('<STR_LIT:utf-8>'))[field]<EOL><DEDENT>except:<EOL><INDENT>message = response.reason<EOL><DEDENT>return message<EOL>

attempt to read the detail provided by the response. If none,
        default to using the reason


def _pull(self, <EOL>file_name, <EOL>names, <EOL>save=True, <EOL>force=False, <EOL>uri="<STR_LIT>", <EOL>**kwargs):

<EOL>if file_name is None:<EOL><INDENT>file_name = self._get_storage_name(names)<EOL><DEDENT>if os.path.exists(file_name) and force is False:<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>digest = names['<STR_LIT:version>'] or names['<STR_LIT>']<EOL>sandbox = get_tmpdir(prefix="<STR_LIT>")<EOL>layers = self._download_layers(names['<STR_LIT:url>'], digest)<EOL>url = self._get_manifest_selfLink(names['<STR_LIT:url>'], digest)<EOL>envtar = self._get_environment_tar()<EOL>layers = [envtar] + layers<EOL>for layer in layers:<EOL><INDENT>bot.info('<STR_LIT>' %layer)<EOL>result = extract_tar(layer, sandbox, handle_whiteout=True)<EOL>if result['<STR_LIT>'] != <NUM_LIT:0>:<EOL><INDENT>bot.error(result['<STR_LIT:message>'])<EOL>sys.exit(<NUM_LIT:1>)        <EOL><DEDENT><DEDENT>sudo = kwargs.get('<STR_LIT>', False)<EOL>image_file = Singularity.build(image=file_name,<EOL>recipe=sandbox,<EOL>sudo=sudo)<EOL>if image_file is None:<EOL><INDENT>bot.info('<STR_LIT>')<EOL>image = image.replace('<STR_LIT>', uri)<EOL>image_file = Singularity.pull(image, pull_folder=sandbox)<EOL><DEDENT>if save is True:<EOL><INDENT>manifests = {}<EOL>if hasattr(self, '<STR_LIT>'):<EOL><INDENT>manifests = self.manifests<EOL><DEDENT>container = self.add(image_path = image_file,<EOL>image_uri = names['<STR_LIT>'],<EOL>metadata = manifests,<EOL>url = url)<EOL>image_file = container.image<EOL><DEDENT>if os.path.exists(image_file):<EOL><INDENT>bot.debug('<STR_LIT>' %image_file)<EOL>bot.custom(prefix="<STR_LIT>", message=image_file)<EOL><DEDENT>shutil.rmtree(sandbox)<EOL>return image_file<EOL>


def pull(self, images, <EOL>file_name=None, <EOL>save=True, <EOL>force=False, <EOL>base=None, <EOL>**kwargs):

if not isinstance(images,list):<EOL><INDENT>images = [images]<EOL><DEDENT>bot.debug('<STR_LIT>' %len(images))<EOL>finished = []<EOL>for image in images:<EOL><INDENT>base = self._update_base(image)<EOL>q = parse_image_name(remove_uri(image), base=base)<EOL>image_file = self._pull(file_name=file_name, <EOL>save=save, <EOL>force=force, <EOL>names=q,<EOL>kwargs=kwargs)<EOL>finished.append(image_file)<EOL><DEDENT>if len(finished) == <NUM_LIT:1>:<EOL><INDENT>finished = finished[<NUM_LIT:0>]<EOL><DEDENT>return finished<EOL>

pull an image from a docker hub. This is a (less than ideal) workaround
       that actually does the following:

       - creates a sandbox folder
       - adds docker layers, metadata folder, and custom metadata to it
       - converts to a squashfs image with build

    the docker manifests are stored with registry metadata.

    Parameters
    ==========
    images: refers to the uri given by the user to pull in the format
    <collection>/<namespace>. You should have an API that is able to
    retrieve a container based on parsing this uri.
    file_name: the user's requested name for the file. It can
               optionally be None if the user wants a default.
    save: if True, you should save the container to the database
          using self.add()
    base: the registry base, in case the client doesn't want to set in env.

    Returns
    =======
    finished: a single container path, or list of paths


base = self._get_setting('<STR_LIT>')<EOL>version = self._get_setting('<STR_LIT>')<EOL>if base is None:<EOL><INDENT>if default_base is None:<EOL><INDENT>base = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>base = default_base<EOL><DEDENT><DEDENT>if version is None:<EOL><INDENT>version = "<STR_LIT>"<EOL><DEDENT>nohttps = self._get_setting('<STR_LIT>')<EOL>if nohttps is None:<EOL><INDENT>nohttps = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>nohttps = "<STR_LIT>"<EOL><DEDENT>self._base = "<STR_LIT>" %(nohttps, base)<EOL>self._version = version<EOL>self.base = "<STR_LIT>" %(nohttps, base.strip('<STR_LIT:/>'), version)<EOL>

set the API base or default to use Docker Hub. The user is able
           to set the base, api version, and protocol via a settings file
           of environment variables:

           SREGISTRY_DOCKERHUB_BASE: defaults to index.docker.io
           SREGISTRY_DOCKERHUB_VERSION: defaults to v1
           SREGISTRY_DOCKERHUB_NO_HTTPS: defaults to not set (so https)


pass<EOL>

if you want to add an extra print (of a parameter, for example)
           for the user when the client initalizes, write it here, eg:
           bot.info('[setting] value')


def get_size(self, add_padding=True, round_up=True, return_mb=True):

if not hasattr(self,'<STR_LIT>'):<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>size = <NUM_LIT>  <EOL>for schemaVersion, manifest in self.manifests.items():<EOL><INDENT>if "<STR_LIT>" in manifest:<EOL><INDENT>size = <NUM_LIT:0><EOL>for layer in manifest["<STR_LIT>"]:<EOL><INDENT>if "<STR_LIT:size>" in layer:<EOL><INDENT>size += layer['<STR_LIT:size>']<EOL><DEDENT><DEDENT>if add_padding is True:<EOL><INDENT>size = size * <NUM_LIT:5><EOL><DEDENT>if return_mb is True:<EOL><INDENT>size = size / (<NUM_LIT> * <NUM_LIT>)  <EOL><DEDENT>if round_up is True:<EOL><INDENT>size = math.ceil(size)<EOL><DEDENT>size = int(size)<EOL><DEDENT>return size<EOL><DEDENT>

get_size will return the image size (must use v.2.0 manifest)

       Parameters
       ==========
       add_padding: if true, return reported size * 5
       round_up: if true, round up to nearest integer
       return_mb: if true, defaults bytes are converted to MB


def get_manifests(self, repo_name, digest=None):

if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self.manifests = {}<EOL><DEDENT>schemaVersions = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>for schemaVersion in schemaVersions:<EOL><INDENT>manifest = self._get_manifest(repo_name, digest, schemaVersion)<EOL>if manifest is not None:<EOL><INDENT>if schemaVersion == "<STR_LIT>" and "<STR_LIT>" in manifest:<EOL><INDENT>bot.debug('<STR_LIT>')<EOL>url = self._get_layerLink(repo_name, manifest['<STR_LIT>']['<STR_LIT>'])        <EOL>headers = {'<STR_LIT>': manifest['<STR_LIT>']['<STR_LIT>']}<EOL>self.manifests['<STR_LIT>'] = self._get(url, headers=headers)<EOL><DEDENT>self.manifests[schemaVersion] = manifest<EOL><DEDENT><DEDENT>return self.manifests<EOL>

get_manifests calls get_manifest for each of the schema versions,
       including v2 and v1. Version 1 includes image layers and metadata,
       and version 2 must be parsed for a specific manifest, and the 2nd
       call includes the layers. If a digest is not provided
       latest is used.

       Parameters
       ==========
       repo_name: reference to the <username>/<repository>:<tag> to obtain
       digest: a tag or shasum version


def get_manifest_selfLink(self, repo_name, digest=None):

url = "<STR_LIT>" % (self.base, repo_name)<EOL>if digest is None:<EOL><INDENT>digest = '<STR_LIT>'<EOL><DEDENT>return "<STR_LIT>" % (url, digest)<EOL>

get a selfLink for the manifest, for use by the client get_manifest
        function, along with the parents pull

       Parameters
       ==========
       repo_name: reference to the <username>/<repository>:<tag> to obtain
       digest: a tag or shasum version


def get_download_cache(self, destination, subfolder='<STR_LIT>'):

<EOL>if destination is None:<EOL><INDENT>destination = self._get_setting('<STR_LIT>', <EOL>SINGULARITY_CACHE)<EOL>destination = get_tmpdir(destination)<EOL><DEDENT>if not destination.endswith(subfolder):<EOL><INDENT>destination = "<STR_LIT>" %(destination, subfolder)<EOL><DEDENT>mkdir_p(destination)<EOL>return destination<EOL>

determine the user preference for atomic download of layers. If
       the user has set a singularity cache directory, honor it. Otherwise,
       use the Singularity default.


def create_metadata_tar(self, destination=None, metadata_folder="<STR_LIT>"):

tar_file = None<EOL>files = []<EOL>environ = self._extract_env()<EOL>if environ not in [None, "<STR_LIT>"]:<EOL><INDENT>bot.verbose3('<STR_LIT>')<EOL>template = get_template('<STR_LIT>')<EOL>template['<STR_LIT:name>'] = '<STR_LIT>' % (metadata_folder)<EOL>template['<STR_LIT:content>'] = environ<EOL>files.append(template)<EOL><DEDENT>labels = self._extract_labels()<EOL>if labels is not None:<EOL><INDENT>labels = print_json(labels)<EOL>bot.verbose3('<STR_LIT>')<EOL>template = get_template('<STR_LIT>')<EOL>template['<STR_LIT:name>'] = "<STR_LIT>" % metadata_folder<EOL>template['<STR_LIT:content>'] = labels<EOL>files.append(template)<EOL><DEDENT>runscript = self._extract_runscript()<EOL>if runscript is not None:<EOL><INDENT>bot.verbose3('<STR_LIT>')<EOL>template = get_template('<STR_LIT>')<EOL>template['<STR_LIT:name>'] = "<STR_LIT>" % metadata_folder<EOL>template['<STR_LIT:content>'] = runscript<EOL>files.append(template)<EOL><DEDENT>if len(files) > <NUM_LIT:0>:<EOL><INDENT>dest = self._get_download_cache(destination, subfolder='<STR_LIT>')<EOL>tar_file = create_tar(files, dest)<EOL><DEDENT>else:<EOL><INDENT>bot.warning("<STR_LIT>")<EOL><DEDENT>return tar_file<EOL>

create a metadata tar (runscript and environment) to add to the
       downloaded image. This function uses all functions in this section
       to obtain key--> values from the manifest config, and write
       to a .tar.gz

       Parameters
       ==========
       metadata_folder: the metadata folder in the singularity image.
                        default is .singularity.d


def search(self, query=None, args=None):

if query is None:<EOL><INDENT>bot.exit('<STR_LIT>')<EOL><DEDENT>return self._search_all(query)<EOL>

query a GitLab artifacts folder for a list of images.
     If query is None, collections are listed.


def _parse_image_name(self, image, retry=True):

try:<EOL><INDENT>job_id, collection, job_name = image.split('<STR_LIT:U+002C>')<EOL><DEDENT>except:<EOL><INDENT>if retry:<EOL><INDENT>return self._parse_image_name("<STR_LIT>" %(image, self.job),<EOL>retry=False)<EOL><DEDENT>bot.exit('''<STR_LIT>''')<EOL><DEDENT>return job_id, collection, job_name<EOL>

starting with an image string in either of the following formats:
           job_id|collection
           job_id|collection|job_name

           Parse the job_name, job_id, and collection uri from it. If the user
           provides the first option, we use the job_name set by the client
           (default is build).

           Parameters
           ==========
           image: the string to parse, with values separated by |
           retry: the client can call itself recursively once, providing the
                  default job_name if the user doesn't.