def get_contradictory_pairs(graph: BELGraph) -> Iterable[Tuple[BaseEntity, BaseEntity]]:

for u, v in graph.edges():<EOL><INDENT>if pair_has_contradiction(graph, u, v):<EOL><INDENT>yield u, v<EOL><DEDENT><DEDENT>

Iterates over contradictory node pairs in the graph based on their causal relationships

    :return: An iterator over (source, target) node pairs that have contradictory causal edges


def relation_set_has_contradictions(relations: Set[str]) -> bool:

has_increases = any(relation in CAUSAL_INCREASE_RELATIONS for relation in relations)<EOL>has_decreases = any(relation in CAUSAL_DECREASE_RELATIONS for relation in relations)<EOL>has_cnc = any(relation == CAUSES_NO_CHANGE for relation in relations)<EOL>return <NUM_LIT:1> < sum([has_cnc, has_decreases, has_increases])<EOL>

Return if the set of BEL relations contains a contradiction.


def calculate_error_by_annotation(graph: BELGraph, annotation: str) -> Mapping[str, List[str]]:

results = defaultdict(list)<EOL>for _, exc, ctx in graph.warnings:<EOL><INDENT>if not ctx or not edge_has_annotation(ctx, annotation):<EOL><INDENT>continue<EOL><DEDENT>values = ctx[ANNOTATIONS][annotation]<EOL>if isinstance(values, str):<EOL><INDENT>results[values].append(exc.__class__.__name__)<EOL><DEDENT>elif isinstance(values, Iterable):<EOL><INDENT>for value in values:<EOL><INDENT>results[value].append(exc.__class__.__name__)<EOL><DEDENT><DEDENT><DEDENT>return dict(results)<EOL>

Group the graph by a given annotation and builds lists of errors for each.

    :return: A dictionary of {annotation value: list of errors}


def group_errors(graph: BELGraph) -> Mapping[str, List[int]]:

warning_summary = defaultdict(list)<EOL>for _, exc, _ in graph.warnings:<EOL><INDENT>warning_summary[str(exc)].append(exc.line_number)<EOL><DEDENT>return dict(warning_summary)<EOL>

Group the errors together for analysis of the most frequent error.

    :return: A dictionary of {error string: list of line numbers}


def get_undefined_namespaces(graph: BELGraph) -> Set[str]:

return {<EOL>exc.namespace<EOL>for _, exc, _ in graph.warnings<EOL>if isinstance(exc, UndefinedNamespaceWarning)<EOL>}<EOL>

Get all namespaces that are used in the BEL graph aren't actually defined.


def get_most_common_errors(graph: BELGraph, n: Optional[int] = <NUM_LIT:20>):

return count_dict_values(group_errors(graph)).most_common(n)<EOL>

Get the (n) most common errors in a graph.


def get_names_including_errors_by_namespace(graph: BELGraph, namespace: str) -> Set[str]:

return get_names_by_namespace(graph, namespace) | get_incorrect_names_by_namespace(graph, namespace)<EOL>

Takes the names from the graph in a given namespace (:func:`pybel.struct.summary.get_names_by_namespace`) and
    the erroneous names from the same namespace (:func:`get_incorrect_names_by_namespace`) and returns them together
    as a unioned set

    :return: The set of all correct and incorrect names from the given namespace in the graph


def get_undefined_annotations(graph: BELGraph) -> Set[str]:

return {<EOL>exc.annotation<EOL>for _, exc, _ in graph.warnings<EOL>if isinstance(exc, UndefinedAnnotationWarning)<EOL>}<EOL>

Get all annotations that aren't actually defined.

    :return: The set of all undefined annotations


def get_incorrect_names_by_namespace(graph: BELGraph, namespace: str) -> Set[str]:

return {<EOL>exc.name<EOL>for _, exc, _ in graph.warnings<EOL>if isinstance(exc, (MissingNamespaceNameWarning, MissingNamespaceRegexWarning)) and exc.namespace == namespace<EOL>}<EOL>

Return the set of all incorrect names from the given namespace in the graph.

    :return: The set of all incorrect names from the given namespace in the graph


def get_names_including_errors(graph: BELGraph) -> Mapping[str, Set[str]]:

return {<EOL>namespace: get_names_including_errors_by_namespace(graph, namespace)<EOL>for namespace in get_namespaces(graph)<EOL>}<EOL>

Takes the names from the graph in a given namespace and the erroneous names from the same namespace and returns
    them together as a unioned set

    :return: The dict of the sets of all correct and incorrect names from the given namespace in the graph


def remove_falsy_values(counter: Mapping[Any, int]) -> Mapping[Any, int]:

return {<EOL>label: count<EOL>for label, count in counter.items()<EOL>if count<EOL>}<EOL>


def get_causal_in_edges(<EOL>graph: BELGraph,<EOL>nbunch: Union[BaseEntity, Iterable[BaseEntity]],<EOL>) -> Set[Tuple[BaseEntity, BaseEntity]]:

return {<EOL>(u, v)<EOL>for u, v, k, d in graph.in_edges(nbunch, keys=True, data=True)<EOL>if is_causal_relation(graph, u, v, k, d)<EOL>}<EOL>

Get the in-edges to the given node that are causal.

    :return: A set of (source, target) pairs where the target is the given node


def get_activities(graph: BELGraph) -> Set[BaseEntity]:

return get_nodes(graph, has_activity)<EOL>

Get all nodes that have molecular activities.


def _generate_citation_dict(graph: BELGraph) -> Mapping[str, Mapping[Tuple[BaseEntity, BaseEntity], str]]:

results = defaultdict(lambda: defaultdict(set))<EOL>for u, v, data in graph.edges(data=True):<EOL><INDENT>if CITATION not in data:<EOL><INDENT>continue<EOL><DEDENT>results[data[CITATION][CITATION_TYPE]][u, v].add(data[CITATION][CITATION_REFERENCE].strip())<EOL><DEDENT>return dict(results)<EOL>

Prepare a citation data dictionary from a graph.

    :return: A dictionary of dictionaries {citation type: {(source, target): citation reference}


def get_evidences_by_pmid(graph: BELGraph, pmids: Union[str, Iterable[str]]):

result = defaultdict(set)<EOL>for _, _, _, data in filter_edges(graph, build_pmid_inclusion_filter(pmids)):<EOL><INDENT>result[data[CITATION][CITATION_REFERENCE]].add(data[EVIDENCE])<EOL><DEDENT>return dict(result)<EOL>

Get a dictionary from the given PubMed identifiers to the sets of all evidence strings associated with each
    in the graph.

    :param graph: A BEL graph
    :param pmids: An iterable of PubMed identifiers, as strings. Is consumed and converted to a set.
    :return: A dictionary of {pmid: set of all evidence strings}
    :rtype: dict


def count_author_publications(graph: BELGraph) -> typing.Counter[str]:

authors = group_as_dict(_iter_author_publiations(graph))<EOL>return Counter(count_dict_values(count_defaultdict(authors)))<EOL>

Count the number of publications of each author to the given graph.


def get_citation_years(graph: BELGraph) -> List[Tuple[int, int]]:

return create_timeline(count_citation_years(graph))<EOL>

Create a citation timeline counter from the graph.


def count_authors_by_annotation(graph: BELGraph, annotation: str = '<STR_LIT>') -> Mapping[str, typing.Counter[str]]:

authors = group_as_dict(_iter_authors_by_annotation(graph, annotation=annotation))<EOL>return count_defaultdict(authors)<EOL>

Group the author counters by sub-graphs induced by the annotation.

    :param graph: A BEL graph
    :param annotation: The annotation to use to group the graph
    :return: A dictionary of Counters {subgraph name: Counter from {author: frequency}}


def get_pmid_by_keyword(keyword: str,<EOL>graph: Optional[BELGraph] = None,<EOL>pubmed_identifiers: Optional[Set[str]] = None,<EOL>) -> Set[str]:

if pubmed_identifiers is not None:<EOL><INDENT>return {<EOL>pubmed_identifier<EOL>for pubmed_identifier in pubmed_identifiers<EOL>if pubmed_identifier.startswith(keyword)<EOL>}<EOL><DEDENT>if graph is None:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>return {<EOL>pubmed_identifier<EOL>for pubmed_identifier in iterate_pubmed_identifiers(graph)<EOL>if pubmed_identifier.startswith(keyword)<EOL>}<EOL>

Get the set of PubMed identifiers beginning with the given keyword string.

    :param keyword: The beginning of a PubMed identifier
    :param graph: A BEL graph
    :param pubmed_identifiers: A set of pre-cached PubMed identifiers
    :return: A set of PubMed identifiers starting with the given string


def count_citations_by_annotation(graph: BELGraph, annotation: str) -> Mapping[str, typing.Counter[str]]:

citations = defaultdict(lambda: defaultdict(set))<EOL>for u, v, data in graph.edges(data=True):<EOL><INDENT>if not edge_has_annotation(data, annotation) or CITATION not in data:<EOL><INDENT>continue<EOL><DEDENT>k = data[ANNOTATIONS][annotation]<EOL>citations[k][u, v].add((data[CITATION][CITATION_TYPE], data[CITATION][CITATION_REFERENCE].strip()))<EOL><DEDENT>return {k: Counter(itt.chain.from_iterable(v.values())) for k, v in citations.items()}<EOL>

Group the citation counters by subgraphs induced by the annotation.

    :param graph: A BEL graph
    :param annotation: The annotation to use to group the graph
    :return: A dictionary of Counters {subgraph name: Counter from {citation: frequency}}


def count_confidences(graph: BELGraph) -> typing.Counter[str]:

return Counter(<EOL>(<EOL>'<STR_LIT:None>'<EOL>if ANNOTATIONS not in data or '<STR_LIT>' not in data[ANNOTATIONS] else<EOL>list(data[ANNOTATIONS]['<STR_LIT>'])[<NUM_LIT:0>]<EOL>)<EOL>for _, _, data in graph.edges(data=True)<EOL>if CITATION in data  <EOL>)<EOL>


def count_pmids(graph: BELGraph) -> Counter:

return Counter(iterate_pubmed_identifiers(graph))<EOL>

Count the frequency of PubMed documents in a graph.

    :return: A Counter from {(pmid, name): frequency}


def create_timeline(year_counter: typing.Counter[int]) -> List[Tuple[int, int]]:

if not year_counter:<EOL><INDENT>return []<EOL><DEDENT>from_year = min(year_counter) - <NUM_LIT:1><EOL>until_year = datetime.now().year + <NUM_LIT:1><EOL>return [<EOL>(year, year_counter.get(year, <NUM_LIT:0>))<EOL>for year in range(from_year, until_year)<EOL>]<EOL>

Complete the Counter timeline.

    :param Counter year_counter: counter dict for each year
    :return: complete timeline


def pairwise(iterable: Iterable[X]) -> Iterable[Tuple[X, X]]:

a, b = itt.tee(iterable)<EOL>next(b, None)<EOL>return zip(a, b)<EOL>

Iterate over pairs in list s -> (s0,s1), (s1,s2), (s2, s3), ...


def barv(d, plt, title=None, rotation='<STR_LIT>'):

labels = sorted(d, key=d.get, reverse=True)<EOL>index = range(len(labels))<EOL>plt.xticks(index, labels, rotation=rotation)<EOL>plt.bar(index, [d[v] for v in labels])<EOL>if title is not None:<EOL><INDENT>plt.title(title)<EOL><DEDENT>

A convenience function for plotting a vertical bar plot from a Counter


def set_percentage(x: Iterable[X], y: Iterable[X]) -> float:

a, b = set(x), set(y)<EOL>if not a:<EOL><INDENT>return <NUM_LIT:0.0><EOL><DEDENT>return len(a & b) / len(a)<EOL>

What percentage of x is contained within y?

    :param set x: A set
    :param set y: Another set
    :return: The percentage of x contained within y


def calculate_betweenness_centality(graph: BELGraph, number_samples: int = CENTRALITY_SAMPLES) -> Counter:

try:<EOL><INDENT>res = nx.betweenness_centrality(graph, k=number_samples)<EOL><DEDENT>except Exception:<EOL><INDENT>res = nx.betweenness_centrality(graph)<EOL><DEDENT>return Counter(res)<EOL>

Calculate the betweenness centrality over nodes in the graph.

    Tries to do it with a certain number of samples, but then tries a complete approach if it fails.


def calculate_single_tanimoto_set_distances(target: Iterable[X], dict_of_sets: Mapping[Y, Set[X]]) -> Mapping[Y, float]:

target_set = set(target)<EOL>return {<EOL>k: tanimoto_set_similarity(target_set, s)<EOL>for k, s in dict_of_sets.items()<EOL>}<EOL>

Return a dictionary of distances keyed by the keys in the given dict.

    Distances are calculated based on pairwise tanimoto similarity of the sets contained

    :param set target: A set
    :param dict_of_sets: A dict of {x: set of y}
    :type dict_of_sets: dict
    :return: A similarity dicationary based on the set overlap (tanimoto) score between the target set and the sets in
            dos
    :rtype: dict


def count_dict_values(dict_of_counters: Mapping[X, Sized]) -> typing.Counter[X]:

return Counter({<EOL>k: len(v)<EOL>for k, v in dict_of_counters.items()<EOL>})<EOL>

Count the number of elements in each value (can be list, Counter, etc).

    :param dict_of_counters: A dictionary of things whose lengths can be measured (lists, Counters, dicts)
    :return: A Counter with the same keys as the input but the count of the length of the values list/tuple/set/Counter


def canonical_circulation(elements: T, key: Optional[Callable[[T], bool]] = None) -> T:

return min(get_circulations(elements), key=key)<EOL>

Get get a canonical representation of the ordered collection by finding its minimum circulation with the
    given sort key


def calculate_global_tanimoto_set_distances(dict_of_sets: Mapping[X, Set]) -> Mapping[X, Mapping[X, float]]:

universe = set(itt.chain.from_iterable(dict_of_sets.values()))<EOL>universe_size = len(universe)<EOL>result: Dict[X, Dict[X, float]] = defaultdict(dict)<EOL>for x, y in itt.combinations(dict_of_sets, <NUM_LIT:2>):<EOL><INDENT>result[x][y] = result[y][x] = <NUM_LIT:1.0> - len(dict_of_sets[x] | dict_of_sets[y]) / universe_size<EOL><DEDENT>for x in dict_of_sets:<EOL><INDENT>result[x][x] = <NUM_LIT:1.0> - len(x) / universe_size<EOL><DEDENT>return dict(result)<EOL>

r"""Calculate an alternative distance matrix based on the following equation.

    .. math:: distance(A, B)=1- \|A \cup B\| / \| \cup_{s \in S} s\|

    :param dict_of_sets: A dict of {x: set of y}
    :return: A similarity matrix based on the alternative tanimoto distance as a dict of dicts


def count_defaultdict(dict_of_lists: Mapping[X, List[Y]]) -> Mapping[X, typing.Counter[Y]]:

return {<EOL>k: Counter(v)<EOL>for k, v in dict_of_lists.items()<EOL>}<EOL>

Count the number of elements in each value of the dictionary.


def min_tanimoto_set_similarity(x: Iterable[X], y: Iterable[X]) -> float:

a, b = set(x), set(y)<EOL>if not a or not b:<EOL><INDENT>return <NUM_LIT:0.0><EOL><DEDENT>return len(a & b) / min(len(a), len(b))<EOL>

Calculate the tanimoto set similarity using the minimum size.

    :param set x: A set
    :param set y: Another set
    :return: The similarity between


def tanimoto_set_similarity(x: Iterable[X], y: Iterable[X]) -> float:

a, b = set(x), set(y)<EOL>union = a | b<EOL>if not union:<EOL><INDENT>return <NUM_LIT:0.0><EOL><DEDENT>return len(a & b) / len(union)<EOL>


for u, v, key, data in graph.edges(keys=True, data=True):<EOL><INDENT>if u != v:<EOL><INDENT>continue<EOL><DEDENT>bel = graph.edge_to_bel(u, v, data)<EOL>line = data.get(LINE)<EOL>if line is None:<EOL><INDENT>continue  <EOL><DEDENT>elif has_protein_modification_increases_activity(graph, u, v, key):<EOL><INDENT>print(line, '<STR_LIT>', bel)<EOL>find_related(graph, v, data)<EOL><DEDENT>elif has_degradation_increases_activity(data):<EOL><INDENT>print(line, '<STR_LIT>', bel)<EOL>find_related(graph, v, data)<EOL><DEDENT>elif has_translocation_increases_activity(data):<EOL><INDENT>print(line, '<STR_LIT>', bel)<EOL>find_related(graph, v, data)<EOL><DEDENT>elif complex_increases_activity(graph, u, v, key):<EOL><INDENT>print(line, '<STR_LIT>', bel)<EOL>find_related(graph, v, data)<EOL><DEDENT>elif has_same_subject_object(graph, u, v, key):<EOL><INDENT>print(line, '<STR_LIT>', bel)<EOL><DEDENT>else:<EOL><INDENT>print(line, '<STR_LIT>', bel)<EOL><DEDENT><DEDENT>

Find edges that are A - A, meaning that some conditions in the edge best describe the interaction.


def self_edge_filter(_: BELGraph, source: BaseEntity, target: BaseEntity, __: str) -> bool:

return source == target<EOL>

Check if the source and target nodes are the same.


@edge_predicate<EOL>def has_translocation_increases_activity(data: Dict) -> bool:

return part_has_modifier(data, SUBJECT, TRANSLOCATION) and part_has_modifier(data, OBJECT, ACTIVITY)<EOL>

Check if the translocation of source causes activity of target.


@edge_predicate<EOL>def has_degradation_increases_activity(data: Dict) -> bool:

return part_has_modifier(data, SUBJECT, DEGRADATION) and part_has_modifier(data, OBJECT, ACTIVITY)<EOL>

Check if the degradation of source causes activity of target.


def compare(graph: BELGraph, annotation: str = '<STR_LIT>') -> Mapping[str, Mapping[str, float]]:

canonical_mechanisms = get_subgraphs_by_annotation(graph, annotation)<EOL>canonical_nodes = _transform_graph_dict_to_node_dict(canonical_mechanisms)<EOL>candidate_mechanisms = generate_bioprocess_mechanisms(graph)<EOL>candidate_nodes = _transform_graph_dict_to_node_dict(candidate_mechanisms)<EOL>results: Dict[str, Dict[str, float]] = defaultdict(dict)<EOL>it = itt.product(canonical_nodes.items(), candidate_nodes.items())<EOL>for (canonical_name, canonical_graph), (candidate_bp, candidate_graph) in it:<EOL><INDENT>tanimoto = tanimoto_set_similarity(candidate_nodes, canonical_nodes)<EOL>results[canonical_name][candidate_bp] = tanimoto<EOL><DEDENT>return dict(results)<EOL>

Compare generated mechanisms to actual ones.

    1. Generates candidate mechanisms for each biological process
    2. Gets sub-graphs for all NeuroMMSig signatures
    3. Make tanimoto similarity comparison for all sets

    :return: A dictionary table comparing the canonical subgraphs to generated ones


def neurommsig_gene_ora(graph: BELGraph, genes: List[Gene]) -> float:

graph_genes = set(get_nodes_by_function(graph, GENE))<EOL>return len(graph_genes.intersection(genes)) / len(graph_genes)<EOL>

Calculate the percentage of target genes mappable to the graph.

    Assume: graph central dogma inferred, collapsed to genes, collapsed variants


def neurommsig_topology(graph: BELGraph, nodes: List[BaseEntity]) -> float:

nodes = list(nodes)<EOL>number_nodes = len(nodes)<EOL>if number_nodes <= <NUM_LIT:1>:<EOL><INDENT>return <NUM_LIT:0.0><EOL><DEDENT>unnormalized_sum = sum(<EOL>u in graph[v]<EOL>for u, v in itt.product(nodes, repeat=<NUM_LIT:2>)<EOL>if v in graph and u != v<EOL>)<EOL>return unnormalized_sum / (number_nodes * (number_nodes - <NUM_LIT:1.0>))<EOL>

Calculate the node neighbor score for a given list of nodes.

    -  Doesn't consider self loops

    .. math::

         \frac{\sum_i^n N_G[i]}{n*(n-1)}


def get_neurommsig_scores(graph: BELGraph,<EOL>genes: List[Gene],<EOL>annotation: str = '<STR_LIT>',<EOL>ora_weight: Optional[float] = None,<EOL>hub_weight: Optional[float] = None,<EOL>top_percent: Optional[float] = None,<EOL>topology_weight: Optional[float] = None,<EOL>preprocess: bool = False<EOL>) -> Optional[Mapping[str, float]]:

if preprocess:<EOL><INDENT>graph = neurommsig_graph_preprocessor.run(graph)<EOL><DEDENT>if not any(gene in graph for gene in genes):<EOL><INDENT>logger.debug('<STR_LIT>')<EOL>return<EOL><DEDENT>subgraphs = get_subgraphs_by_annotation(graph, annotation=annotation)<EOL>return get_neurommsig_scores_prestratified(<EOL>subgraphs=subgraphs,<EOL>genes=genes,<EOL>ora_weight=ora_weight,<EOL>hub_weight=hub_weight,<EOL>top_percent=top_percent,<EOL>topology_weight=topology_weight,<EOL>)<EOL>

Preprocess the graph, stratify by the given annotation, then run the NeuroMMSig algorithm on each.

    :param graph: A BEL graph
    :param genes: A list of gene nodes
    :param annotation: The annotation to use to stratify the graph to subgraphs
    :param ora_weight: The relative weight of the over-enrichment analysis score from
     :py:func:`neurommsig_gene_ora`. Defaults to 1.0.
    :param hub_weight: The relative weight of the hub analysis score from :py:func:`neurommsig_hubs`.
     Defaults to 1.0.
    :param top_percent: The percentage of top genes to use as hubs. Defaults to 5% (0.05).
    :param topology_weight: The relative weight of the topolgical analysis core from
     :py:func:`neurommsig_topology`. Defaults to 1.0.
    :param preprocess: If true, preprocess the graph.
    :return: A dictionary from {annotation value: NeuroMMSig composite score}

    Pre-processing steps:

    1. Infer the central dogma with :func:``
    2. Collapse all proteins, RNAs and miRNAs to genes with :func:``
    3. Collapse variants to genes with :func:``


def neurommsig_hubs(graph: BELGraph, genes: List[Gene], top_percent: Optional[float] = None) -> float:

top_percent = top_percent or <NUM_LIT><EOL>if graph.number_of_nodes() < <NUM_LIT:20>:<EOL><INDENT>logger.debug('<STR_LIT>')<EOL>return <NUM_LIT:0.0><EOL><DEDENT>graph_genes = set(get_nodes_by_function(graph, GENE))<EOL>bc = Counter({<EOL>node: betweenness_centrality<EOL>for node, betweenness_centrality in calculate_betweenness_centality(graph).items()<EOL>if node in graph_genes<EOL>})<EOL>number_central_nodes = int(len(graph_genes) * top_percent)<EOL>if number_central_nodes < <NUM_LIT:1>:<EOL><INDENT>number_central_nodes = <NUM_LIT:1><EOL><DEDENT>number_mappable_central_nodes = sum(<EOL>node in genes<EOL>for node in bc.most_common(number_central_nodes)<EOL>)<EOL>return number_mappable_central_nodes / number_central_nodes<EOL>

Calculate the percentage of target genes mappable to the graph.

    Assume: graph central dogma inferred, collapsed to genes, collapsed variants, graph has more than 20 nodes

    :param graph: A BEL graph
    :param genes: A list of nodes
    :param top_percent: The percentage of top genes to use as hubs. Defaults to 5% (0.05).


logging.basicConfig(level=logging.INFO)<EOL>log.setLevel(logging.INFO)<EOL>bms_base = get_bms_base()<EOL>neurommsig_base = get_neurommsig_base()<EOL>neurommsig_excel_dir = os.path.join(neurommsig_base, '<STR_LIT>', '<STR_LIT>', '<STR_LIT>')<EOL>nift_values = get_nift_values()<EOL>log.info('<STR_LIT>')<EOL>ad_path = os.path.join(neurommsig_excel_dir, '<STR_LIT>', '<STR_LIT>')<EOL>ad_df = preprocess(ad_path)<EOL>with open(os.path.join(bms_base, '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'), '<STR_LIT:w>') as ad_file:<EOL><INDENT>write_neurommsig_bel(ad_file, ad_df, mesh_alzheimer, nift_values)<EOL><DEDENT>log.info('<STR_LIT>')<EOL>pd_path = os.path.join(neurommsig_excel_dir, '<STR_LIT>', '<STR_LIT>')<EOL>pd_df = preprocess(pd_path)<EOL>with open(os.path.join(bms_base, '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'), '<STR_LIT:w>') as pd_file:<EOL><INDENT>write_neurommsig_bel(pd_file, pd_df, mesh_parkinson, nift_values)<EOL><DEDENT>

Convert the Alzheimer's and Parkinson's disease NeuroMMSig excel sheets to BEL.


def write_neurommsig_bel(file,<EOL>df: pd.DataFrame,<EOL>disease: str,<EOL>nift_values: Mapping[str, str],<EOL>):

write_neurommsig_biolerplate(disease, file)<EOL>missing_features = set()<EOL>fixed_caps = set()<EOL>nift_value_originals = set(nift_values.values())<EOL>graph = BELGraph(<EOL>name=f'<STR_LIT>',<EOL>description=f'<STR_LIT>',<EOL>authors='<STR_LIT>',<EOL>contact='<STR_LIT>',<EOL>version=time.strftime('<STR_LIT>'),<EOL>)<EOL>for pathway, pathway_df in df.groupby(pathway_column):<EOL><INDENT>sorted_pathway_df = pathway_df.sort_values(genes_column)<EOL>sliced_df = sorted_pathway_df[columns].itertuples()<EOL>for _, gene, pubmeds, lit_snps, gwas_snps, ld_block_snps, clinical_features, clinical_snps in sliced_df:<EOL><INDENT>gene = ensure_quotes(gene)<EOL>for snp in itt.chain(lit_snps or [], gwas_snps or [], ld_block_snps or [], clinical_snps or []):<EOL><INDENT>if not snp.strip():<EOL><INDENT>continue<EOL><DEDENT>graph.add_association(<EOL>Gene('<STR_LIT>', gene),<EOL>Gene('<STR_LIT>', snp),<EOL>evidence='<STR_LIT>',<EOL>citation='<STR_LIT>',<EOL>annotations={<EOL>'<STR_LIT>': disease,<EOL>},<EOL>)<EOL><DEDENT>for clinical_feature in clinical_features or []:<EOL><INDENT>if not clinical_feature.strip():<EOL><INDENT>continue<EOL><DEDENT>if clinical_feature.lower() not in nift_values:<EOL><INDENT>missing_features.add(clinical_feature)<EOL>continue<EOL><DEDENT>if clinical_feature not in nift_value_originals:<EOL><INDENT>fixed_caps.add((clinical_feature, nift_values[clinical_feature.lower()]))<EOL>clinical_feature = nift_values[clinical_feature.lower()]  <EOL><DEDENT>graph.add_association(<EOL>Gene('<STR_LIT>', gene),<EOL>Abundance('<STR_LIT>', clinical_feature),<EOL>evidence='<STR_LIT>',<EOL>citation='<STR_LIT>',<EOL>annotations={<EOL>'<STR_LIT>': disease,<EOL>},<EOL>)<EOL>if clinical_snps:<EOL><INDENT>for clinical_snp in clinical_snps:<EOL><INDENT>graph.add_association(<EOL>Gene('<STR_LIT>', clinical_snp),<EOL>Abundance('<STR_LIT>', clinical_feature),<EOL>evidence='<STR_LIT>',<EOL>citation='<STR_LIT>',<EOL>annotations={<EOL>'<STR_LIT>': disease,<EOL>},<EOL>)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if missing_features:<EOL><INDENT>log.warning('<STR_LIT>', disease)<EOL>for feature in missing_features:<EOL><INDENT>log.warning(feature)<EOL><DEDENT><DEDENT>if fixed_caps:<EOL><INDENT>log.warning('<STR_LIT>')<EOL>for broken, fixed in fixed_caps:<EOL><INDENT>log.warning('<STR_LIT>', broken, fixed)<EOL><DEDENT><DEDENT>

Writes the NeuroMMSigDB excel sheet to BEL

    :param file: a file or file-like that can be writen to
    :param df:
    :param disease:
    :param nift_values: a dictionary of lowercased to normal names in NIFT


def munge_cell(cell, line=None, validators=None):

if pd.isnull(cell) or isinstance(cell, int):<EOL><INDENT>return None<EOL><DEDENT>c = '<STR_LIT:U+0020>'.join(cell.split())<EOL>if validators is not None and all(re.match(validator, c) is None for validator in validators):<EOL><INDENT>if line:<EOL><INDENT>log.info("<STR_LIT>", line, c)<EOL><DEDENT>return None<EOL><DEDENT>return [x.strip() for x in str(c).strip().split('<STR_LIT:U+002C>')]<EOL>

:param cell:
:param line:
:param validators:
:return:


def get_dampened_pairs(graph: BELGraph) -> SetOfNodePairs:

cg = get_causal_subgraph(graph)<EOL>results = set()<EOL>for u, v, d in cg.edges(data=True):<EOL><INDENT>if d[RELATION] not in CAUSAL_DECREASE_RELATIONS:<EOL><INDENT>continue<EOL><DEDENT>if cg.has_edge(v, u) and any(dd[RELATION] in CAUSAL_DECREASE_RELATIONS for dd in cg[v][u].values()):<EOL><INDENT>results.add(tuple(sorted([u, v], key=str)))<EOL><DEDENT><DEDENT>return results<EOL>

Find pairs of nodes that have mutual causal edges that are decreasing each other such that ``A -| B`` and
    ``B -| A``.

    :return: A set of pairs of nodes with mutual causal edges


def get_regulatory_pairs(graph: BELGraph) -> Set[NodePair]:

cg = get_causal_subgraph(graph)<EOL>results = set()<EOL>for u, v, d in cg.edges(data=True):<EOL><INDENT>if d[RELATION] not in CAUSAL_INCREASE_RELATIONS:<EOL><INDENT>continue<EOL><DEDENT>if cg.has_edge(v, u) and any(dd[RELATION] in CAUSAL_DECREASE_RELATIONS for dd in cg[v][u].values()):<EOL><INDENT>results.add((u, v))<EOL><DEDENT><DEDENT>return results<EOL>

Find pairs of nodes that have mutual causal edges that are regulating each other such that ``A -> B`` and
    ``B -| A``.

    :return: A set of pairs of nodes with mutual causal edges


def get_chaotic_triplets(graph: BELGraph) -> Iterable[NodeTriple]:

return _get_disregulated_triplets_helper(graph, CAUSAL_INCREASE_RELATIONS)<EOL>

Yield triples of nodes (A, B, C) that mutually increase each other, such as when ``A -> B``, ``B -> C``, and
    ``C -> A``.


def summarize_stability(graph: BELGraph) -> Mapping[str, int]:

regulatory_pairs = get_regulatory_pairs(graph)<EOL>chaotic_pairs = get_chaotic_pairs(graph)<EOL>dampened_pairs = get_dampened_pairs(graph)<EOL>contraditory_pairs = get_contradiction_summary(graph)<EOL>separately_unstable_triples = get_separate_unstable_correlation_triples(graph)<EOL>mutually_unstable_triples = get_mutually_unstable_correlation_triples(graph)<EOL>jens_unstable_triples = get_jens_unstable(graph)<EOL>increase_mismatch_triples = get_increase_mismatch_triplets(graph)<EOL>decrease_mismatch_triples = get_decrease_mismatch_triplets(graph)<EOL>chaotic_triples = get_chaotic_triplets(graph)<EOL>dampened_triples = get_dampened_triplets(graph)<EOL>return {<EOL>'<STR_LIT>': _count_or_len(regulatory_pairs),<EOL>'<STR_LIT>': _count_or_len(chaotic_pairs),<EOL>'<STR_LIT>': _count_or_len(dampened_pairs),<EOL>'<STR_LIT>': _count_or_len(contraditory_pairs),<EOL>'<STR_LIT>': _count_or_len(separately_unstable_triples),<EOL>'<STR_LIT>': _count_or_len(mutually_unstable_triples),<EOL>'<STR_LIT>': _count_or_len(jens_unstable_triples),<EOL>'<STR_LIT>': _count_or_len(increase_mismatch_triples),<EOL>'<STR_LIT>': _count_or_len(decrease_mismatch_triples),<EOL>'<STR_LIT>': _count_or_len(chaotic_triples),<EOL>'<STR_LIT>': _count_or_len(dampened_triples)<EOL>}<EOL>


def jens_transformation_beta(graph: BELGraph) -> DiGraph:

result = DiGraph()<EOL>for u, v, d in graph.edges(data=True):<EOL><INDENT>relation = d[RELATION]<EOL>if relation == NEGATIVE_CORRELATION:<EOL><INDENT>result.add_edge(u, v)<EOL>result.add_edge(v, u)<EOL><DEDENT>elif relation in CAUSAL_INCREASE_RELATIONS:<EOL><INDENT>result.add_edge(v, u)<EOL><DEDENT>elif relation in CAUSAL_DECREASE_RELATIONS:<EOL><INDENT>result.add_edge(u, v)<EOL><DEDENT><DEDENT>return result<EOL>

Apply Jens' Transformation (Type 2) to the graph.

    1. Induce a sub-graph over causal and correlative relations
    2. Transform edges with the following rules:
        - increases => backwards decreases
        - decreases => decreases
        - positive correlation => delete
        - negative correlation => two way decreases

    The resulting graph can be used to search for 3-cycles, which now symbolize stable triples where ``A -> B``,
    ``A -| C`` and ``B negativeCorrelation C``.


def get_correlation_graph(graph: BELGraph) -> Graph:

result = Graph()<EOL>for u, v, d in graph.edges(data=True):<EOL><INDENT>if d[RELATION] not in CORRELATIVE_RELATIONS:<EOL><INDENT>continue<EOL><DEDENT>if not result.has_edge(u, v):<EOL><INDENT>result.add_edge(u, v, **{d[RELATION]: True})<EOL><DEDENT>elif d[RELATION] not in result[u][v]:<EOL><INDENT>log.log(<NUM_LIT:5>, '<STR_LIT>', u, v)<EOL>result[u][v][d[RELATION]] = True<EOL>result[v][u][d[RELATION]] = True<EOL><DEDENT><DEDENT>return result<EOL>

Extract an undirected graph of only correlative relationships.


def get_dampened_triplets(graph: BELGraph) -> Iterable[NodeTriple]:

return _get_disregulated_triplets_helper(graph, CAUSAL_DECREASE_RELATIONS)<EOL>

Yield triples of nodes (A, B, C) that mutually decreases each other, such as when ``A -| B``,
    ``B -| C``, and ``C -| A``.


def jens_transformation_alpha(graph: BELGraph) -> DiGraph:

result = DiGraph()<EOL>for u, v, d in graph.edges(data=True):<EOL><INDENT>relation = d[RELATION]<EOL>if relation == POSITIVE_CORRELATION:<EOL><INDENT>result.add_edge(u, v)<EOL>result.add_edge(v, u)<EOL><DEDENT>elif relation in CAUSAL_INCREASE_RELATIONS:<EOL><INDENT>result.add_edge(u, v)<EOL><DEDENT>elif relation in CAUSAL_DECREASE_RELATIONS:<EOL><INDENT>result.add_edge(v, u)<EOL><DEDENT><DEDENT>return result<EOL>

Apply Jens' transformation (Type 1) to the graph.

    1. Induce a sub-graph over causal + correlative edges
    2. Transform edges by the following rules:
        - increases => increases
        - decreases => backwards increases
        - positive correlation => two way increases
        - negative correlation => delete

    The resulting graph can be used to search for 3-cycles, which now symbolize unstable triplets where ``A -> B``,
    ``A -| C`` and ``B positiveCorrelation C``.


def get_triangles(graph: DiGraph) -> SetOfNodeTriples:

return {<EOL>tuple(sorted([a, b, c], key=str))<EOL>for a, b in graph.edges()<EOL>for c in graph.successors(b)<EOL>if graph.has_edge(c, a)<EOL>}<EOL>

Get a set of triples representing the 3-cycles from a directional graph.

    Each 3-cycle is returned once, with nodes in sorted order.


def get_contradiction_summary(graph: BELGraph) -> Iterable[Tuple[BaseEntity, BaseEntity, str]]:

for u, v in set(graph.edges()):<EOL><INDENT>relations = {data[RELATION] for data in graph[u][v].values()}<EOL>if relation_set_has_contradictions(relations):<EOL><INDENT>yield u, v, relations<EOL><DEDENT><DEDENT>

Yield triplets of (source node, target node, set of relations) for (source node, target node) pairs
    that have multiple, contradictory relations.


def get_correlation_triangles(graph: BELGraph) -> SetOfNodeTriples:

return {<EOL>tuple(sorted([n, u, v], key=str))<EOL>for n in graph<EOL>for u, v in itt.combinations(graph[n], <NUM_LIT:2>)<EOL>if graph.has_edge(u, v)<EOL>}<EOL>

Return a set of all triangles pointed by the given node.


def get_jens_unstable(graph: BELGraph) -> Iterable[NodeTriple]:

r = jens_transformation_alpha(graph)<EOL>return get_triangles(r)<EOL>

Yield triples of nodes (A, B, C) where ``A -> B``, ``A -| C``, and ``C positiveCorrelation A``.

    Calculated efficiently using the Jens Transformation.


def calculate_concordance_probability_by_annotation(graph, annotation, key, cutoff=None, permutations=None,<EOL>percentage=None,<EOL>use_ambiguous=False):

result = [<EOL>(value, calculate_concordance_probability(<EOL>subgraph,<EOL>key,<EOL>cutoff=cutoff,<EOL>permutations=permutations,<EOL>percentage=percentage,<EOL>use_ambiguous=use_ambiguous,<EOL>))<EOL>for value, subgraph in get_subgraphs_by_annotation(graph, annotation).items()<EOL>]<EOL>return dict(result)<EOL>

Returns the results of concordance analysis on each subgraph, stratified by the given annotation.

    :param pybel.BELGraph graph: A BEL graph
    :param str annotation: The annotation to group by.
    :param str key: The node data dictionary key storing the logFC
    :param float cutoff: The optional logFC cutoff for significance
    :param int permutations: The number of random permutations to test. Defaults to 500
    :param float percentage: The percentage of the graph's edges to maintain. Defaults to 0.9
    :param bool use_ambiguous: Compare to ambiguous edges as well
    :rtype: dict[str,tuple]


def get_cutoff(value: float, cutoff: Optional[float] = None) -> int:

cutoff = cutoff if cutoff is not None else <NUM_LIT:0><EOL>if value > cutoff:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>if value < (-<NUM_LIT:1> * cutoff):<EOL><INDENT>return - <NUM_LIT:1><EOL><DEDENT>return <NUM_LIT:0><EOL>

Assign if a value is greater than or less than a cutoff.


def bel_to_spia_matrices(graph: BELGraph) -> Mapping[str, pd.DataFrame]:

index_nodes = get_matrix_index(graph)<EOL>spia_matrices = build_spia_matrices(index_nodes)<EOL>for u, v, edge_data in graph.edges(data=True):<EOL><INDENT>if isinstance(u, CentralDogma) and isinstance(v, CentralDogma):<EOL><INDENT>update_spia_matrices(spia_matrices, u, v, edge_data)<EOL><DEDENT>elif isinstance(u, CentralDogma) and isinstance(v, ListAbundance):<EOL><INDENT>for node in v.members:<EOL><INDENT>if not isinstance(node, CentralDogma):<EOL><INDENT>continue<EOL><DEDENT>update_spia_matrices(spia_matrices, u, node, edge_data)<EOL><DEDENT><DEDENT>elif isinstance(u, ListAbundance) and isinstance(v, CentralDogma):<EOL><INDENT>for node in u.members:<EOL><INDENT>if not isinstance(node, CentralDogma):<EOL><INDENT>continue<EOL><DEDENT>update_spia_matrices(spia_matrices, node, v, edge_data)<EOL><DEDENT><DEDENT>elif isinstance(u, ListAbundance) and isinstance(v, ListAbundance):<EOL><INDENT>for sub_member, obj_member in product(u.members, v.members):<EOL><INDENT>if isinstance(sub_member, CentralDogma) and isinstance(obj_member, CentralDogma):<EOL><INDENT>update_spia_matrices(spia_matrices, sub_member, obj_member, edge_data)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return spia_matrices<EOL>

Create an excel sheet ready to be used in SPIA software.

    :param graph: BELGraph
    :return: dictionary with matrices


def build_spia_matrices(nodes: Set[str]) -> Dict[str, pd.DataFrame]:

nodes = list(sorted(nodes))<EOL>matrices = OrderedDict()<EOL>for relation in KEGG_RELATIONS:<EOL><INDENT>matrices[relation] = pd.DataFrame(<NUM_LIT:0>, index=nodes, columns=nodes)<EOL><DEDENT>return matrices<EOL>

Build an adjacency matrix for each KEGG relationship and return in a dictionary.

    :param nodes: A set of HGNC gene symbols
    :return: Dictionary of adjacency matrix for each relationship


run_epicom(graph, directory)<EOL>


@main.command()<EOL>@click.option('<STR_LIT>', '<STR_LIT>', type=click.File('<STR_LIT:w>'), default=sys.stdout)<EOL>def multi(output):

graphs = [<EOL>get_ad_graph(),<EOL>get_ep_graph(),<EOL>get_pd_graph(),<EOL>]<EOL>multi_run_epicom(graphs, output)<EOL>


def build_database(manager: pybel.Manager, annotation_url: Optional[str] = None) -> None:

annotation_url = annotation_url or NEUROMMSIG_DEFAULT_URL<EOL>annotation = manager.get_namespace_by_url(annotation_url)<EOL>if annotation is None:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>networks = get_networks_using_annotation(manager, annotation)<EOL>dtis = ...<EOL>for network in networks:<EOL><INDENT>graph = network.as_bel()<EOL>scores = epicom_on_graph(graph, dtis)<EOL>for (drug_name, subgraph_name), score in scores.items():<EOL><INDENT>drug_model = get_drug_model(manager, drug_name)<EOL>subgraph_model = manager.get_annotation_entry(annotation_url, subgraph_name)<EOL>score_model = Score(<EOL>network=network,<EOL>annotation=subgraph_model,<EOL>drug=drug_model,<EOL>score=score<EOL>)<EOL>manager.session.add(score_model)<EOL><DEDENT><DEDENT>t = time.time()<EOL>logger.info('<STR_LIT>')<EOL>manager.session.commit()<EOL>logger.info('<STR_LIT>', time.time() - t)<EOL>

Build a database of scores for NeuroMMSig annotated graphs.

    1. Get all networks that use the Subgraph annotation
    2. run on each


def get_drug_model(manager: pybel.Manager, name: str):

raise NotImplementedError<EOL>

:param pybel.manager.Manager manager:
:param str name:
:return: pybel.manager.models.NamespaceEntry


edge_ranking = default_edge_ranking if edge_ranking is None else edge_ranking<EOL>edges_scores = [<EOL>(edge_id, edge_data[RELATION], edge_ranking[edge_data[RELATION]])<EOL>for edge_id, edge_data in edges.items()<EOL>]<EOL>return max(edges_scores, key=itemgetter(<NUM_LIT:2>))<EOL>

Return the highest ranked edge from a multiedge.

    :param dict edges: dictionary with all edges between two nodes
    :param dict edge_ranking: A dictionary of {relationship: score}
    :return: Highest ranked edge
    :rtype: tuple: (edge id, relation, score given ranking)


def get_path_effect(graph, path, relationship_dict):

causal_effect = []<EOL>for predecessor, successor in pairwise(path):<EOL><INDENT>if pair_has_contradiction(graph, predecessor, successor):<EOL><INDENT>return Effect.ambiguous<EOL><DEDENT>edges = graph.get_edge_data(predecessor, successor)<EOL>edge_key, edge_relation, _ = rank_edges(edges)<EOL>relation = graph[predecessor][successor][edge_key][RELATION]<EOL>if relation not in relationship_dict or relationship_dict[relation] == <NUM_LIT:0>:<EOL><INDENT>return Effect.no_effect<EOL><DEDENT>causal_effect.append(relationship_dict[relation])<EOL><DEDENT>final_effect = reduce(lambda x, y: x * y, causal_effect)<EOL>return Effect.activation if final_effect == <NUM_LIT:1> else Effect.inhibition<EOL>

Calculate the final effect of the root node to the sink node in the path.

    :param pybel.BELGraph graph: A BEL graph
    :param list path: Path from root to sink node
    :param dict relationship_dict: dictionary with relationship effects
    :rtype: Effect


nodes = [<EOL>(n, self.in_out_ratio(n))<EOL>for n in self.unscored_nodes_iter()<EOL>if n != self.target_node<EOL>]<EOL>node, deg = min(nodes, key=itemgetter(<NUM_LIT:1>))<EOL>log.log(<NUM_LIT:5>, '<STR_LIT>', node, deg)<EOL>possible_edges = self.graph.in_edges(node, keys=True)<EOL>log.log(<NUM_LIT:5>, '<STR_LIT>', possible_edges)<EOL>edge_to_remove = random.choice(possible_edges)<EOL>log.log(<NUM_LIT:5>, '<STR_LIT>', edge_to_remove)<EOL>return edge_to_remove<EOL>

This function should be run when there are no leaves, but there are still unscored nodes. It will introduce
        a probabilistic element to the algorithm, where some edges are disregarded randomly to eventually get a score
        for the network. This means that the score can be averaged over many runs for a given graph, and a better
        data structure will have to be later developed that doesn't destroy the graph (instead, annotates which edges
        have been disregarded, later)

           1. get all un-scored
           2. rank by in-degree
           3. weighted probability over all in-edges where lower in-degree means higher probability
           4. pick randomly which edge

        :return: A random in-edge to the lowest in/out degree ratio node. This is a 3-tuple of (node, node, key)
        :rtype: tuple


def multirun(graph: BELGraph,<EOL>node: BaseEntity,<EOL>key: Optional[str] = None,<EOL>tag: Optional[str] = None,<EOL>default_score: Optional[float] = None,<EOL>runs: Optional[int] = None,<EOL>use_tqdm: bool = False,<EOL>) -> Iterable['<STR_LIT>']:

if runs is None:<EOL><INDENT>runs = <NUM_LIT:100><EOL><DEDENT>it = range(runs)<EOL>if use_tqdm:<EOL><INDENT>it = tqdm(it, total=runs)<EOL><DEDENT>for i in it:<EOL><INDENT>try:<EOL><INDENT>runner = Runner(graph, node, key=key, tag=tag, default_score=default_score)<EOL>runner.run()<EOL>yield runner<EOL><DEDENT>except Exception:<EOL><INDENT>log.debug('<STR_LIT>', i, node)<EOL><DEDENT><DEDENT>

Run the heat diffusion workflow multiple times, each time yielding a :class:`Runner` object upon completion.

    :param graph: A BEL graph
    :param node: The BEL node that is the focus of this analysis
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param use_tqdm: Should there be a progress bar for runners?
    :return: An iterable over the runners after each iteration


def calculate_average_score_by_annotation(<EOL>graph: BELGraph,<EOL>annotation: str,<EOL>key: Optional[str] = None,<EOL>runs: Optional[int] = None,<EOL>use_tqdm: bool = False,<EOL>) -> Mapping[str, float]:

candidate_mechanisms = generate_bioprocess_mechanisms(graph, key=key)<EOL>scores: Mapping[BaseEntity, Tuple] = calculate_average_scores_on_subgraphs(<EOL>subgraphs=candidate_mechanisms,<EOL>key=key,<EOL>runs=runs,<EOL>use_tqdm=use_tqdm,<EOL>)<EOL>subgraph_bp: Mapping[str, List[BaseEntity]] = defaultdict(list)<EOL>subgraphs: Mapping[str, BELGraph] = get_subgraphs_by_annotation(graph, annotation)<EOL>for annotation_value, subgraph in subgraphs.items():<EOL><INDENT>subgraph_bp[annotation_value].extend(get_nodes_by_function(subgraph, BIOPROCESS))<EOL><DEDENT>return {<EOL>annotation_value: np.average(scores[bp][<NUM_LIT:0>] for bp in bps)<EOL>for annotation_value, bps in subgraph_bp.items()<EOL>}<EOL>

For each sub-graph induced over the edges matching the annotation, calculate the average score
    for all of the contained biological processes

    Assumes you haven't done anything yet

    1. Generates biological process upstream candidate mechanistic sub-graphs with
       :func:`generate_bioprocess_mechanisms`
    2. Calculates scores for each sub-graph with :func:`calculate_average_scores_on_sub-graphs`
    3. Overlays data with pbt.integration.overlay_data
    4. Calculates averages with pbt.selection.group_nodes.average_node_annotation

    :param graph: A BEL graph
    :param annotation: A BEL annotation
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param use_tqdm: Should there be a progress bar for runners?
    :return: A dictionary from {str annotation value: tuple scores}

    Example Usage:

    >>> import pybel
    >>> from pybel_tools.integration import overlay_data
    >>> from pybel_tools.analysis.heat import calculate_average_score_by_annotation
    >>> graph = pybel.from_path(...)
    >>> scores = calculate_average_score_by_annotation(graph, 'subgraph')


def get_remaining_graph(self) -> BELGraph:

return self.graph.subgraph(self.unscored_nodes_iter())<EOL>

Allows for introspection on the algorithm at a given point by returning the sub-graph induced
        by all unscored nodes

        :return: The remaining un-scored BEL graph


def run_with_graph_transformation(self) -> Iterable[BELGraph]:

yield self.get_remaining_graph()<EOL>while not self.done_chomping():<EOL><INDENT>while not list(self.iter_leaves()):<EOL><INDENT>self.remove_random_edge()<EOL>yield self.get_remaining_graph()<EOL><DEDENT>self.score_leaves()<EOL>yield self.get_remaining_graph()<EOL><DEDENT>

Calculate scores for all leaves until there are none, removes edges until there are, and repeats until
        all nodes have been scored. Also, yields the current graph at every step so you can make a cool animation
        of how the graph changes throughout the course of the algorithm

        :return: An iterable of BEL graphs


def calculate_average_scores_on_subgraphs(<EOL>subgraphs: Mapping[H, BELGraph],<EOL>key: Optional[str] = None,<EOL>tag: Optional[str] = None,<EOL>default_score: Optional[float] = None,<EOL>runs: Optional[int] = None,<EOL>use_tqdm: bool = False,<EOL>tqdm_kwargs: Optional[Mapping[str, Any]] = None,<EOL>) -> Mapping[H, Tuple[float, float, float, float, int, int]]:

results = {}<EOL>log.info('<STR_LIT>', len(subgraphs), runs)<EOL>it = subgraphs.items()<EOL>if use_tqdm:<EOL><INDENT>_tqdm_kwargs = dict(total=len(subgraphs), desc='<STR_LIT>')<EOL>if tqdm_kwargs:<EOL><INDENT>_tqdm_kwargs.update(tqdm_kwargs)<EOL><DEDENT>it = tqdm(it, **_tqdm_kwargs)<EOL><DEDENT>for node, subgraph in it:<EOL><INDENT>number_first_neighbors = subgraph.in_degree(node)<EOL>number_first_neighbors = <NUM_LIT:0> if isinstance(number_first_neighbors, dict) else number_first_neighbors<EOL>mechanism_size = subgraph.number_of_nodes()<EOL>runners = workflow(subgraph, node, key=key, tag=tag, default_score=default_score, runs=runs)<EOL>scores = [runner.get_final_score() for runner in runners]<EOL>if <NUM_LIT:0> == len(scores):<EOL><INDENT>results[node] = (<EOL>None,<EOL>None,<EOL>None,<EOL>None,<EOL>number_first_neighbors,<EOL>mechanism_size,<EOL>)<EOL>continue<EOL><DEDENT>scores = np.array(scores)<EOL>average_score = np.average(scores)<EOL>score_std = np.std(scores)<EOL>med_score = np.median(scores)<EOL>chi_2_stat, norm_p = stats.normaltest(scores)<EOL>results[node] = (<EOL>average_score,<EOL>score_std,<EOL>norm_p,<EOL>med_score,<EOL>number_first_neighbors,<EOL>mechanism_size,<EOL>)<EOL><DEDENT>return results<EOL>

Calculate the scores over precomputed candidate mechanisms.

    :param subgraphs: A dictionary of keys to their corresponding subgraphs
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param use_tqdm: Should there be a progress bar for runners?
    :return: A dictionary of keys to results tuples

    Example Usage:

    >>> import pandas as pd
    >>> from pybel_tools.generation import generate_bioprocess_mechanisms
    >>> from pybel_tools.analysis.heat import calculate_average_scores_on_subgraphs
    >>> # load graph and data
    >>> graph = ...
    >>> candidate_mechanisms = generate_bioprocess_mechanisms(graph)
    >>> scores = calculate_average_scores_on_subgraphs(candidate_mechanisms)
    >>> pd.DataFrame.from_items(scores.items(), orient='index', columns=RESULT_LABELS)


def in_out_ratio(self, node: BaseEntity) -> float:

return self.graph.in_degree(node) / float(self.graph.out_degree(node))<EOL>

Calculate the ratio of in-degree / out-degree of a node.


def score_leaves(self) -> Set[BaseEntity]:

leaves = set(self.iter_leaves())<EOL>if not leaves:<EOL><INDENT>log.warning('<STR_LIT>')<EOL>return set()<EOL><DEDENT>for leaf in leaves:<EOL><INDENT>self.graph.nodes[leaf][self.tag] = self.calculate_score(leaf)<EOL>log.log(<NUM_LIT:5>, '<STR_LIT>', leaf)<EOL><DEDENT>return leaves<EOL>

Calculate the score for all leaves.

        :return: The set of leaf nodes that were scored


def __init__(self,<EOL>graph: BELGraph,<EOL>target_node: BaseEntity,<EOL>key: Optional[str] = None,<EOL>tag: Optional[str] = None,<EOL>default_score: Optional[float] = None,<EOL>) -> None:

self.graph: BELGraph = graph.copy()<EOL>self.target_node = target_node<EOL>self.key = key or WEIGHT<EOL>self.default_score = default_score or DEFAULT_SCORE<EOL>self.tag = tag or SCORE<EOL>for node, data in self.graph.nodes(data=True):<EOL><INDENT>if not self.graph.predecessors(node):<EOL><INDENT>self.graph.nodes[node][self.tag] = data.get(self.key, <NUM_LIT:0>)<EOL>log.log(<NUM_LIT:5>, '<STR_LIT>', target_node, self.graph.nodes[node][self.tag])<EOL><DEDENT><DEDENT>

Initialize the heat diffusion runner class.

        :param graph: A BEL graph
        :param target_node: The BEL node that is the focus of this analysis
        :param key: The key in the node data dictionary representing the experimental data. Defaults to
         :data:`pybel_tools.constants.WEIGHT`.
        :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
        :param default_score: The initial score for all nodes. This number can go up or down.


def workflow_all(graph: BELGraph,<EOL>key: Optional[str] = None,<EOL>tag: Optional[str] = None,<EOL>default_score: Optional[float] = None,<EOL>runs: Optional[int] = None,<EOL>) -> Mapping[BaseEntity, List[Runner]]:

results = {}<EOL>for node in get_nodes_by_function(graph, BIOPROCESS):<EOL><INDENT>results[node] = workflow(graph, node, key=key, tag=tag, default_score=default_score, runs=runs)<EOL><DEDENT>return results<EOL>

Run the heat diffusion workflow and get runners for every possible candidate mechanism

    1. Get all biological processes
    2. Get candidate mechanism induced two level back from each biological process
    3. Heat diffusion workflow for each candidate mechanism for multiple runs
    4. Return all runner results

    :param graph: A BEL graph
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :return: A dictionary of {node: list of runners}


def calculate_average_scores_on_graph(<EOL>graph: BELGraph,<EOL>key: Optional[str] = None,<EOL>tag: Optional[str] = None,<EOL>default_score: Optional[float] = None,<EOL>runs: Optional[int] = None,<EOL>use_tqdm: bool = False,<EOL>):

subgraphs = generate_bioprocess_mechanisms(graph, key=key)<EOL>scores = calculate_average_scores_on_subgraphs(<EOL>subgraphs,<EOL>key=key,<EOL>tag=tag,<EOL>default_score=default_score,<EOL>runs=runs,<EOL>use_tqdm=use_tqdm<EOL>)<EOL>return scores<EOL>

Calculate the scores over all biological processes in the sub-graph.

    As an implementation, it simply computes the sub-graphs then calls :func:`calculate_average_scores_on_subgraphs` as
    described in that function's documentation.

    :param graph: A BEL graph with heats already on the nodes
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param use_tqdm: Should there be a progress bar for runners?
    :return: A dictionary of {pybel node tuple: results tuple}
    :rtype: dict[tuple, tuple]

    Suggested usage with :mod:`pandas`:

    >>> import pandas as pd
    >>> from pybel_tools.analysis.heat import calculate_average_scores_on_graph
    >>> graph = ...  # load graph and data
    >>> scores = calculate_average_scores_on_graph(graph)
    >>> pd.DataFrame.from_items(scores.items(), orient='index', columns=RESULT_LABELS)


return self.tag in self.graph.nodes[self.target_node]<EOL>

Determines if the algorithm is complete by checking if the target node of this analysis has been scored
        yet. Because the algorithm removes edges when it gets stuck until it is un-stuck, it is always guaranteed to
        finish.

        :return: Is the algorithm done running?


if os.path.exists(self.path):<EOL><INDENT>os.remove(self.path)<EOL><DEDENT>

Called after each test to remove the file


<EOL>if isinstance(data, str):<EOL><INDENT>data = json.loads(data)<EOL><DEDENT>if os.path.exists(path):<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>f = File(path)<EOL>f.data = data<EOL>return f<EOL><DEDENT>

Initialize a new file that starts out with some data. Pass data
        as a list, dict, or JSON string.


data = {} if data.lower() == "<STR_LIT>" else []<EOL>if not os.path.exists(path):  <EOL><INDENT>dirname = os.path.dirname(path)<EOL>if dirname and not os.path.exists(dirname):<EOL><INDENT>raise IOError(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>").format(os.path.dirname(path))<EOL>)<EOL><DEDENT>with open(path, "<STR_LIT:w>") as f:<EOL><INDENT>json.dump(data, f)<EOL><DEDENT>return True<EOL><DEDENT>elif os.path.getsize(path) == <NUM_LIT:0>:  <EOL><INDENT>with open(path, "<STR_LIT:w>") as f:<EOL><INDENT>json.dump(data, f)<EOL><DEDENT><DEDENT>else:  <EOL><INDENT>return False<EOL><DEDENT>


return hasattr(self, "<STR_LIT>")<EOL>

Returns a boolean value describing whether a grouped write is
        underway.


def get_reference_data(<EOL>self,<EOL>modified_since: Optional[datetime.datetime] = None<EOL>) -> GetReferenceDataResponse:

if modified_since is None:<EOL><INDENT>modified_since = datetime.datetime(year=<NUM_LIT>, month=<NUM_LIT:1>, day=<NUM_LIT:1>)<EOL><DEDENT>response = requests.get(<EOL>'<STR_LIT>'.format(API_URL_BASE),<EOL>headers={<EOL>'<STR_LIT>': self._format_dt(modified_since),<EOL>**self._get_headers(),<EOL>},<EOL>timeout=self._timeout,<EOL>)<EOL>if not response.ok:<EOL><INDENT>raise FuelCheckError.create(response)<EOL><DEDENT>return GetReferenceDataResponse.deserialize(response.json())<EOL>

Fetches API reference data.

:param modified_since: The response will be empty if no
changes have been made to the reference data since this
timestamp, otherwise all reference data will be returned.


def get_fuel_prices_for_station(<EOL>self,<EOL>station: int<EOL>) -> List[Price]:

response = requests.get(<EOL>'<STR_LIT>'.format(API_URL_BASE, station),<EOL>headers=self._get_headers(),<EOL>timeout=self._timeout,<EOL>)<EOL>if not response.ok:<EOL><INDENT>raise FuelCheckError.create(response)<EOL><DEDENT>data = response.json()<EOL>return [Price.deserialize(data) for data in data['<STR_LIT>']]<EOL>

Gets the fuel prices for a specific fuel station.


def generate_test_func(name, original_function, num, params):<EOL><INDENT>if original_function._provider_name_suffix:<EOL><INDENT>data_name = params[<NUM_LIT:0>]<EOL>params = params[<NUM_LIT:1>:]<EOL><DEDENT>else:<EOL><INDENT>data_name = num<EOL><DEDENT>expanded_name = '<STR_LIT>' % (name, data_name)<EOL>def generated_test_function(self):<EOL><INDENT>original_function(self, *params)<EOL><DEDENT>setattr(cls, expanded_name, generated_test_function)<EOL><DEDENT>for name in dir(cls):<EOL><INDENT>func = getattr(cls, name)<EOL>if hasattr(func, '<STR_LIT>'):<EOL><INDENT>num = <NUM_LIT:1><EOL>for params in getattr(cls, func._provider_method)():<EOL><INDENT>generate_test_func(name, func, num, params)<EOL>num += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>return cls<EOL>

A class decorator that works with the @provider decorator to generate test
method from a data provider


if not self.socket:<EOL><INDENT>return<EOL><DEDENT>self.socket.close()<EOL>self.socket = None<EOL>


if self.socket:<EOL><INDENT>return True<EOL><DEDENT>host, port = self.url_info<EOL>session = tls.TLSSession()<EOL>self.socket = tls.TLSSocket(host, port, timeout=self.timeout, session=session)<EOL>return False<EOL>

Make sure a valid tls.TLSSocket() is open to the server

:return:
    A boolean indicating if the connection was reused


setup_file = os.path.join(package_root, '<STR_LIT>')<EOL>git_wc_proc = subprocess.Popen(<EOL>['<STR_LIT>', '<STR_LIT:status>', '<STR_LIT>', '<STR_LIT>'],<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.STDOUT,<EOL>cwd=package_root<EOL>)<EOL>git_wc_status, _ = git_wc_proc.communicate()<EOL>if len(git_wc_status) > <NUM_LIT:0>:<EOL><INDENT>print(git_wc_status.decode('<STR_LIT:utf-8>').rstrip(), file=sys.stderr)<EOL>print('<STR_LIT>', file=sys.stderr)<EOL>return False<EOL><DEDENT>git_tag_proc = subprocess.Popen(<EOL>['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'],<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE,<EOL>cwd=package_root<EOL>)<EOL>tag, tag_error = git_tag_proc.communicate()<EOL>if len(tag_error) > <NUM_LIT:0>:<EOL><INDENT>print(tag_error.decode('<STR_LIT:utf-8>').rstrip(), file=sys.stderr)<EOL>print('<STR_LIT>', file=sys.stderr)<EOL>return False<EOL><DEDENT>if len(tag) == <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>', file=sys.stderr)<EOL>return False<EOL><DEDENT>tag = tag.decode('<STR_LIT:ascii>').strip()<EOL>setuptools.sandbox.run_setup(<EOL>setup_file,<EOL>['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>)<EOL>twine.cli.dispatch(['<STR_LIT>', '<STR_LIT>' % (package_name, tag)])<EOL>setuptools.sandbox.run_setup(<EOL>setup_file,<EOL>['<STR_LIT>']<EOL>)<EOL>

Creates a sdist .tar.gz and a bdist_wheel --univeral .whl and uploads
them to pypi

:return:
    A bool - if the packaging and upload process was successful


print('<STR_LIT>')<EOL>md_files = []<EOL>for root, _, filenames in os.walk(os.path.join(package_root, '<STR_LIT>')):<EOL><INDENT>for filename in filenames:<EOL><INDENT>if not filename.endswith('<STR_LIT>'):<EOL><INDENT>continue<EOL><DEDENT>md_files.append(os.path.join(root, filename))<EOL><DEDENT><DEDENT>parser = CommonMark.Parser()<EOL>for md_file in md_files:<EOL><INDENT>md_file_relative = md_file[len(package_root) + <NUM_LIT:1>:]<EOL>if md_file_relative in md_source_map:<EOL><INDENT>py_files = md_source_map[md_file_relative]<EOL>py_paths = [os.path.join(package_root, py_file) for py_file in py_files]<EOL><DEDENT>else:<EOL><INDENT>py_files = [os.path.basename(md_file).replace('<STR_LIT>', '<STR_LIT>')]<EOL>py_paths = [os.path.join(package_root, package_name, py_files[<NUM_LIT:0>])]<EOL>if not os.path.exists(py_paths[<NUM_LIT:0>]):<EOL><INDENT>continue<EOL><DEDENT><DEDENT>with open(md_file, '<STR_LIT:rb>') as f:<EOL><INDENT>markdown = f.read().decode('<STR_LIT:utf-8>')<EOL><DEDENT>original_markdown = markdown<EOL>md_lines = list(markdown.splitlines())<EOL>md_ast = parser.parse(markdown)<EOL>last_class = []<EOL>last = {}<EOL>sections = OrderedDict()<EOL>find_sections(md_ast, sections, last, last_class, markdown.count("<STR_LIT:\n>") + <NUM_LIT:1>)<EOL>md_chunks = {}<EOL>for index, py_file in enumerate(py_files):<EOL><INDENT>py_path = py_paths[index]<EOL>with open(os.path.join(py_path), '<STR_LIT:rb>') as f:<EOL><INDENT>code = f.read().decode('<STR_LIT:utf-8>')<EOL>module_ast = ast.parse(code, filename=py_file)<EOL>code_lines = list(code.splitlines())<EOL><DEDENT>for node in ast.iter_child_nodes(module_ast):<EOL><INDENT>walk_ast(node, code_lines, sections, md_chunks)<EOL><DEDENT><DEDENT>added_lines = <NUM_LIT:0><EOL>def _replace_md(key, sections, md_chunk, md_lines, added_lines):<EOL><INDENT>start, end = sections[key]<EOL>start -= <NUM_LIT:1><EOL>start += added_lines<EOL>end += added_lines<EOL>new_lines = md_chunk.split('<STR_LIT:\n>')<EOL>added_lines += len(new_lines) - (end - start)<EOL>if start > <NUM_LIT:0> and md_lines[start][<NUM_LIT:0>:<NUM_LIT:4>] == '<STR_LIT>' and md_lines[start - <NUM_LIT:1>][<NUM_LIT:0>:<NUM_LIT:1>] == '<STR_LIT:>>':<EOL><INDENT>added_lines += <NUM_LIT:1><EOL>new_lines.insert(<NUM_LIT:0>, '<STR_LIT>')<EOL><DEDENT>md_lines[start:end] = new_lines<EOL>return added_lines<EOL><DEDENT>for key in sections:<EOL><INDENT>if key not in md_chunks:<EOL><INDENT>raise ValueError('<STR_LIT>' % key[<NUM_LIT:1>])<EOL><DEDENT>added_lines = _replace_md(key, sections, md_chunks[key], md_lines, added_lines)<EOL><DEDENT>markdown = '<STR_LIT:\n>'.join(md_lines).strip() + '<STR_LIT:\n>'<EOL>if original_markdown != markdown:<EOL><INDENT>with open(md_file, '<STR_LIT:wb>') as f:<EOL><INDENT>f.write(markdown.encode('<STR_LIT:utf-8>'))<EOL><DEDENT><DEDENT><DEDENT>

Looks through the docs/ dir and parses each markdown document, looking for
sections to update from Python docstrings. Looks for section headers in
the format:

 - ### `ClassName()` class
 - ##### `.method_name()` method
 - ##### `.attribute_name` attribute
 - ### `function_name()` function

The markdown content following these section headers up until the next
section header will be replaced by new markdown generated from the Python
docstrings of the associated source files.

By default maps docs/{name}.md to {modulename}/{name}.py. Allows for
custom mapping via the md_source_map variable.


def _find_sections(md_ast, sections, last, last_class, total_lines=None):

def child_walker(node):<EOL><INDENT>for child, entering in node.walker():<EOL><INDENT>if child == node:<EOL><INDENT>continue<EOL><DEDENT>yield child, entering<EOL><DEDENT><DEDENT>for child, entering in child_walker(md_ast):<EOL><INDENT>if child.t == '<STR_LIT>':<EOL><INDENT>start_line = child.sourcepos[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>if child.level == <NUM_LIT:2>:<EOL><INDENT>if last:<EOL><INDENT>sections[(last['<STR_LIT>'], last['<STR_LIT>'])] = (last['<STR_LIT>'], start_line - <NUM_LIT:1>)<EOL>last.clear()<EOL><DEDENT><DEDENT>if child.level in set([<NUM_LIT:3>, <NUM_LIT:5>]):<EOL><INDENT>heading_elements = []<EOL>for heading_child, _ in child_walker(child):<EOL><INDENT>heading_elements.append(heading_child)<EOL><DEDENT>if len(heading_elements) != <NUM_LIT:2>:<EOL><INDENT>continue<EOL><DEDENT>first = heading_elements[<NUM_LIT:0>]<EOL>second = heading_elements[<NUM_LIT:1>]<EOL>if first.t != '<STR_LIT:code>':<EOL><INDENT>continue<EOL><DEDENT>if second.t != '<STR_LIT:text>':<EOL><INDENT>continue<EOL><DEDENT>type_name = second.literal.strip()<EOL>identifier = first.literal.strip().replace('<STR_LIT>', '<STR_LIT>').lstrip('<STR_LIT:.>')<EOL>if last:<EOL><INDENT>sections[(last['<STR_LIT>'], last['<STR_LIT>'])] = (last['<STR_LIT>'], start_line - <NUM_LIT:1>)<EOL>last.clear()<EOL><DEDENT>if type_name == '<STR_LIT>':<EOL><INDENT>if child.level != <NUM_LIT:3>:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>if type_name == '<STR_LIT:class>':<EOL><INDENT>if child.level != <NUM_LIT:3>:<EOL><INDENT>continue<EOL><DEDENT>last_class.append(identifier)<EOL><DEDENT>if type_name in set(['<STR_LIT>', '<STR_LIT>']):<EOL><INDENT>if child.level != <NUM_LIT:5>:<EOL><INDENT>continue<EOL><DEDENT>identifier = last_class[-<NUM_LIT:1>] + '<STR_LIT:.>' + identifier<EOL><DEDENT>last.update({<EOL>'<STR_LIT>': type_name,<EOL>'<STR_LIT>': identifier,<EOL>'<STR_LIT>': start_line,<EOL>})<EOL><DEDENT><DEDENT>elif child.t == '<STR_LIT>':<EOL><INDENT>find_sections(child, sections, last, last_class)<EOL><DEDENT><DEDENT>if last:<EOL><INDENT>sections[(last['<STR_LIT>'], last['<STR_LIT>'])] = (last['<STR_LIT>'], total_lines)<EOL><DEDENT>

Walks through a CommonMark AST to find section headers that delineate
content that should be updated by this script

:param md_ast:
    The AST of the markdown document

:param sections:
    A dict to store the start and end lines of a section. The key will be
    a two-element tuple of the section type ("class", "function",
    "method" or "attribute") and identifier. The values are a two-element
    tuple of the start and end line number in the markdown document of the
    section.

:param last:
    A dict containing information about the last section header seen.
    Includes the keys "type_name", "identifier", "start_line".

:param last_class:
    A unicode string of the name of the last class found - used when
    processing methods and attributes.

:param total_lines:
    An integer of the total number of lines in the markdown document -
    used to work around a bug in the API of the Python port of CommonMark


try:<EOL><INDENT>import _manylinux<EOL>return bool(_manylinux.manylinux1_compatible)<EOL><DEDENT>except (ImportError, AttributeError):<EOL><INDENT>pass<EOL><DEDENT>try:<EOL><INDENT>proc = ctypes.CDLL(None)<EOL>gnu_get_libc_version = proc.gnu_get_libc_version<EOL>gnu_get_libc_version.restype = ctypes.c_char_p<EOL>ver = gnu_get_libc_version()<EOL>if not isinstance(ver, str_cls):<EOL><INDENT>ver = ver.decode('<STR_LIT:ascii>')<EOL><DEDENT>match = re.match(r'<STR_LIT>', ver)<EOL>return match and match.group(<NUM_LIT:1>) == '<STR_LIT:2>' and int(match.group(<NUM_LIT:2>)) >= <NUM_LIT:5><EOL><DEDENT>except (AttributeError):<EOL><INDENT>return False<EOL><DEDENT>

:return:
    A boolean indicating if the machine can use manylinux1 packages


tags = []<EOL>versions = []<EOL>version_info = _pep425_version()<EOL>major = version_info[:-<NUM_LIT:1>]<EOL>for minor in range(version_info[-<NUM_LIT:1>], -<NUM_LIT:1>, -<NUM_LIT:1>):<EOL><INDENT>versions.append('<STR_LIT>'.join(map(str, major + (minor,))))<EOL><DEDENT>impl = _pep425_implementation()<EOL>abis = []<EOL>abi = _pep425_get_abi()<EOL>if abi:<EOL><INDENT>abis.append(abi)<EOL><DEDENT>abi3 = _pep425_implementation() == '<STR_LIT>' and sys.version_info >= (<NUM_LIT:3>,)<EOL>if abi3:<EOL><INDENT>abis.append('<STR_LIT>')<EOL><DEDENT>abis.append('<STR_LIT:none>')<EOL>if sys.platform == '<STR_LIT>':<EOL><INDENT>plat_ver = platform.mac_ver()<EOL>ver_parts = plat_ver[<NUM_LIT:0>].split('<STR_LIT:.>')<EOL>minor = int(ver_parts[<NUM_LIT:1>])<EOL>arch = plat_ver[<NUM_LIT:2>]<EOL>if sys.maxsize == <NUM_LIT>:<EOL><INDENT>arch = '<STR_LIT>'<EOL><DEDENT>arches = []<EOL>while minor > <NUM_LIT:5>:<EOL><INDENT>arches.append('<STR_LIT>' % (minor, arch))<EOL>arches.append('<STR_LIT>' % (minor,))<EOL>arches.append('<STR_LIT>' % (minor,))<EOL>minor -= <NUM_LIT:1><EOL><DEDENT><DEDENT>else:<EOL><INDENT>if sys.platform == '<STR_LIT:win32>':<EOL><INDENT>if '<STR_LIT>' in sys.version.lower():<EOL><INDENT>arches = ['<STR_LIT>']<EOL><DEDENT>arches = [sys.platform]<EOL><DEDENT>elif hasattr(os, '<STR_LIT>'):<EOL><INDENT>(plat, _, _, _, machine) = os.uname()<EOL>plat = plat.lower().replace('<STR_LIT:/>', '<STR_LIT>')<EOL>machine.replace('<STR_LIT:U+0020>', '<STR_LIT:_>').replace('<STR_LIT:/>', '<STR_LIT:_>')<EOL>if plat == '<STR_LIT>' and sys.maxsize == <NUM_LIT>:<EOL><INDENT>machine = '<STR_LIT>'<EOL><DEDENT>arch = '<STR_LIT>' % (plat, machine)<EOL>if _pep425_supports_manylinux():<EOL><INDENT>arches = [arch.replace('<STR_LIT>', '<STR_LIT>'), arch]<EOL><DEDENT>else:<EOL><INDENT>arches = [arch]<EOL><DEDENT><DEDENT><DEDENT>for abi in abis:<EOL><INDENT>for arch in arches:<EOL><INDENT>tags.append(('<STR_LIT>' % (impl, versions[<NUM_LIT:0>]), abi, arch))<EOL><DEDENT><DEDENT>if abi3:<EOL><INDENT>for version in versions[<NUM_LIT:1>:]:<EOL><INDENT>for arch in arches:<EOL><INDENT>tags.append(('<STR_LIT>' % (impl, version), '<STR_LIT>', arch))<EOL><DEDENT><DEDENT><DEDENT>for arch in arches:<EOL><INDENT>tags.append(('<STR_LIT>' % (versions[<NUM_LIT:0>][<NUM_LIT:0>]), '<STR_LIT:none>', arch))<EOL><DEDENT>tags.append(('<STR_LIT>' % (impl, versions[<NUM_LIT:0>]), '<STR_LIT:none>', '<STR_LIT>'))<EOL>tags.append(('<STR_LIT>' % (impl, versions[<NUM_LIT:0>][<NUM_LIT:0>]), '<STR_LIT:none>', '<STR_LIT>'))<EOL>for i, version in enumerate(versions):<EOL><INDENT>tags.append(('<STR_LIT>' % (version,), '<STR_LIT:none>', '<STR_LIT>'))<EOL>if i == <NUM_LIT:0>:<EOL><INDENT>tags.append(('<STR_LIT>' % (version[<NUM_LIT:0>]), '<STR_LIT:none>', '<STR_LIT>'))<EOL><DEDENT><DEDENT>tags.append(('<STR_LIT>', '<STR_LIT:none>', '<STR_LIT>'))<EOL>return tags<EOL>

:return:
    A list of 3-element tuples with unicode strings or None:
     [0] implementation tag - cp33, pp27, cp26, py2, py2.py3
     [1] abi tag - cp26m, None
     [2] arch tag - linux_x86_64, macosx_10_10_x85_64, etc


dir_patterns, file_patterns = _gitignore(root)<EOL>paths = []<EOL>prefix = os.path.abspath(root) + os.sep<EOL>for base, dirs, files in os.walk(root):<EOL><INDENT>for d in dirs:<EOL><INDENT>for dir_pattern in dir_patterns:<EOL><INDENT>if fnmatch(d, dir_pattern):<EOL><INDENT>dirs.remove(d)<EOL>break<EOL><DEDENT><DEDENT><DEDENT>for f in files:<EOL><INDENT>skip = False<EOL>for file_pattern in file_patterns:<EOL><INDENT>if fnmatch(f, file_pattern):<EOL><INDENT>skip = True<EOL>break<EOL><DEDENT><DEDENT>if skip:<EOL><INDENT>continue<EOL><DEDENT>full_path = os.path.join(base, f)<EOL>if full_path[:len(prefix)] == prefix:<EOL><INDENT>full_path = full_path[len(prefix):]<EOL><DEDENT>paths.append(full_path)<EOL><DEDENT><DEDENT>return sorted(paths)<EOL>

Lists all of the files in a directory, taking into account any .gitignore
file that is present

:param root:
    A unicode filesystem path

:return:
    A list of unicode strings, containing paths of all files not ignored
    by .gitignore with root, using relative paths


def _do_request(method, url, headers, data=None, query_params=None, timeout=<NUM_LIT:20>):

if query_params:<EOL><INDENT>url += '<STR_LIT:?>' + urlencode(query_params).replace('<STR_LIT:+>', '<STR_LIT>')<EOL><DEDENT>if isinstance(data, dict):<EOL><INDENT>data_bytes = {}<EOL>for key in data:<EOL><INDENT>data_bytes[key.encode('<STR_LIT:utf-8>')] = data[key].encode('<STR_LIT:utf-8>')<EOL><DEDENT>data = urlencode(data_bytes)<EOL>headers['<STR_LIT:Content-Type>'] = '<STR_LIT>'<EOL><DEDENT>if isinstance(data, str_cls):<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>try:<EOL><INDENT>tempfd, tempf_path = tempfile.mkstemp('<STR_LIT>')<EOL>os.write(tempfd, data or b'<STR_LIT>')<EOL>os.close(tempfd)<EOL>if sys.platform == '<STR_LIT:win32>':<EOL><INDENT>powershell_exe = os.path.join('<STR_LIT>')<EOL>code = "<STR_LIT>"<EOL>code += "<STR_LIT>"<EOL>for key in headers:<EOL><INDENT>code += "<STR_LIT>" % (key, headers[key])<EOL><DEDENT>code += "<STR_LIT>" % (url, method, tempf_path)<EOL>code += "<STR_LIT>"<EOL>code += "<STR_LIT>"<EOL>stdout, stderr = _execute([powershell_exe, '<STR_LIT>', code], os.getcwd())<EOL>if stdout[-<NUM_LIT:2>:] == b'<STR_LIT:\r\n>' and b'<STR_LIT>' in stdout:<EOL><INDENT>stdout = stdout[<NUM_LIT:0>:-<NUM_LIT:2>]<EOL>parts = stdout.split(b'<STR_LIT>', <NUM_LIT:1>)<EOL>if len(parts) == <NUM_LIT:2>:<EOL><INDENT>stdout = parts[<NUM_LIT:0>] + b'<STR_LIT>' + codecs.decode(parts[<NUM_LIT:1>].replace(b'<STR_LIT:->', b'<STR_LIT>'), '<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>args = [<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>method,<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>'<EOL>]<EOL>for key in headers:<EOL><INDENT>args.append('<STR_LIT>')<EOL>args.append("<STR_LIT>" % (key, headers[key]))<EOL><DEDENT>args.append('<STR_LIT>')<EOL>args.append('<STR_LIT>' % tempf_path)<EOL>args.append(url)<EOL>stdout, stderr = _execute(args, os.getcwd())<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>if tempf_path and os.path.exists(tempf_path):<EOL><INDENT>os.remove(tempf_path)<EOL><DEDENT><DEDENT>if len(stderr) > <NUM_LIT:0>:<EOL><INDENT>raise URLError("<STR_LIT>" % (method, url, stderr))<EOL><DEDENT>parts = stdout.split(b'<STR_LIT>', <NUM_LIT:1>)<EOL>if len(parts) != <NUM_LIT:2>:<EOL><INDENT>raise URLError("<STR_LIT>" % (method, url, stdout))<EOL><DEDENT>header_block, body = parts<EOL>content_type_header = None<EOL>content_len_header = None<EOL>for hline in header_block.decode('<STR_LIT>').splitlines():<EOL><INDENT>hline_parts = hline.split('<STR_LIT::>', <NUM_LIT:1>)<EOL>if len(hline_parts) != <NUM_LIT:2>:<EOL><INDENT>continue<EOL><DEDENT>name, val = hline_parts<EOL>name = name.strip().lower()<EOL>val = val.strip()<EOL>if name == '<STR_LIT>':<EOL><INDENT>content_type_header = val<EOL><DEDENT>if name == '<STR_LIT>':<EOL><INDENT>content_len_header = val<EOL><DEDENT><DEDENT>if content_type_header is None and content_len_header != '<STR_LIT:0>':<EOL><INDENT>raise URLError("<STR_LIT>" % (method, url, stdout))<EOL><DEDENT>if content_type_header is None:<EOL><INDENT>content_type = '<STR_LIT>'<EOL>encoding = '<STR_LIT:utf-8>'<EOL><DEDENT>else:<EOL><INDENT>content_type, params = cgi.parse_header(content_type_header)<EOL>encoding = params.get('<STR_LIT>')<EOL><DEDENT>return (content_type, encoding, body)<EOL>

Performs an HTTP request

:param method:
    A unicode string of 'POST' or 'PUT'

:param url;
    A unicode string of the URL to request

:param headers:
    A dict of unicode strings, where keys are header names and values are
    the header values.

:param data:
    A dict of unicode strings (to be encoded as
    application/x-www-form-urlencoded), or a byte string of data.

:param query_params:
    A dict of unicode keys and values to pass as query params

:param timeout:
    An integer number of seconds to use as the timeout

:return:
    A 3-element tuple:
     - 0: A unicode string of the response content-type
     - 1: A unicode string of the response encoding, or None
     - 2: A byte string of the response body


xml_report_path = os.path.join(package_root, '<STR_LIT>')<EOL>if os.path.exists(xml_report_path):<EOL><INDENT>os.unlink(xml_report_path)<EOL><DEDENT>cov = coverage.Coverage(include='<STR_LIT>' % package_name)<EOL>cov.start()<EOL>from .tests import run as run_tests<EOL>result = run_tests()<EOL>print()<EOL>if ci:<EOL><INDENT>suite = unittest.TestSuite()<EOL>loader = unittest.TestLoader()<EOL>for other_package in other_packages:<EOL><INDENT>for test_class in _load_package_tests(other_package):<EOL><INDENT>suite.addTest(loader.loadTestsFromTestCase(test_class))<EOL><DEDENT><DEDENT>if suite.countTestCases() > <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>')<EOL>sys.stdout.flush()<EOL>runner_result = unittest.TextTestRunner(stream=sys.stdout, verbosity=<NUM_LIT:1>).run(suite)<EOL>result = runner_result.wasSuccessful() and result<EOL>print()<EOL>sys.stdout.flush()<EOL><DEDENT><DEDENT>cov.stop()<EOL>cov.save()<EOL>cov.report(show_missing=False)<EOL>print()<EOL>sys.stdout.flush()<EOL>if ci:<EOL><INDENT>cov.xml_report()<EOL><DEDENT>if ci and result and os.path.exists(xml_report_path):<EOL><INDENT>_codecov_submit()<EOL>print()<EOL><DEDENT>return result<EOL>

Runs the tests while measuring coverage

:param ci:
    If coverage is being run in a CI environment - this triggers trying to
    run the tests for the rest of modularcrypto and uploading coverage data

:return:
    A bool - if the tests ran successfully


output = {}<EOL>for line in data.splitlines():<EOL><INDENT>line = line.strip()<EOL>if not line or '<STR_LIT:=>' not in line:<EOL><INDENT>continue<EOL><DEDENT>parts = line.split('<STR_LIT:=>')<EOL>if len(parts) != <NUM_LIT:2>:<EOL><INDENT>continue<EOL><DEDENT>name = parts[<NUM_LIT:0>]<EOL>value = parts[<NUM_LIT:1>]<EOL>if len(value) > <NUM_LIT:1>:<EOL><INDENT>if value[<NUM_LIT:0>] == '<STR_LIT:">' and value[-<NUM_LIT:1>] == '<STR_LIT:">':<EOL><INDENT>value = value[<NUM_LIT:1>:-<NUM_LIT:1>]<EOL><DEDENT><DEDENT>output[name] = value<EOL><DEDENT>return output<EOL>

Parses a basic VAR="value data" file contents into a dict

:param data:
    A unicode string of the file data

:return:
    A dict of parsed name/value data


if isinstance(archive, zipfile.ZipFile):<EOL><INDENT>fn = info.filename<EOL>is_dir = fn.endswith('<STR_LIT:/>') or fn.endswith('<STR_LIT:\\>')<EOL>out = archive.read(info)<EOL>if is_dir and out == b'<STR_LIT>':<EOL><INDENT>return None<EOL><DEDENT>return out<EOL><DEDENT>info_file = archive.extractfile(info)<EOL>if info_file:<EOL><INDENT>return info_file.read()<EOL><DEDENT>return None<EOL>

Extracts the contents of an archive info object

;param archive:
    An archive from _open_archive()

:param info:
    An info object from _list_archive_members()

:return:
    None, or a byte string of the file contents


if pkg_path.endswith('<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>zf = None<EOL>zf = zipfile.ZipFile(pkg_path, '<STR_LIT:r>')<EOL>for zi in zf.infolist():<EOL><INDENT>if not zi.filename.startswith('<STR_LIT>'):<EOL><INDENT>continue<EOL><DEDENT>data = _extract_info(zf, zi)<EOL>if data is not None:<EOL><INDENT>dst_path = os.path.join(deps_dir, zi.filename[<NUM_LIT:8>:])<EOL>dst_dir = os.path.dirname(dst_path)<EOL>if not os.path.exists(dst_dir):<EOL><INDENT>os.makedirs(dst_dir)<EOL><DEDENT>with open(dst_path, '<STR_LIT:wb>') as f:<EOL><INDENT>f.write(data)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>finally:<EOL><INDENT>if zf:<EOL><INDENT>zf.close()<EOL><DEDENT><DEDENT>return<EOL><DEDENT>if pkg_path.endswith('<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>zf = None<EOL>zf = zipfile.ZipFile(pkg_path, '<STR_LIT:r>')<EOL>zf.extractall(deps_dir)<EOL><DEDENT>finally:<EOL><INDENT>if zf:<EOL><INDENT>zf.close()<EOL><DEDENT><DEDENT>return<EOL><DEDENT>try:<EOL><INDENT>ar = None<EOL>ar = _open_archive(pkg_path)<EOL>pkg_name = None<EOL>base_path = _archive_single_dir(ar) or '<STR_LIT>'<EOL>if len(base_path):<EOL><INDENT>if '<STR_LIT:->' in base_path:<EOL><INDENT>pkg_name, _ = base_path.split('<STR_LIT:->', <NUM_LIT:1>)<EOL><DEDENT>base_path += '<STR_LIT:/>'<EOL><DEDENT>base_pkg_path = None<EOL>if pkg_name is not None:<EOL><INDENT>base_pkg_path = base_path + pkg_name + '<STR_LIT:/>'<EOL><DEDENT>src_path = base_path + '<STR_LIT>'<EOL>members = []<EOL>for info in _list_archive_members(ar):<EOL><INDENT>fn = _info_name(info)<EOL>if base_pkg_path is not None and fn.startswith(base_pkg_path):<EOL><INDENT>dst_path = fn[len(base_pkg_path) - len(pkg_name) - <NUM_LIT:1>:]<EOL>members.append((info, dst_path))<EOL>continue<EOL><DEDENT>if fn.startswith(src_path):<EOL><INDENT>members.append((info, fn[len(src_path):]))<EOL>continue<EOL><DEDENT><DEDENT>for info, path in members:<EOL><INDENT>info_data = _extract_info(ar, info)<EOL>if info_data is not None:<EOL><INDENT>dst_path = os.path.join(deps_dir, path)<EOL>dst_dir = os.path.dirname(dst_path)<EOL>if not os.path.exists(dst_dir):<EOL><INDENT>os.makedirs(dst_dir)<EOL><DEDENT>with open(dst_path, '<STR_LIT:wb>') as f:<EOL><INDENT>f.write(info_data)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>finally:<EOL><INDENT>if ar:<EOL><INDENT>ar.close()<EOL><DEDENT><DEDENT>

Extract a .whl, .zip, .tar.gz or .tar.bz2 into a package path to
use when running CI tasks

:param deps_dir:
    A unicode string of the directory the package should be extracted to

:param pkg_path:
    A unicode string of the path to the archive


if isinstance(info, zipfile.ZipInfo):<EOL><INDENT>return info.filename.replace('<STR_LIT:\\>', '<STR_LIT:/>')<EOL><DEDENT>return info.name.replace('<STR_LIT:\\>', '<STR_LIT:/>')<EOL>

Returns a normalized file path for an archive info object

:param info:
    An info object from _list_archive_members()

:return:
    A unicode string with all directory separators normalized to "/"


proc = subprocess.Popen(<EOL>params,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE,<EOL>cwd=cwd<EOL>)<EOL>stdout, stderr = proc.communicate()<EOL>code = proc.wait()<EOL>if code != <NUM_LIT:0>:<EOL><INDENT>e = OSError('<STR_LIT>' % (params, code, stderr))<EOL>e.stdout = stdout<EOL>e.stderr = stderr<EOL>raise e<EOL><DEDENT>return (stdout, stderr)<EOL>

Executes a subprocess

:param params:
    A list of the executable and arguments to pass to it

:param cwd:
    The working directory to execute the command in

:return:
    A 2-element tuple of (stdout, stderr)


python_version = '<STR_LIT:.>'.join(map(str_cls, sys.version_info[<NUM_LIT:0>:<NUM_LIT:2>]))<EOL>sys_platform = sys.platform<EOL>packages = []<EOL>with open(path, '<STR_LIT:rb>') as f:<EOL><INDENT>contents = f.read().decode('<STR_LIT:utf-8>')<EOL><DEDENT>for line in re.split(r'<STR_LIT>', contents):<EOL><INDENT>line = line.strip()<EOL>if not len(line):<EOL><INDENT>continue<EOL><DEDENT>if re.match(r'<STR_LIT>', line):<EOL><INDENT>continue<EOL><DEDENT>if '<STR_LIT:;>' in line:<EOL><INDENT>package, cond = line.split('<STR_LIT:;>', <NUM_LIT:1>)<EOL>package = package.strip()<EOL>cond = cond.strip()<EOL>cond = cond.replace('<STR_LIT>', repr(sys_platform))<EOL>cond = cond.replace('<STR_LIT>', repr(python_version))<EOL>if not eval(cond):<EOL><INDENT>continue<EOL><DEDENT><DEDENT>else:<EOL><INDENT>package = line.strip()<EOL><DEDENT>if re.match(r'<STR_LIT>', package):<EOL><INDENT>sub_req_file = re.sub(r'<STR_LIT>', '<STR_LIT>', package)<EOL>sub_req_file = os.path.abspath(os.path.join(os.path.dirname(path), sub_req_file))<EOL>packages.extend(_parse_requires(sub_req_file))<EOL>continue<EOL><DEDENT>if re.match(r'<STR_LIT>', package):<EOL><INDENT>packages.append({'<STR_LIT:type>': '<STR_LIT:url>', '<STR_LIT>': package})<EOL>continue<EOL><DEDENT>if '<STR_LIT>' in package:<EOL><INDENT>parts = package.split('<STR_LIT>')<EOL>package = parts[<NUM_LIT:0>].strip()<EOL>ver = parts[<NUM_LIT:1>].strip()<EOL>packages.append({'<STR_LIT:type>': '<STR_LIT>', '<STR_LIT>': package, '<STR_LIT>': ver})<EOL>continue<EOL><DEDENT>if '<STR_LIT>' in package:<EOL><INDENT>parts = package.split('<STR_LIT>')<EOL>package = parts[<NUM_LIT:0>].strip()<EOL>ver = parts[<NUM_LIT:1>].strip()<EOL>packages.append({'<STR_LIT:type>': '<STR_LIT>', '<STR_LIT>': package, '<STR_LIT>': ver})<EOL>continue<EOL><DEDENT>if re.search(r'<STR_LIT>', package):<EOL><INDENT>raise Exception('<STR_LIT>' % package)<EOL><DEDENT>packages.append({'<STR_LIT:type>': '<STR_LIT>', '<STR_LIT>': package})<EOL><DEDENT>return packages<EOL>

Does basic parsing of pip requirements files, to allow for
using something other than Python to do actual TLS requests

:param path:
    A path to a requirements file

:return:
    A list of dict objects containing the keys:
     - 'type' ('any', 'url', '==', '>=')
     - 'pkg'
     - 'ver' (if 'type' == '==' or 'type' == '>=')


if isinstance(archive, zipfile.ZipFile):<EOL><INDENT>return archive.infolist()<EOL><DEDENT>return archive.getmembers()<EOL>

:param archive:
    An archive from _open_archive()

:return:
    A list of info objects to be used with _info_name() and _extract_info()


if path.endswith('<STR_LIT>'):<EOL><INDENT>return zipfile.ZipFile(path, '<STR_LIT:r>')<EOL><DEDENT>return tarfile.open(path, '<STR_LIT:r>')<EOL>

:param path:
    A unicode string of the filesystem path to the archive

:return:
    An archive object


common_root = None<EOL>for info in _list_archive_members(archive):<EOL><INDENT>fn = _info_name(info)<EOL>if fn in set(['<STR_LIT:.>', '<STR_LIT:/>']):<EOL><INDENT>continue<EOL><DEDENT>sep = None<EOL>if '<STR_LIT:/>' in fn:<EOL><INDENT>sep = '<STR_LIT:/>'<EOL><DEDENT>elif '<STR_LIT:\\>' in fn:<EOL><INDENT>sep = '<STR_LIT:\\>'<EOL><DEDENT>if sep is None:<EOL><INDENT>root_dir = fn<EOL><DEDENT>else:<EOL><INDENT>root_dir, _ = fn.split(sep, <NUM_LIT:1>)<EOL><DEDENT>if common_root is None:<EOL><INDENT>common_root = root_dir<EOL><DEDENT>else:<EOL><INDENT>if common_root != root_dir:<EOL><INDENT>return None<EOL><DEDENT><DEDENT><DEDENT>return common_root<EOL>

Check if all members of the archive are in a single top-level directory

:param archive:
    An archive from _open_archive()

:return:
    None if not a single top level directory in archive, otherwise a
    unicode string of the top level directory name


try:<EOL><INDENT>return str_cls(byte_string, _encoding)<EOL><DEDENT>except (UnicodeDecodeError):<EOL><INDENT>for encoding in _fallback_encodings:<EOL><INDENT>try:<EOL><INDENT>return str_cls(byte_string, encoding, errors='<STR_LIT:strict>')<EOL><DEDENT>except (UnicodeDecodeError):<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>return str_cls(byte_string, errors='<STR_LIT:replace>')<EOL>

Tries decoding a byte string from the OS into a unicode string

:param byte_string:
    A byte string

:return:
    A unicode string


if not isinstance(marker, byte_cls) and not isinstance(marker, Pattern):<EOL><INDENT>raise TypeError(pretty_message(<EOL>'''<STR_LIT>''',<EOL>type_name(marker)<EOL>))<EOL><DEDENT>output = b'<STR_LIT>'<EOL>is_regex = isinstance(marker, Pattern)<EOL>while True:<EOL><INDENT>if len(self._decrypted_bytes) > <NUM_LIT:0>:<EOL><INDENT>chunk = self._decrypted_bytes<EOL>self._decrypted_bytes = b'<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>chunk = self.read(<NUM_LIT>)<EOL><DEDENT>offset = len(output)<EOL>output += chunk<EOL>if is_regex:<EOL><INDENT>match = marker.search(output)<EOL>if match is not None:<EOL><INDENT>end = match.end()<EOL>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>start = max(<NUM_LIT:0>, offset - len(marker) - <NUM_LIT:1>)<EOL>match = output.find(marker, start)<EOL>if match != -<NUM_LIT:1>:<EOL><INDENT>end = match + len(marker)<EOL>break<EOL><DEDENT><DEDENT><DEDENT>self._decrypted_bytes = output[end:] + self._decrypted_bytes<EOL>return output[<NUM_LIT:0>:end]<EOL>

Reads data from the socket until a marker is found. Data read may
include data beyond the marker.

:param marker:
    A byte string or regex object from re.compile(). Used to determine
    when to stop reading. Regex objects are more inefficient since
    they must scan the entire byte string of read data each time data
    is read off the socket.

:return:
    A byte string of the data read


return self.socket.getpeername()[<NUM_LIT:1>]<EOL>

An integer of the port number the socket is connected to


return self._cipher_suite<EOL>

A unicode string of the IANA cipher suite name of the negotiated
cipher suite


in_buffers = None<EOL>out_buffers = None<EOL>new_context_handle_pointer = None<EOL>try:<EOL><INDENT>if renegotiate:<EOL><INDENT>temp_context_handle_pointer = self._context_handle_pointer<EOL><DEDENT>else:<EOL><INDENT>new_context_handle_pointer = new(secur32, '<STR_LIT>')<EOL>temp_context_handle_pointer = new_context_handle_pointer<EOL><DEDENT>requested_flags = {<EOL>Secur32Const.ISC_REQ_REPLAY_DETECT: '<STR_LIT>',<EOL>Secur32Const.ISC_REQ_SEQUENCE_DETECT: '<STR_LIT>',<EOL>Secur32Const.ISC_REQ_CONFIDENTIALITY: '<STR_LIT>',<EOL>Secur32Const.ISC_REQ_ALLOCATE_MEMORY: '<STR_LIT>',<EOL>Secur32Const.ISC_REQ_INTEGRITY: '<STR_LIT>',<EOL>Secur32Const.ISC_REQ_STREAM: '<STR_LIT>',<EOL>Secur32Const.ISC_REQ_USE_SUPPLIED_CREDS: '<STR_LIT>',<EOL>}<EOL>self._context_flags = <NUM_LIT:0><EOL>for flag in requested_flags:<EOL><INDENT>self._context_flags |= flag<EOL><DEDENT>in_sec_buffer_desc_pointer, in_buffers = self._create_buffers(<NUM_LIT:2>)<EOL>in_buffers[<NUM_LIT:0>].BufferType = Secur32Const.SECBUFFER_TOKEN<EOL>out_sec_buffer_desc_pointer, out_buffers = self._create_buffers(<NUM_LIT:2>)<EOL>out_buffers[<NUM_LIT:0>].BufferType = Secur32Const.SECBUFFER_TOKEN<EOL>out_buffers[<NUM_LIT:1>].BufferType = Secur32Const.SECBUFFER_ALERT<EOL>output_context_flags_pointer = new(secur32, '<STR_LIT>')<EOL>if renegotiate:<EOL><INDENT>first_handle = temp_context_handle_pointer<EOL>second_handle = null()<EOL><DEDENT>else:<EOL><INDENT>first_handle = null()<EOL>second_handle = temp_context_handle_pointer<EOL><DEDENT>result = secur32.InitializeSecurityContextW(<EOL>self._session._credentials_handle,<EOL>first_handle,<EOL>self._hostname,<EOL>self._context_flags,<EOL><NUM_LIT:0>,<EOL><NUM_LIT:0>,<EOL>null(),<EOL><NUM_LIT:0>,<EOL>second_handle,<EOL>out_sec_buffer_desc_pointer,<EOL>output_context_flags_pointer,<EOL>null()<EOL>)<EOL>if result not in set([Secur32Const.SEC_E_OK, Secur32Const.SEC_I_CONTINUE_NEEDED]):<EOL><INDENT>handle_error(result, TLSError)<EOL><DEDENT>if not renegotiate:<EOL><INDENT>temp_context_handle_pointer = second_handle<EOL><DEDENT>else:<EOL><INDENT>temp_context_handle_pointer = first_handle<EOL><DEDENT>handshake_server_bytes = b'<STR_LIT>'<EOL>handshake_client_bytes = b'<STR_LIT>'<EOL>if out_buffers[<NUM_LIT:0>].cbBuffer > <NUM_LIT:0>:<EOL><INDENT>token = bytes_from_buffer(out_buffers[<NUM_LIT:0>].pvBuffer, out_buffers[<NUM_LIT:0>].cbBuffer)<EOL>handshake_client_bytes += token<EOL>self._socket.send(token)<EOL>out_buffers[<NUM_LIT:0>].cbBuffer = <NUM_LIT:0><EOL>secur32.FreeContextBuffer(out_buffers[<NUM_LIT:0>].pvBuffer)<EOL>out_buffers[<NUM_LIT:0>].pvBuffer = null()<EOL><DEDENT>in_data_buffer = buffer_from_bytes(<NUM_LIT>)<EOL>in_buffers[<NUM_LIT:0>].pvBuffer = cast(secur32, '<STR_LIT>', in_data_buffer)<EOL>bytes_read = b'<STR_LIT>'<EOL>while result != Secur32Const.SEC_E_OK:<EOL><INDENT>try:<EOL><INDENT>fail_late = False<EOL>bytes_read = self._socket.recv(<NUM_LIT>)<EOL>if bytes_read == b'<STR_LIT>':<EOL><INDENT>raise_disconnection()<EOL><DEDENT><DEDENT>except (socket_error_cls):<EOL><INDENT>fail_late = True<EOL><DEDENT>handshake_server_bytes += bytes_read<EOL>self._received_bytes += bytes_read<EOL>in_buffers[<NUM_LIT:0>].cbBuffer = len(self._received_bytes)<EOL>write_to_buffer(in_data_buffer, self._received_bytes)<EOL>result = secur32.InitializeSecurityContextW(<EOL>self._session._credentials_handle,<EOL>temp_context_handle_pointer,<EOL>self._hostname,<EOL>self._context_flags,<EOL><NUM_LIT:0>,<EOL><NUM_LIT:0>,<EOL>in_sec_buffer_desc_pointer,<EOL><NUM_LIT:0>,<EOL>null(),<EOL>out_sec_buffer_desc_pointer,<EOL>output_context_flags_pointer,<EOL>null()<EOL>)<EOL>if result == Secur32Const.SEC_E_INCOMPLETE_MESSAGE:<EOL><INDENT>in_buffers[<NUM_LIT:0>].BufferType = Secur32Const.SECBUFFER_TOKEN<EOL>if in_buffers[<NUM_LIT:1>].BufferType != Secur32Const.SECBUFFER_EMPTY:<EOL><INDENT>in_buffers[<NUM_LIT:1>].BufferType = Secur32Const.SECBUFFER_EMPTY<EOL>in_buffers[<NUM_LIT:1>].cbBuffer = <NUM_LIT:0><EOL>if not is_null(in_buffers[<NUM_LIT:1>].pvBuffer):<EOL><INDENT>secur32.FreeContextBuffer(in_buffers[<NUM_LIT:1>].pvBuffer)<EOL>in_buffers[<NUM_LIT:1>].pvBuffer = null()<EOL><DEDENT><DEDENT>if fail_late:<EOL><INDENT>raise_disconnection()<EOL><DEDENT>continue<EOL><DEDENT>if result == Secur32Const.SEC_E_ILLEGAL_MESSAGE:<EOL><INDENT>if detect_client_auth_request(handshake_server_bytes):<EOL><INDENT>raise_client_auth()<EOL><DEDENT>alert_info = parse_alert(handshake_server_bytes)<EOL>if alert_info and alert_info == (<NUM_LIT:2>, <NUM_LIT>):<EOL><INDENT>raise_protocol_version()<EOL><DEDENT>raise_handshake()<EOL><DEDENT>if result == Secur32Const.SEC_E_WRONG_PRINCIPAL:<EOL><INDENT>chain = extract_chain(handshake_server_bytes)<EOL>raise_hostname(chain[<NUM_LIT:0>], self._hostname)<EOL><DEDENT>if result == Secur32Const.SEC_E_CERT_EXPIRED:<EOL><INDENT>chain = extract_chain(handshake_server_bytes)<EOL>raise_expired_not_yet_valid(chain[<NUM_LIT:0>])<EOL><DEDENT>if result == Secur32Const.SEC_E_UNTRUSTED_ROOT:<EOL><INDENT>chain = extract_chain(handshake_server_bytes)<EOL>cert = chain[<NUM_LIT:0>]<EOL>oscrypto_cert = load_certificate(cert)<EOL>if not oscrypto_cert.self_signed:<EOL><INDENT>raise_no_issuer(cert)<EOL><DEDENT>raise_self_signed(cert)<EOL><DEDENT>if result == Secur32Const.SEC_E_INTERNAL_ERROR:<EOL><INDENT>if get_dh_params_length(handshake_server_bytes) < <NUM_LIT>:<EOL><INDENT>raise_dh_params()<EOL><DEDENT><DEDENT>if result == Secur32Const.SEC_I_INCOMPLETE_CREDENTIALS:<EOL><INDENT>raise_client_auth()<EOL><DEDENT>if result == Crypt32Const.TRUST_E_CERT_SIGNATURE:<EOL><INDENT>raise_weak_signature(cert)<EOL><DEDENT>if result == Secur32Const.SEC_E_INVALID_TOKEN:<EOL><INDENT>if out_buffers[<NUM_LIT:1>].cbBuffer > <NUM_LIT:0>:<EOL><INDENT>alert_bytes = bytes_from_buffer(out_buffers[<NUM_LIT:1>].pvBuffer, out_buffers[<NUM_LIT:1>].cbBuffer)<EOL>handshake_client_bytes += alert_bytes<EOL>alert_number = alert_bytes[<NUM_LIT:6>:<NUM_LIT:7>]<EOL>if alert_number == b'<STR_LIT>' or alert_number == b'<STR_LIT>':<EOL><INDENT>if '<STR_LIT>' in self._session._protocols and len(self._session._protocols) > <NUM_LIT:1>:<EOL><INDENT>chain = extract_chain(handshake_server_bytes)<EOL>raise _TLSDowngradeError(<EOL>'<STR_LIT>',<EOL>chain[<NUM_LIT:0>]<EOL>)<EOL><DEDENT><DEDENT><DEDENT>if detect_client_auth_request(handshake_server_bytes):<EOL><INDENT>raise_client_auth()<EOL><DEDENT>if detect_other_protocol(handshake_server_bytes):<EOL><INDENT>raise_protocol_error(handshake_server_bytes)<EOL><DEDENT>raise_handshake()<EOL><DEDENT>if result == Secur32Const.SEC_E_BUFFER_TOO_SMALL or result == Secur32Const.SEC_E_MESSAGE_ALTERED:<EOL><INDENT>if '<STR_LIT>' in self._session._protocols:<EOL><INDENT>raise _TLSRetryError('<STR_LIT>')<EOL><DEDENT><DEDENT>if fail_late:<EOL><INDENT>raise_disconnection()<EOL><DEDENT>if result == Secur32Const.SEC_E_INVALID_PARAMETER:<EOL><INDENT>if get_dh_params_length(handshake_server_bytes) < <NUM_LIT>:<EOL><INDENT>raise_dh_params()<EOL><DEDENT><DEDENT>if result not in set([Secur32Const.SEC_E_OK, Secur32Const.SEC_I_CONTINUE_NEEDED]):<EOL><INDENT>handle_error(result, TLSError)<EOL><DEDENT>if out_buffers[<NUM_LIT:0>].cbBuffer > <NUM_LIT:0>:<EOL><INDENT>token = bytes_from_buffer(out_buffers[<NUM_LIT:0>].pvBuffer, out_buffers[<NUM_LIT:0>].cbBuffer)<EOL>handshake_client_bytes += token<EOL>self._socket.send(token)<EOL>out_buffers[<NUM_LIT:0>].cbBuffer = <NUM_LIT:0><EOL>secur32.FreeContextBuffer(out_buffers[<NUM_LIT:0>].pvBuffer)<EOL>out_buffers[<NUM_LIT:0>].pvBuffer = null()<EOL><DEDENT>if in_buffers[<NUM_LIT:1>].BufferType == Secur32Const.SECBUFFER_EXTRA:<EOL><INDENT>extra_amount = in_buffers[<NUM_LIT:1>].cbBuffer<EOL>self._received_bytes = self._received_bytes[-extra_amount:]<EOL>in_buffers[<NUM_LIT:1>].BufferType = Secur32Const.SECBUFFER_EMPTY<EOL>in_buffers[<NUM_LIT:1>].cbBuffer = <NUM_LIT:0><EOL>secur32.FreeContextBuffer(in_buffers[<NUM_LIT:1>].pvBuffer)<EOL>in_buffers[<NUM_LIT:1>].pvBuffer = null()<EOL>if result == Secur32Const.SEC_E_OK:<EOL><INDENT>handshake_server_bytes = handshake_server_bytes[-extra_amount:]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self._received_bytes = b'<STR_LIT>'<EOL><DEDENT><DEDENT>connection_info_pointer = struct(secur32, '<STR_LIT>')<EOL>result = secur32.QueryContextAttributesW(<EOL>temp_context_handle_pointer,<EOL>Secur32Const.SECPKG_ATTR_CONNECTION_INFO,<EOL>connection_info_pointer<EOL>)<EOL>handle_error(result, TLSError)<EOL>connection_info = unwrap(connection_info_pointer)<EOL>self._protocol = {<EOL>Secur32Const.SP_PROT_SSL2_CLIENT: '<STR_LIT>',<EOL>Secur32Const.SP_PROT_SSL3_CLIENT: '<STR_LIT>',<EOL>Secur32Const.SP_PROT_TLS1_CLIENT: '<STR_LIT>',<EOL>Secur32Const.SP_PROT_TLS1_1_CLIENT: '<STR_LIT>',<EOL>Secur32Const.SP_PROT_TLS1_2_CLIENT: '<STR_LIT>',<EOL>}.get(native(int, connection_info.dwProtocol), str_cls(connection_info.dwProtocol))<EOL>if self._protocol in set(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']):<EOL><INDENT>session_info = parse_session_info(handshake_server_bytes, handshake_client_bytes)<EOL>self._cipher_suite = session_info['<STR_LIT>']<EOL>self._compression = session_info['<STR_LIT>']<EOL>self._session_id = session_info['<STR_LIT>']<EOL>self._session_ticket = session_info['<STR_LIT>']<EOL><DEDENT>output_context_flags = deref(output_context_flags_pointer)<EOL>for flag in requested_flags:<EOL><INDENT>if (flag | output_context_flags) == <NUM_LIT:0>:<EOL><INDENT>raise OSError(pretty_message(<EOL>'''<STR_LIT>''',<EOL>requested_flags[flag]<EOL>))<EOL><DEDENT><DEDENT>if not renegotiate:<EOL><INDENT>self._context_handle_pointer = temp_context_handle_pointer<EOL>new_context_handle_pointer = None<EOL>stream_sizes_pointer = struct(secur32, '<STR_LIT>')<EOL>result = secur32.QueryContextAttributesW(<EOL>self._context_handle_pointer,<EOL>Secur32Const.SECPKG_ATTR_STREAM_SIZES,<EOL>stream_sizes_pointer<EOL>)<EOL>handle_error(result)<EOL>stream_sizes = unwrap(stream_sizes_pointer)<EOL>self._header_size = native(int, stream_sizes.cbHeader)<EOL>self._message_size = native(int, stream_sizes.cbMaximumMessage)<EOL>self._trailer_size = native(int, stream_sizes.cbTrailer)<EOL>self._buffer_size = self._header_size + self._message_size + self._trailer_size<EOL><DEDENT>if self._session._extra_trust_roots:<EOL><INDENT>self._extra_trust_root_validation()<EOL><DEDENT><DEDENT>except (OSError, socket_.error):<EOL><INDENT>self.close()<EOL>raise<EOL><DEDENT>finally:<EOL><INDENT>if out_buffers:<EOL><INDENT>if not is_null(out_buffers[<NUM_LIT:0>].pvBuffer):<EOL><INDENT>secur32.FreeContextBuffer(out_buffers[<NUM_LIT:0>].pvBuffer)<EOL><DEDENT>if not is_null(out_buffers[<NUM_LIT:1>].pvBuffer):<EOL><INDENT>secur32.FreeContextBuffer(out_buffers[<NUM_LIT:1>].pvBuffer)<EOL><DEDENT><DEDENT>if new_context_handle_pointer:<EOL><INDENT>secur32.DeleteSecurityContext(new_context_handle_pointer)<EOL><DEDENT><DEDENT>

Perform an initial TLS handshake, or a renegotiation

:param renegotiate:
    If the handshake is for a renegotiation


return self._session_id<EOL>

A unicode string of "new" or "reused" or None for no ticket


output = b'<STR_LIT>'<EOL>remaining = num_bytes<EOL>while remaining > <NUM_LIT:0>:<EOL><INDENT>output += self.read(remaining)<EOL>remaining = num_bytes - len(output)<EOL><DEDENT>return output<EOL>

Reads exactly the specified number of bytes from the socket

:param num_bytes:
    An integer - the exact number of bytes to read

:return:
    A byte string of the data that was read


return self._hostname<EOL>

A unicode string of the TLS server domain name or IP address


_, write_ready, _ = select.select([], [self._socket], [], timeout)<EOL>return len(write_ready) > <NUM_LIT:0><EOL>

Blocks until the socket is ready to be written to, or the timeout is hit

:param timeout:
    A float - the period of time to wait for the socket to be ready to
    written to. None for no time limit.

:return:
    A boolean - if the socket is ready for writing. Will only be False
    if timeout is not None.


def __init__(self, protocol=None, manual_validation=False, extra_trust_roots=None):

if not isinstance(manual_validation, bool):<EOL><INDENT>raise TypeError(pretty_message(<EOL>'''<STR_LIT>''',<EOL>type_name(manual_validation)<EOL>))<EOL><DEDENT>self._manual_validation = manual_validation<EOL>if protocol is None:<EOL><INDENT>protocol = set(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL><DEDENT>if isinstance(protocol, str_cls):<EOL><INDENT>protocol = set([protocol])<EOL><DEDENT>elif not isinstance(protocol, set):<EOL><INDENT>raise TypeError(pretty_message(<EOL>'''<STR_LIT>''',<EOL>type_name(protocol)<EOL>))<EOL><DEDENT>unsupported_protocols = protocol - set(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>if unsupported_protocols:<EOL><INDENT>raise ValueError(pretty_message(<EOL>'''<STR_LIT>''',<EOL>repr(unsupported_protocols)<EOL>))<EOL><DEDENT>self._protocols = protocol<EOL>self._extra_trust_roots = []<EOL>if extra_trust_roots:<EOL><INDENT>for extra_trust_root in extra_trust_roots:<EOL><INDENT>if isinstance(extra_trust_root, Certificate):<EOL><INDENT>extra_trust_root = extra_trust_root.asn1<EOL><DEDENT>elif isinstance(extra_trust_root, byte_cls):<EOL><INDENT>extra_trust_root = parse_certificate(extra_trust_root)<EOL><DEDENT>elif isinstance(extra_trust_root, str_cls):<EOL><INDENT>with open(extra_trust_root, '<STR_LIT:rb>') as f:<EOL><INDENT>extra_trust_root = parse_certificate(f.read())<EOL><DEDENT><DEDENT>elif not isinstance(extra_trust_root, x509.Certificate):<EOL><INDENT>raise TypeError(pretty_message(<EOL>'''<STR_LIT>''',<EOL>type_name(extra_trust_root)<EOL>))<EOL><DEDENT>self._extra_trust_roots.append(extra_trust_root)<EOL><DEDENT><DEDENT>self._obtain_credentials()<EOL>

:param protocol:
    A unicode string or set of unicode strings representing allowable
    protocols to negotiate with the server:

     - "TLSv1.2"
     - "TLSv1.1"
     - "TLSv1"
     - "SSLv3"

    Default is: {"TLSv1", "TLSv1.1", "TLSv1.2"}

:param manual_validation:
    If certificate and certificate path validation should be skipped
    and left to the developer to implement

:param extra_trust_roots:
    A list containing one or more certificates to be treated as trust
    roots, in one of the following formats:
     - A byte string of the DER encoded certificate
     - A unicode string of the certificate filename
     - An asn1crypto.x509.Certificate object
     - An oscrypto.asymmetric.Certificate object

:raises:
    ValueError - when any of the parameters contain an invalid value
    TypeError - when any of the parameters are of the wrong type
    OSError - when an error is returned by the OS crypto library