def calculate_concordance_probability(graph: BELGraph,<EOL>key: str,<EOL>cutoff: Optional[float] = None,<EOL>permutations: Optional[int] = None,<EOL>percentage: Optional[float] = None,<EOL>use_ambiguous: bool = False,<EOL>permute_type: str = '<STR_LIT>',<EOL>) -> Tuple[float, List[float], float]:

if permute_type == '<STR_LIT>':<EOL><INDENT>permute_func = partial(random_by_edges, percentage=percentage)<EOL><DEDENT>elif permute_type == '<STR_LIT>':<EOL><INDENT>permute_func = partial(shuffle_node_data, key=key, percentage=percentage)<EOL><DEDENT>elif permute_type == '<STR_LIT>':<EOL><INDENT>permute_func = partial(shuffle_relations, percentage=percentage)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(permute_type))<EOL><DEDENT>graph: BELGraph = graph.copy()<EOL>collapse_to_genes(graph)<EOL>collapse_all_variants(graph)<EOL>score = calculate_concordance(graph, key, cutoff=cutoff)<EOL>distribution = []<EOL>for _ in range(permutations or <NUM_LIT>):<EOL><INDENT>permuted_graph = permute_func(graph)<EOL>permuted_graph_scores = calculate_concordance(permuted_graph, key, cutoff=cutoff, use_ambiguous=use_ambiguous)<EOL>distribution.append(permuted_graph_scores)<EOL><DEDENT>return score, distribution, one_sided(score, distribution)<EOL>

Calculates a graph's concordance as well as its statistical probability.

    :param graph: A BEL graph
    :param str key: The node data dictionary key storing the logFC
    :param float cutoff: The optional logFC cutoff for significance
    :param int permutations: The number of random permutations to test. Defaults to 500
    :param float percentage: The percentage of the graph's edges to maintain. Defaults to 0.9
    :param bool use_ambiguous: Compare to ambiguous edges as well
    :returns: A triple of the concordance score, the null distribution, and the p-value.


def calculate_concordance_by_annotation(graph, annotation, key, cutoff=None):

return {<EOL>value: calculate_concordance(subgraph, key, cutoff=cutoff)<EOL>for value, subgraph in get_subgraphs_by_annotation(graph, annotation).items()<EOL>}<EOL>

Returns the concordance scores for each stratified graph based on the given annotation

    :param pybel.BELGraph graph: A BEL graph
    :param str annotation: The annotation to group by.
    :param str key: The node data dictionary key storing the logFC
    :param float cutoff: The optional logFC cutoff for significance
    :rtype: dict[str,tuple]


def calculate_concordance_probability_by_annotation(graph, annotation, key, cutoff=None, permutations=None,<EOL>percentage=None,<EOL>use_ambiguous=False):

result = [<EOL>(value, calculate_concordance_probability(<EOL>subgraph,<EOL>key,<EOL>cutoff=cutoff,<EOL>permutations=permutations,<EOL>percentage=percentage,<EOL>use_ambiguous=use_ambiguous,<EOL>))<EOL>for value, subgraph in get_subgraphs_by_annotation(graph, annotation).items()<EOL>]<EOL>return dict(result)<EOL>

Returns the results of concordance analysis on each subgraph, stratified by the given annotation.

    :param pybel.BELGraph graph: A BEL graph
    :param str annotation: The annotation to group by.
    :param str key: The node data dictionary key storing the logFC
    :param float cutoff: The optional logFC cutoff for significance
    :param int permutations: The number of random permutations to test. Defaults to 500
    :param float percentage: The percentage of the graph's edges to maintain. Defaults to 0.9
    :param bool use_ambiguous: Compare to ambiguous edges as well
    :rtype: dict[str,tuple]


def bel_to_spia_matrices(graph: BELGraph) -> Mapping[str, pd.DataFrame]:

index_nodes = get_matrix_index(graph)<EOL>spia_matrices = build_spia_matrices(index_nodes)<EOL>for u, v, edge_data in graph.edges(data=True):<EOL><INDENT>if isinstance(u, CentralDogma) and isinstance(v, CentralDogma):<EOL><INDENT>update_spia_matrices(spia_matrices, u, v, edge_data)<EOL><DEDENT>elif isinstance(u, CentralDogma) and isinstance(v, ListAbundance):<EOL><INDENT>for node in v.members:<EOL><INDENT>if not isinstance(node, CentralDogma):<EOL><INDENT>continue<EOL><DEDENT>update_spia_matrices(spia_matrices, u, node, edge_data)<EOL><DEDENT><DEDENT>elif isinstance(u, ListAbundance) and isinstance(v, CentralDogma):<EOL><INDENT>for node in u.members:<EOL><INDENT>if not isinstance(node, CentralDogma):<EOL><INDENT>continue<EOL><DEDENT>update_spia_matrices(spia_matrices, node, v, edge_data)<EOL><DEDENT><DEDENT>elif isinstance(u, ListAbundance) and isinstance(v, ListAbundance):<EOL><INDENT>for sub_member, obj_member in product(u.members, v.members):<EOL><INDENT>if isinstance(sub_member, CentralDogma) and isinstance(obj_member, CentralDogma):<EOL><INDENT>update_spia_matrices(spia_matrices, sub_member, obj_member, edge_data)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return spia_matrices<EOL>

Create an excel sheet ready to be used in SPIA software.

    :param graph: BELGraph
    :return: dictionary with matrices


def get_matrix_index(graph: BELGraph) -> Set[str]:

<EOL>return {<EOL>node.name<EOL>for node in graph<EOL>if isinstance(node, CentralDogma) and node.namespace.upper() == '<STR_LIT>'<EOL>}<EOL>

Return set of HGNC names from Proteins/Rnas/Genes/miRNA, nodes that can be used by SPIA.


def build_spia_matrices(nodes: Set[str]) -> Dict[str, pd.DataFrame]:

nodes = list(sorted(nodes))<EOL>matrices = OrderedDict()<EOL>for relation in KEGG_RELATIONS:<EOL><INDENT>matrices[relation] = pd.DataFrame(<NUM_LIT:0>, index=nodes, columns=nodes)<EOL><DEDENT>return matrices<EOL>

Build an adjacency matrix for each KEGG relationship and return in a dictionary.

    :param nodes: A set of HGNC gene symbols
    :return: Dictionary of adjacency matrix for each relationship


def update_spia_matrices(spia_matrices: Dict[str, pd.DataFrame],<EOL>u: CentralDogma,<EOL>v: CentralDogma,<EOL>edge_data: EdgeData,<EOL>) -> None:

if u.namespace.upper() != '<STR_LIT>' or v.namespace.upper() != '<STR_LIT>':<EOL><INDENT>return<EOL><DEDENT>u_name = u.name<EOL>v_name = v.name<EOL>relation = edge_data[RELATION]<EOL>if relation in CAUSAL_INCREASE_RELATIONS:<EOL><INDENT>if v.variants and any(isinstance(variant, ProteinModification) for variant in v.variants):<EOL><INDENT>for variant in v.variants:<EOL><INDENT>if not isinstance(variant, ProteinModification):<EOL><INDENT>continue<EOL><DEDENT>if variant[IDENTIFIER][NAME] == "<STR_LIT>":<EOL><INDENT>spia_matrices["<STR_LIT>"][u_name][v_name] = <NUM_LIT:1><EOL><DEDENT>elif variant[IDENTIFIER][NAME] == "<STR_LIT>":<EOL><INDENT>spia_matrices["<STR_LIT>"][u_name][v_name] = <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>elif isinstance(v, (Gene, Rna)):  <EOL><INDENT>spia_matrices['<STR_LIT>'][u_name][v_name] = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>spia_matrices['<STR_LIT>'][u_name][v_name] = <NUM_LIT:1><EOL><DEDENT><DEDENT>elif relation in CAUSAL_DECREASE_RELATIONS:<EOL><INDENT>if v.variants and any(isinstance(variant, ProteinModification) for variant in v.variants):<EOL><INDENT>for variant in v.variants:<EOL><INDENT>if not isinstance(variant, ProteinModification):<EOL><INDENT>continue<EOL><DEDENT>if variant[IDENTIFIER][NAME] == "<STR_LIT>":<EOL><INDENT>spia_matrices['<STR_LIT>'][u_name][v_name] = <NUM_LIT:1><EOL><DEDENT>elif variant[IDENTIFIER][NAME] == "<STR_LIT>":<EOL><INDENT>spia_matrices["<STR_LIT>"][u_name][v_name] = <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>elif isinstance(v, (Gene, Rna)):  <EOL><INDENT>spia_matrices["<STR_LIT>"][u_name][v_name] = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>spia_matrices["<STR_LIT>"][u_name][v_name] = <NUM_LIT:1><EOL><DEDENT><DEDENT>elif relation == ASSOCIATION:<EOL><INDENT>spia_matrices["<STR_LIT>"][u_name][v_name] = <NUM_LIT:1><EOL><DEDENT>


def spia_matrices_to_excel(spia_matrices: Mapping[str, pd.DataFrame], path: str) -> None:

writer = pd.ExcelWriter(path, engine='<STR_LIT>')<EOL>for relation, df in spia_matrices.items():<EOL><INDENT>df.to_excel(writer, sheet_name=relation, index=False)<EOL><DEDENT>writer.save()<EOL>

Export a SPIA data dictionary into an Excel sheet at the given path.

    .. note::

        # The R import should add the values:
        # ["nodes"] from the columns
        # ["title"] from the name of the file
        # ["NumberOfReactions"] set to "0"


def spia_matrices_to_tsvs(spia_matrices: Mapping[str, pd.DataFrame], directory: str) -> None:

os.makedirs(directory, exist_ok=True)<EOL>for relation, df in spia_matrices.items():<EOL><INDENT>df.to_csv(os.path.join(directory, f'<STR_LIT>'), index=True)<EOL><DEDENT>

Export a SPIA data dictionary into a directory as several TSV documents.


@click.command()<EOL>@graph_pickle_argument<EOL>@click.option('<STR_LIT>', type=click.Path(file_okay=True, dir_okay=False))<EOL>@click.option('<STR_LIT>', type=click.Path(file_okay=False, dir_okay=True))<EOL>def main(graph: BELGraph, xlsx: str, tsvs: str):

if not xlsx and not tsvs:<EOL><INDENT>click.secho('<STR_LIT>', fg='<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>spia_matrices = bel_to_spia_matrices(graph)<EOL>if xlsx:<EOL><INDENT>spia_matrices_to_excel(spia_matrices, xlsx)<EOL><DEDENT>if tsvs:<EOL><INDENT>spia_matrices_to_tsvs(spia_matrices, tsvs)<EOL><DEDENT>


def _get_drug_target_interactions(manager: Optional['<STR_LIT>'] = None) -> Mapping[str, List[str]]:

if manager is None:<EOL><INDENT>import bio2bel_drugbank<EOL>manager = bio2bel_drugbank.Manager()<EOL><DEDENT>if not manager.is_populated():<EOL><INDENT>manager.populate()<EOL><DEDENT>return manager.get_drug_to_hgnc_symbols()<EOL>

Get a mapping from drugs to their list of gene.


def multi_run_epicom(graphs: Iterable[BELGraph], path: Union[None, str, TextIO]) -> None:

if isinstance(path, str):<EOL><INDENT>with open(path, '<STR_LIT:w>') as file:<EOL><INDENT>_multi_run_helper_file_wrapper(graphs, file)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>_multi_run_helper_file_wrapper(graphs, path)<EOL><DEDENT>


run_epicom(graph, directory)<EOL>


graph = get_ad_graph()<EOL>run_epicom(graph, directory)<EOL>


graph = get_pd_graph()<EOL>run_epicom(graph, directory)<EOL>


graph = get_ep_graph()<EOL>run_epicom(graph, directory)<EOL>


@main.command()<EOL>@click.option('<STR_LIT>', '<STR_LIT>', type=click.File('<STR_LIT:w>'), default=sys.stdout)<EOL>def multi(output):

graphs = [<EOL>get_ad_graph(),<EOL>get_ep_graph(),<EOL>get_pd_graph(),<EOL>]<EOL>multi_run_epicom(graphs, output)<EOL>


def get_networks_using_annotation(manager: pybel.Manager, annotation: str):

raise NotImplementedError<EOL>

:param pybel.manager.Manager manager:
:param str annotation:
:return: list[pybel.manager.models.Network]


def get_drug_model(manager: pybel.Manager, name: str):

raise NotImplementedError<EOL>

:param pybel.manager.Manager manager:
:param str name:
:return: pybel.manager.models.NamespaceEntry


def build_database(manager: pybel.Manager, annotation_url: Optional[str] = None) -> None:

annotation_url = annotation_url or NEUROMMSIG_DEFAULT_URL<EOL>annotation = manager.get_namespace_by_url(annotation_url)<EOL>if annotation is None:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>networks = get_networks_using_annotation(manager, annotation)<EOL>dtis = ...<EOL>for network in networks:<EOL><INDENT>graph = network.as_bel()<EOL>scores = epicom_on_graph(graph, dtis)<EOL>for (drug_name, subgraph_name), score in scores.items():<EOL><INDENT>drug_model = get_drug_model(manager, drug_name)<EOL>subgraph_model = manager.get_annotation_entry(annotation_url, subgraph_name)<EOL>score_model = Score(<EOL>network=network,<EOL>annotation=subgraph_model,<EOL>drug=drug_model,<EOL>score=score<EOL>)<EOL>manager.session.add(score_model)<EOL><DEDENT><DEDENT>t = time.time()<EOL>logger.info('<STR_LIT>')<EOL>manager.session.commit()<EOL>logger.info('<STR_LIT>', time.time() - t)<EOL>

Build a database of scores for NeuroMMSig annotated graphs.

    1. Get all networks that use the Subgraph annotation
    2. run on each


raise NotImplementedError<EOL>

Generates a spanning tree from the directed graph using the random walk approach proposed independently by
    by Broder (1989) and Aldous (1990). It simply generates random walks until all nodes have been covered.

    Algorithm:

    1. Choose a starting vertex s arbitrarily. Set T_V ← {s} and T_E ← ∅.
    2. Do a simple random walk starting at s. Whenever we cross an edge e = {u, v} with v ∈ V ,
        add v to TV and add e to TE.
    3. Stop the random walk when TV = V . Output T = (T_V , T_E) as our spanning tree

    :param networkx.DiGraph graph: The input graph
    :rtype: networkx.DiGraph

    .. seealso::

        - https://math.dartmouth.edu/~pw/math100w13/kothari.pdf
        - http://keyulux.com/pdf/spanning_tree.pdf


def rank_causalr_hypothesis(graph, node_to_regulation, regulator_node):

upregulation_hypothesis = {<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0><EOL>}<EOL>downregulation_hypothesis = {<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT:0><EOL>}<EOL>targets = [<EOL>node<EOL>for node in node_to_regulation<EOL>if node != regulator_node<EOL>]<EOL>predicted_regulations = run_cna(graph, regulator_node, targets)  <EOL>for _, target_node, predicted_regulation in predicted_regulations:<EOL><INDENT>if (predicted_regulation is Effect.inhibition or predicted_regulation is Effect.activation) and (<EOL>predicted_regulation.value == node_to_regulation[target_node]):<EOL><INDENT>upregulation_hypothesis['<STR_LIT>'] += <NUM_LIT:1><EOL>downregulation_hypothesis['<STR_LIT>'] += <NUM_LIT:1><EOL><DEDENT>elif predicted_regulation is Effect.ambiguous:<EOL><INDENT>upregulation_hypothesis['<STR_LIT>'] += <NUM_LIT:1><EOL>downregulation_hypothesis['<STR_LIT>'] += <NUM_LIT:1><EOL><DEDENT>elif predicted_regulation is Effect.no_effect:<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>downregulation_hypothesis['<STR_LIT>'] += <NUM_LIT:1><EOL>upregulation_hypothesis['<STR_LIT>'] += <NUM_LIT:1><EOL><DEDENT><DEDENT>upregulation_hypothesis['<STR_LIT>'] = upregulation_hypothesis['<STR_LIT>'] - upregulation_hypothesis['<STR_LIT>']<EOL>downregulation_hypothesis['<STR_LIT>'] = downregulation_hypothesis['<STR_LIT>'] - downregulation_hypothesis['<STR_LIT>']<EOL>return upregulation_hypothesis, downregulation_hypothesis<EOL>

Test the regulator hypothesis of the given node on the input data using the algorithm.

    Note: this method returns both +/- signed hypotheses evaluated

    Algorithm:

    1. Calculate the shortest path between the regulator node and each node in observed_regulation
    2. Calculate the concordance of the causal network and the observed regulation when there is path
       between target node and regulator node

    :param networkx.DiGraph graph: A causal graph
    :param dict node_to_regulation: Nodes to score (1,-1,0)
    :return Dictionaries with hypothesis results (keys: score, correct, incorrect, ambiguous)
    :rtype: dict


def run_cna(graph, root, targets, relationship_dict=None):

causal_effects = []<EOL>relationship_dict = causal_effect_dict if relationship_dict is None else relationship_dict<EOL>for target in targets:<EOL><INDENT>try:<EOL><INDENT>shortest_paths = nx.all_shortest_paths(graph, source=root, target=target)<EOL>effects_in_path = set()<EOL>for shortest_path in shortest_paths:<EOL><INDENT>effects_in_path.add(get_path_effect(graph, shortest_path, relationship_dict))<EOL><DEDENT>if len(effects_in_path) == <NUM_LIT:1>:<EOL><INDENT>causal_effects.append((root, target, next(iter(effects_in_path))))  <EOL><DEDENT>elif Effect.activation in effects_in_path and Effect.inhibition in effects_in_path:<EOL><INDENT>causal_effects.append((root, target, Effect.ambiguous))<EOL><DEDENT>elif Effect.activation in effects_in_path and Effect.inhibition not in effects_in_path:<EOL><INDENT>causal_effects.append((root, target, Effect.activation))<EOL><DEDENT>elif Effect.inhibition in effects_in_path and Effect.activation not in effects_in_path:<EOL><INDENT>causal_effects.append((root, target, Effect.inhibition))<EOL><DEDENT>else:<EOL><INDENT>log.warning('<STR_LIT>'.format(effects_in_path))<EOL><DEDENT><DEDENT>except nx.NetworkXNoPath:<EOL><INDENT>log.warning('<STR_LIT>'.format(root, target))<EOL><DEDENT><DEDENT>return causal_effects<EOL>

Returns the effect from the root to the target nodes represented as {-1,1}

    :param pybel.BELGraph graph: A BEL graph
    :param BaseEntity root: The root node
    :param iter targets: The targets nodes
    :param dict relationship_dict: dictionary with relationship effects
    :return list[tuple]:


def get_path_effect(graph, path, relationship_dict):

causal_effect = []<EOL>for predecessor, successor in pairwise(path):<EOL><INDENT>if pair_has_contradiction(graph, predecessor, successor):<EOL><INDENT>return Effect.ambiguous<EOL><DEDENT>edges = graph.get_edge_data(predecessor, successor)<EOL>edge_key, edge_relation, _ = rank_edges(edges)<EOL>relation = graph[predecessor][successor][edge_key][RELATION]<EOL>if relation not in relationship_dict or relationship_dict[relation] == <NUM_LIT:0>:<EOL><INDENT>return Effect.no_effect<EOL><DEDENT>causal_effect.append(relationship_dict[relation])<EOL><DEDENT>final_effect = reduce(lambda x, y: x * y, causal_effect)<EOL>return Effect.activation if final_effect == <NUM_LIT:1> else Effect.inhibition<EOL>

Calculate the final effect of the root node to the sink node in the path.

    :param pybel.BELGraph graph: A BEL graph
    :param list path: Path from root to sink node
    :param dict relationship_dict: dictionary with relationship effects
    :rtype: Effect


edge_ranking = default_edge_ranking if edge_ranking is None else edge_ranking<EOL>edges_scores = [<EOL>(edge_id, edge_data[RELATION], edge_ranking[edge_data[RELATION]])<EOL>for edge_id, edge_data in edges.items()<EOL>]<EOL>return max(edges_scores, key=itemgetter(<NUM_LIT:2>))<EOL>

Return the highest ranked edge from a multiedge.

    :param dict edges: dictionary with all edges between two nodes
    :param dict edge_ranking: A dictionary of {relationship: score}
    :return: Highest ranked edge
    :rtype: tuple: (edge id, relation, score given ranking)


def calculate_average_scores_on_graph(<EOL>graph: BELGraph,<EOL>key: Optional[str] = None,<EOL>tag: Optional[str] = None,<EOL>default_score: Optional[float] = None,<EOL>runs: Optional[int] = None,<EOL>use_tqdm: bool = False,<EOL>):

subgraphs = generate_bioprocess_mechanisms(graph, key=key)<EOL>scores = calculate_average_scores_on_subgraphs(<EOL>subgraphs,<EOL>key=key,<EOL>tag=tag,<EOL>default_score=default_score,<EOL>runs=runs,<EOL>use_tqdm=use_tqdm<EOL>)<EOL>return scores<EOL>

Calculate the scores over all biological processes in the sub-graph.

    As an implementation, it simply computes the sub-graphs then calls :func:`calculate_average_scores_on_subgraphs` as
    described in that function's documentation.

    :param graph: A BEL graph with heats already on the nodes
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param use_tqdm: Should there be a progress bar for runners?
    :return: A dictionary of {pybel node tuple: results tuple}
    :rtype: dict[tuple, tuple]

    Suggested usage with :mod:`pandas`:

    >>> import pandas as pd
    >>> from pybel_tools.analysis.heat import calculate_average_scores_on_graph
    >>> graph = ...  # load graph and data
    >>> scores = calculate_average_scores_on_graph(graph)
    >>> pd.DataFrame.from_items(scores.items(), orient='index', columns=RESULT_LABELS)


def calculate_average_scores_on_subgraphs(<EOL>subgraphs: Mapping[H, BELGraph],<EOL>key: Optional[str] = None,<EOL>tag: Optional[str] = None,<EOL>default_score: Optional[float] = None,<EOL>runs: Optional[int] = None,<EOL>use_tqdm: bool = False,<EOL>tqdm_kwargs: Optional[Mapping[str, Any]] = None,<EOL>) -> Mapping[H, Tuple[float, float, float, float, int, int]]:

results = {}<EOL>log.info('<STR_LIT>', len(subgraphs), runs)<EOL>it = subgraphs.items()<EOL>if use_tqdm:<EOL><INDENT>_tqdm_kwargs = dict(total=len(subgraphs), desc='<STR_LIT>')<EOL>if tqdm_kwargs:<EOL><INDENT>_tqdm_kwargs.update(tqdm_kwargs)<EOL><DEDENT>it = tqdm(it, **_tqdm_kwargs)<EOL><DEDENT>for node, subgraph in it:<EOL><INDENT>number_first_neighbors = subgraph.in_degree(node)<EOL>number_first_neighbors = <NUM_LIT:0> if isinstance(number_first_neighbors, dict) else number_first_neighbors<EOL>mechanism_size = subgraph.number_of_nodes()<EOL>runners = workflow(subgraph, node, key=key, tag=tag, default_score=default_score, runs=runs)<EOL>scores = [runner.get_final_score() for runner in runners]<EOL>if <NUM_LIT:0> == len(scores):<EOL><INDENT>results[node] = (<EOL>None,<EOL>None,<EOL>None,<EOL>None,<EOL>number_first_neighbors,<EOL>mechanism_size,<EOL>)<EOL>continue<EOL><DEDENT>scores = np.array(scores)<EOL>average_score = np.average(scores)<EOL>score_std = np.std(scores)<EOL>med_score = np.median(scores)<EOL>chi_2_stat, norm_p = stats.normaltest(scores)<EOL>results[node] = (<EOL>average_score,<EOL>score_std,<EOL>norm_p,<EOL>med_score,<EOL>number_first_neighbors,<EOL>mechanism_size,<EOL>)<EOL><DEDENT>return results<EOL>

Calculate the scores over precomputed candidate mechanisms.

    :param subgraphs: A dictionary of keys to their corresponding subgraphs
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param use_tqdm: Should there be a progress bar for runners?
    :return: A dictionary of keys to results tuples

    Example Usage:

    >>> import pandas as pd
    >>> from pybel_tools.generation import generate_bioprocess_mechanisms
    >>> from pybel_tools.analysis.heat import calculate_average_scores_on_subgraphs
    >>> # load graph and data
    >>> graph = ...
    >>> candidate_mechanisms = generate_bioprocess_mechanisms(graph)
    >>> scores = calculate_average_scores_on_subgraphs(candidate_mechanisms)
    >>> pd.DataFrame.from_items(scores.items(), orient='index', columns=RESULT_LABELS)


def workflow(<EOL>graph: BELGraph,<EOL>node: BaseEntity,<EOL>key: Optional[str] = None,<EOL>tag: Optional[str] = None,<EOL>default_score: Optional[float] = None,<EOL>runs: Optional[int] = None,<EOL>minimum_nodes: int = <NUM_LIT:1>,<EOL>) -> List['<STR_LIT>']:

subgraph = generate_mechanism(graph, node, key=key)<EOL>if subgraph.number_of_nodes() <= minimum_nodes:<EOL><INDENT>return []<EOL><DEDENT>runners = multirun(subgraph, node, key=key, tag=tag, default_score=default_score, runs=runs)<EOL>return list(runners)<EOL>

Generate candidate mechanisms and run the heat diffusion workflow.

    :param graph: A BEL graph
    :param node: The BEL node that is the focus of this analysis
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param minimum_nodes: The minimum number of nodes a sub-graph needs to try running heat diffusion
    :return: A list of runners


def multirun(graph: BELGraph,<EOL>node: BaseEntity,<EOL>key: Optional[str] = None,<EOL>tag: Optional[str] = None,<EOL>default_score: Optional[float] = None,<EOL>runs: Optional[int] = None,<EOL>use_tqdm: bool = False,<EOL>) -> Iterable['<STR_LIT>']:

if runs is None:<EOL><INDENT>runs = <NUM_LIT:100><EOL><DEDENT>it = range(runs)<EOL>if use_tqdm:<EOL><INDENT>it = tqdm(it, total=runs)<EOL><DEDENT>for i in it:<EOL><INDENT>try:<EOL><INDENT>runner = Runner(graph, node, key=key, tag=tag, default_score=default_score)<EOL>runner.run()<EOL>yield runner<EOL><DEDENT>except Exception:<EOL><INDENT>log.debug('<STR_LIT>', i, node)<EOL><DEDENT><DEDENT>

Run the heat diffusion workflow multiple times, each time yielding a :class:`Runner` object upon completion.

    :param graph: A BEL graph
    :param node: The BEL node that is the focus of this analysis
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param use_tqdm: Should there be a progress bar for runners?
    :return: An iterable over the runners after each iteration


def workflow_aggregate(graph: BELGraph,<EOL>node: BaseEntity,<EOL>key: Optional[str] = None,<EOL>tag: Optional[str] = None,<EOL>default_score: Optional[float] = None,<EOL>runs: Optional[int] = None,<EOL>aggregator: Optional[Callable[[Iterable[float]], float]] = None,<EOL>) -> Optional[float]:

runners = workflow(graph, node, key=key, tag=tag, default_score=default_score, runs=runs)<EOL>scores = [runner.get_final_score() for runner in runners]<EOL>if not scores:<EOL><INDENT>log.warning('<STR_LIT>', node)<EOL>return<EOL><DEDENT>if aggregator is None:<EOL><INDENT>return np.average(scores)<EOL><DEDENT>return aggregator(scores)<EOL>

Get the average score over multiple runs.

    This function is very simple, and can be copied to do more interesting statistics over the :class:`Runner`
    instances. To iterate over the runners themselves, see :func:`workflow`

    :param graph: A BEL graph
    :param node: The BEL node that is the focus of this analysis
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param aggregator: A function that aggregates a list of scores. Defaults to :func:`numpy.average`.
                       Could also use: :func:`numpy.mean`, :func:`numpy.median`, :func:`numpy.min`, :func:`numpy.max`
    :return: The average score for the target node


def workflow_all(graph: BELGraph,<EOL>key: Optional[str] = None,<EOL>tag: Optional[str] = None,<EOL>default_score: Optional[float] = None,<EOL>runs: Optional[int] = None,<EOL>) -> Mapping[BaseEntity, List[Runner]]:

results = {}<EOL>for node in get_nodes_by_function(graph, BIOPROCESS):<EOL><INDENT>results[node] = workflow(graph, node, key=key, tag=tag, default_score=default_score, runs=runs)<EOL><DEDENT>return results<EOL>

Run the heat diffusion workflow and get runners for every possible candidate mechanism

    1. Get all biological processes
    2. Get candidate mechanism induced two level back from each biological process
    3. Heat diffusion workflow for each candidate mechanism for multiple runs
    4. Return all runner results

    :param graph: A BEL graph
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :return: A dictionary of {node: list of runners}


def workflow_all_aggregate(graph: BELGraph,<EOL>key: Optional[str] = None,<EOL>tag: Optional[str] = None,<EOL>default_score: Optional[float] = None,<EOL>runs: Optional[int] = None,<EOL>aggregator: Optional[Callable[[Iterable[float]], float]] = None,<EOL>):

results = {}<EOL>bioprocess_nodes = list(get_nodes_by_function(graph, BIOPROCESS))<EOL>for bioprocess_node in tqdm(bioprocess_nodes):<EOL><INDENT>subgraph = generate_mechanism(graph, bioprocess_node, key=key)<EOL>try:<EOL><INDENT>results[bioprocess_node] = workflow_aggregate(<EOL>graph=subgraph,<EOL>node=bioprocess_node,<EOL>key=key,<EOL>tag=tag,<EOL>default_score=default_score,<EOL>runs=runs,<EOL>aggregator=aggregator<EOL>)<EOL><DEDENT>except Exception:<EOL><INDENT>log.exception('<STR_LIT>', bioprocess_node)<EOL><DEDENT><DEDENT>return results<EOL>

Run the heat diffusion workflow to get average score for every possible candidate mechanism.

    1. Get all biological processes
    2. Get candidate mechanism induced two level back from each biological process
    3. Heat diffusion workflow on each candidate mechanism for multiple runs
    4. Report average scores for each candidate mechanism

    :param graph: A BEL graph
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param aggregator: A function that aggregates a list of scores. Defaults to :func:`numpy.average`.
                       Could also use: :func:`numpy.mean`, :func:`numpy.median`, :func:`numpy.min`, :func:`numpy.max`
    :return: A dictionary of {node: upstream causal subgraph}


def calculate_average_score_by_annotation(<EOL>graph: BELGraph,<EOL>annotation: str,<EOL>key: Optional[str] = None,<EOL>runs: Optional[int] = None,<EOL>use_tqdm: bool = False,<EOL>) -> Mapping[str, float]:

candidate_mechanisms = generate_bioprocess_mechanisms(graph, key=key)<EOL>scores: Mapping[BaseEntity, Tuple] = calculate_average_scores_on_subgraphs(<EOL>subgraphs=candidate_mechanisms,<EOL>key=key,<EOL>runs=runs,<EOL>use_tqdm=use_tqdm,<EOL>)<EOL>subgraph_bp: Mapping[str, List[BaseEntity]] = defaultdict(list)<EOL>subgraphs: Mapping[str, BELGraph] = get_subgraphs_by_annotation(graph, annotation)<EOL>for annotation_value, subgraph in subgraphs.items():<EOL><INDENT>subgraph_bp[annotation_value].extend(get_nodes_by_function(subgraph, BIOPROCESS))<EOL><DEDENT>return {<EOL>annotation_value: np.average(scores[bp][<NUM_LIT:0>] for bp in bps)<EOL>for annotation_value, bps in subgraph_bp.items()<EOL>}<EOL>

For each sub-graph induced over the edges matching the annotation, calculate the average score
    for all of the contained biological processes

    Assumes you haven't done anything yet

    1. Generates biological process upstream candidate mechanistic sub-graphs with
       :func:`generate_bioprocess_mechanisms`
    2. Calculates scores for each sub-graph with :func:`calculate_average_scores_on_sub-graphs`
    3. Overlays data with pbt.integration.overlay_data
    4. Calculates averages with pbt.selection.group_nodes.average_node_annotation

    :param graph: A BEL graph
    :param annotation: A BEL annotation
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param use_tqdm: Should there be a progress bar for runners?
    :return: A dictionary from {str annotation value: tuple scores}

    Example Usage:

    >>> import pybel
    >>> from pybel_tools.integration import overlay_data
    >>> from pybel_tools.analysis.heat import calculate_average_score_by_annotation
    >>> graph = pybel.from_path(...)
    >>> scores = calculate_average_score_by_annotation(graph, 'subgraph')


def __init__(self,<EOL>graph: BELGraph,<EOL>target_node: BaseEntity,<EOL>key: Optional[str] = None,<EOL>tag: Optional[str] = None,<EOL>default_score: Optional[float] = None,<EOL>) -> None:

self.graph: BELGraph = graph.copy()<EOL>self.target_node = target_node<EOL>self.key = key or WEIGHT<EOL>self.default_score = default_score or DEFAULT_SCORE<EOL>self.tag = tag or SCORE<EOL>for node, data in self.graph.nodes(data=True):<EOL><INDENT>if not self.graph.predecessors(node):<EOL><INDENT>self.graph.nodes[node][self.tag] = data.get(self.key, <NUM_LIT:0>)<EOL>log.log(<NUM_LIT:5>, '<STR_LIT>', target_node, self.graph.nodes[node][self.tag])<EOL><DEDENT><DEDENT>

Initialize the heat diffusion runner class.

        :param graph: A BEL graph
        :param target_node: The BEL node that is the focus of this analysis
        :param key: The key in the node data dictionary representing the experimental data. Defaults to
         :data:`pybel_tools.constants.WEIGHT`.
        :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
        :param default_score: The initial score for all nodes. This number can go up or down.


def iter_leaves(self) -> Iterable[BaseEntity]:

for node in self.graph:<EOL><INDENT>if self.tag in self.graph.nodes[node]:<EOL><INDENT>continue<EOL><DEDENT>if not any(self.tag not in self.graph.nodes[p] for p in self.graph.predecessors(node)):<EOL><INDENT>yield node<EOL><DEDENT><DEDENT>

Return an iterable over all nodes that are leaves.

        A node is a leaf if either:

         - it doesn't have any predecessors, OR
         - all of its predecessors have a score in their data dictionaries


def has_leaves(self) -> List[BaseEntity]:

leaves = list(self.iter_leaves())<EOL>return leaves<EOL>

Return if the current graph has any leaves.

        Implementation is not that smart currently, and does a full sweep.


def in_out_ratio(self, node: BaseEntity) -> float:

return self.graph.in_degree(node) / float(self.graph.out_degree(node))<EOL>

Calculate the ratio of in-degree / out-degree of a node.


def unscored_nodes_iter(self) -> BaseEntity:

for node, data in self.graph.nodes(data=True):<EOL><INDENT>if self.tag not in data:<EOL><INDENT>yield node<EOL><DEDENT><DEDENT>


nodes = [<EOL>(n, self.in_out_ratio(n))<EOL>for n in self.unscored_nodes_iter()<EOL>if n != self.target_node<EOL>]<EOL>node, deg = min(nodes, key=itemgetter(<NUM_LIT:1>))<EOL>log.log(<NUM_LIT:5>, '<STR_LIT>', node, deg)<EOL>possible_edges = self.graph.in_edges(node, keys=True)<EOL>log.log(<NUM_LIT:5>, '<STR_LIT>', possible_edges)<EOL>edge_to_remove = random.choice(possible_edges)<EOL>log.log(<NUM_LIT:5>, '<STR_LIT>', edge_to_remove)<EOL>return edge_to_remove<EOL>

This function should be run when there are no leaves, but there are still unscored nodes. It will introduce
        a probabilistic element to the algorithm, where some edges are disregarded randomly to eventually get a score
        for the network. This means that the score can be averaged over many runs for a given graph, and a better
        data structure will have to be later developed that doesn't destroy the graph (instead, annotates which edges
        have been disregarded, later)

           1. get all un-scored
           2. rank by in-degree
           3. weighted probability over all in-edges where lower in-degree means higher probability
           4. pick randomly which edge

        :return: A random in-edge to the lowest in/out degree ratio node. This is a 3-tuple of (node, node, key)
        :rtype: tuple


u, v, k = self.get_random_edge()<EOL>log.log(<NUM_LIT:5>, '<STR_LIT>', u, v, k)<EOL>self.graph.remove_edge(u, v, k)<EOL>

Remove a random in-edge from the node with the lowest in/out degree ratio.


def remove_random_edge_until_has_leaves(self) -> None:

while True:<EOL><INDENT>leaves = set(self.iter_leaves())<EOL>if leaves:<EOL><INDENT>return<EOL><DEDENT>self.remove_random_edge()<EOL><DEDENT>

Remove random edges until there is at least one leaf node.


def score_leaves(self) -> Set[BaseEntity]:

leaves = set(self.iter_leaves())<EOL>if not leaves:<EOL><INDENT>log.warning('<STR_LIT>')<EOL>return set()<EOL><DEDENT>for leaf in leaves:<EOL><INDENT>self.graph.nodes[leaf][self.tag] = self.calculate_score(leaf)<EOL>log.log(<NUM_LIT:5>, '<STR_LIT>', leaf)<EOL><DEDENT>return leaves<EOL>

Calculate the score for all leaves.

        :return: The set of leaf nodes that were scored


while not self.done_chomping():<EOL><INDENT>self.remove_random_edge_until_has_leaves()<EOL>self.score_leaves()<EOL><DEDENT>

Calculate scores for all leaves until there are none, removes edges until there are, and repeats until
        all nodes have been scored.


def run_with_graph_transformation(self) -> Iterable[BELGraph]:

yield self.get_remaining_graph()<EOL>while not self.done_chomping():<EOL><INDENT>while not list(self.iter_leaves()):<EOL><INDENT>self.remove_random_edge()<EOL>yield self.get_remaining_graph()<EOL><DEDENT>self.score_leaves()<EOL>yield self.get_remaining_graph()<EOL><DEDENT>

Calculate scores for all leaves until there are none, removes edges until there are, and repeats until
        all nodes have been scored. Also, yields the current graph at every step so you can make a cool animation
        of how the graph changes throughout the course of the algorithm

        :return: An iterable of BEL graphs


return self.tag in self.graph.nodes[self.target_node]<EOL>

Determines if the algorithm is complete by checking if the target node of this analysis has been scored
        yet. Because the algorithm removes edges when it gets stuck until it is un-stuck, it is always guaranteed to
        finish.

        :return: Is the algorithm done running?


if not self.done_chomping():<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>return self.graph.nodes[self.target_node][self.tag]<EOL>

Return the final score for the target node.

        :return: The final score for the target node


def calculate_score(self, node: BaseEntity) -> float:

score = (<EOL>self.graph.nodes[node][self.tag]<EOL>if self.tag in self.graph.nodes[node] else<EOL>self.default_score<EOL>)<EOL>for predecessor, _, d in self.graph.in_edges(node, data=True):<EOL><INDENT>if d[RELATION] in CAUSAL_INCREASE_RELATIONS:<EOL><INDENT>score += self.graph.nodes[predecessor][self.tag]<EOL><DEDENT>elif d[RELATION] in CAUSAL_DECREASE_RELATIONS:<EOL><INDENT>score -= self.graph.nodes[predecessor][self.tag]<EOL><DEDENT><DEDENT>return score<EOL>

Calculate the new score of the given node.


def get_remaining_graph(self) -> BELGraph:

return self.graph.subgraph(self.unscored_nodes_iter())<EOL>

Allows for introspection on the algorithm at a given point by returning the sub-graph induced
        by all unscored nodes

        :return: The remaining un-scored BEL graph


nums = []<EOL>for c in columns:<EOL><INDENT>for p in c.split('<STR_LIT:U+002C>'):<EOL><INDENT>p = p.strip()<EOL>try:<EOL><INDENT>c = int(p)<EOL>nums.append(c)<EOL><DEDENT>except (TypeError, ValueError):<EOL><INDENT>start, ignore, end = p.partition('<STR_LIT:->')<EOL>try:<EOL><INDENT>start = int(start)<EOL>end = int(end)<EOL><DEDENT>except (TypeError, ValueError):<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>' % c<EOL>)<EOL><DEDENT>inc = <NUM_LIT:1> if start < end else -<NUM_LIT:1><EOL>nums.extend(range(start, end + inc, inc))<EOL><DEDENT><DEDENT><DEDENT>return [n - <NUM_LIT:1> for n in nums]<EOL>

Turn column inputs from user into list of simple numbers.

    Inputs can be:

      - individual number: 1
      - range: 1-3
      - comma separated list: 1,2,3,4-6


if not columns:<EOL><INDENT>return row<EOL><DEDENT>return tuple(row[c] for c in columns)<EOL>

Return only the part of the row which should be printed.


def extract_json(fileobj, keywords, comment_tags, options):

data=fileobj.read()<EOL>json_extractor=JsonExtractor(data)<EOL>strings_data=json_extractor.get_lines_data()<EOL>for item in strings_data:<EOL><INDENT>messages = [item['<STR_LIT:content>']]<EOL>if item.get('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>messages.append(item['<STR_LIT>'])<EOL><DEDENT>yield item['<STR_LIT>'],item.get('<STR_LIT>','<STR_LIT>'),tuple(messages),[]<EOL><DEDENT>

Supports: gettext, ngettext. See package README or github ( https://github.com/tigrawap/pybabel-json ) for more usage info.


encoding = '<STR_LIT:utf-8>'<EOL>for token in tokenize(self.data.decode(encoding)):<EOL><INDENT>if token.type == '<STR_LIT>':<EOL><INDENT>if token.value == '<STR_LIT:{>':<EOL><INDENT>self.start_object()<EOL><DEDENT>elif token.value =='<STR_LIT::>':<EOL><INDENT>self.with_separator(token)<EOL><DEDENT>elif token.value == '<STR_LIT:}>':<EOL><INDENT>self.end_object()<EOL><DEDENT>elif token.value == '<STR_LIT:U+002C>':<EOL><INDENT>self.end_pair()<EOL><DEDENT><DEDENT>elif token.type=='<STR_LIT:string>':<EOL><INDENT>if self.state=='<STR_LIT:key>':<EOL><INDENT>self.current_key=unquote_string(token.value)<EOL>if self.current_key==JSON_GETTEXT_KEYWORD:<EOL><INDENT>self.gettext_mode=True<EOL><DEDENT><DEDENT>if self.gettext_mode:<EOL><INDENT>if self.current_key==JSON_GETTEXT_KEY_CONTENT:<EOL><INDENT>self.token_to_add=token<EOL><DEDENT>elif self.current_key==JSON_GETTEXT_KEY_ALT_CONTENT:<EOL><INDENT>self.token_params['<STR_LIT>']=token<EOL><DEDENT>elif self.current_key==JSON_GETTEXT_KEY_FUNCNAME:<EOL><INDENT>self.token_params['<STR_LIT>']=token.value<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.token_to_add=token<EOL><DEDENT><DEDENT><DEDENT>return self.results<EOL>

Returns string:line_numbers list
Since all strings are unique it is OK to get line numbers this way.
Since same string can occur several times inside single .json file the values should be popped(FIFO) from the list
:rtype: list


if os.path.exists(self.path):<EOL><INDENT>os.remove(self.path)<EOL><DEDENT>

Called after each test to remove the file


data = {} if data.lower() == "<STR_LIT>" else []<EOL>if not os.path.exists(path):  <EOL><INDENT>dirname = os.path.dirname(path)<EOL>if dirname and not os.path.exists(dirname):<EOL><INDENT>raise IOError(<EOL>("<STR_LIT>"<EOL>"<STR_LIT>").format(os.path.dirname(path))<EOL>)<EOL><DEDENT>with open(path, "<STR_LIT:w>") as f:<EOL><INDENT>json.dump(data, f)<EOL><DEDENT>return True<EOL><DEDENT>elif os.path.getsize(path) == <NUM_LIT:0>:  <EOL><INDENT>with open(path, "<STR_LIT:w>") as f:<EOL><INDENT>json.dump(data, f)<EOL><DEDENT><DEDENT>else:  <EOL><INDENT>return False<EOL><DEDENT>


pass<EOL>

Make sure the type of a key is appropriate.


if self.is_caching:<EOL><INDENT>return self.cache<EOL><DEDENT>with open(self.path, "<STR_LIT:r>") as f:<EOL><INDENT>return json.load(f)<EOL><DEDENT>

A simpler version of data to avoid infinite recursion in some cases.

        Don't use this.


<EOL>self._updateType()<EOL>return self._data()<EOL>

Get a vanilla dict object to represent the file.


if self.is_caching:<EOL><INDENT>self.cache = data<EOL><DEDENT>else:<EOL><INDENT>fcontents = self.file_contents<EOL>with open(self.path, "<STR_LIT:w>") as f:<EOL><INDENT>try:<EOL><INDENT>indent = self.indent if self.pretty else None<EOL>json.dump(data, f, sort_keys=self.sort_keys, indent=indent)<EOL><DEDENT>except Exception as e:<EOL><INDENT>f.seek(<NUM_LIT:0>)<EOL>f.truncate()<EOL>f.write(fcontents)<EOL>raise e<EOL><DEDENT><DEDENT><DEDENT>self._updateType()<EOL>

Overwrite the file with new data. You probably shouldn't do
        this yourself, it's easy to screw up your whole file with this.


data = self._data()<EOL>if isinstance(data, dict) and isinstance(self, ListFile):<EOL><INDENT>self.__class__ = DictFile<EOL><DEDENT>elif isinstance(data, list) and isinstance(self, DictFile):<EOL><INDENT>self.__class__ = ListFile<EOL><DEDENT>

Make sure that the class behaves like the data structure that it
        is, so that we don't get a ListFile trying to represent a dict.


warnings.warn(<EOL>"<STR_LIT>",<EOL>DeprecationWarning<EOL>)<EOL>self.data = data<EOL>

Equivalent to setting the "data" attribute. Exists for backwards
        compatibility.


os.remove(self.path)<EOL>

Delete the file from the disk completely.


with open(self.path, "<STR_LIT:r>") as f:<EOL><INDENT>return f.read()<EOL><DEDENT>


return hasattr(self, "<STR_LIT>")<EOL>

Returns a boolean value describing whether a grouped write is
        underway.


self.data = []<EOL>

L.clear() -> None -- remove all items from L.


<EOL>if isinstance(data, str):<EOL><INDENT>data = json.loads(data)<EOL><DEDENT>if os.path.exists(path):<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>f = File(path)<EOL>f.data = data<EOL>return f<EOL><DEDENT>

Initialize a new file that starts out with some data. Pass data
        as a list, dict, or JSON string.


def get_fuel_prices(self) -> GetFuelPricesResponse:

response = requests.get(<EOL>'<STR_LIT>'.format(API_URL_BASE),<EOL>headers=self._get_headers(),<EOL>timeout=self._timeout,<EOL>)<EOL>if not response.ok:<EOL><INDENT>raise FuelCheckError.create(response)<EOL><DEDENT>return GetFuelPricesResponse.deserialize(response.json())<EOL>


def get_fuel_prices_for_station(<EOL>self,<EOL>station: int<EOL>) -> List[Price]:

response = requests.get(<EOL>'<STR_LIT>'.format(API_URL_BASE, station),<EOL>headers=self._get_headers(),<EOL>timeout=self._timeout,<EOL>)<EOL>if not response.ok:<EOL><INDENT>raise FuelCheckError.create(response)<EOL><DEDENT>data = response.json()<EOL>return [Price.deserialize(data) for data in data['<STR_LIT>']]<EOL>

Gets the fuel prices for a specific fuel station.


def get_fuel_prices_within_radius(<EOL>self, latitude: float, longitude: float, radius: int,<EOL>fuel_type: str, brands: Optional[List[str]] = None<EOL>) -> List[StationPrice]:

if brands is None:<EOL><INDENT>brands = []<EOL><DEDENT>response = requests.post(<EOL>'<STR_LIT>'.format(API_URL_BASE),<EOL>json={<EOL>'<STR_LIT>': fuel_type,<EOL>'<STR_LIT>': latitude,<EOL>'<STR_LIT>': longitude,<EOL>'<STR_LIT>': radius,<EOL>'<STR_LIT>': brands,<EOL>},<EOL>headers=self._get_headers(),<EOL>timeout=self._timeout,<EOL>)<EOL>if not response.ok:<EOL><INDENT>raise FuelCheckError.create(response)<EOL><DEDENT>data = response.json()<EOL>stations = {<EOL>station['<STR_LIT:code>']: Station.deserialize(station)<EOL>for station in data['<STR_LIT>']<EOL>}<EOL>station_prices = []  <EOL>for serialized_price in data['<STR_LIT>']:<EOL><INDENT>price = Price.deserialize(serialized_price)<EOL>station_prices.append(StationPrice(<EOL>price=price,<EOL>station=stations[price.station_code]<EOL>))<EOL><DEDENT>return station_prices<EOL>

Gets all the fuel prices within the specified radius.


def get_fuel_price_trends(self, latitude: float, longitude: float,<EOL>fuel_types: List[str]) -> PriceTrends:

response = requests.post(<EOL>'<STR_LIT>'.format(API_URL_BASE),<EOL>json={<EOL>'<STR_LIT:location>': {<EOL>'<STR_LIT>': latitude,<EOL>'<STR_LIT>': longitude,<EOL>},<EOL>'<STR_LIT>': [{'<STR_LIT:code>': type} for type in fuel_types],<EOL>},<EOL>headers=self._get_headers(),<EOL>timeout=self._timeout,<EOL>)<EOL>if not response.ok:<EOL><INDENT>raise FuelCheckError.create(response)<EOL><DEDENT>data = response.json()<EOL>return PriceTrends(<EOL>variances=[<EOL>Variance.deserialize(variance)<EOL>for variance in data['<STR_LIT>']<EOL>],<EOL>average_prices=[<EOL>AveragePrice.deserialize(avg_price)<EOL>for avg_price in data['<STR_LIT>']<EOL>]<EOL>)<EOL>

Gets the fuel price trends for the given location and fuel types.


def get_reference_data(<EOL>self,<EOL>modified_since: Optional[datetime.datetime] = None<EOL>) -> GetReferenceDataResponse:

if modified_since is None:<EOL><INDENT>modified_since = datetime.datetime(year=<NUM_LIT>, month=<NUM_LIT:1>, day=<NUM_LIT:1>)<EOL><DEDENT>response = requests.get(<EOL>'<STR_LIT>'.format(API_URL_BASE),<EOL>headers={<EOL>'<STR_LIT>': self._format_dt(modified_since),<EOL>**self._get_headers(),<EOL>},<EOL>timeout=self._timeout,<EOL>)<EOL>if not response.ok:<EOL><INDENT>raise FuelCheckError.create(response)<EOL><DEDENT>return GetReferenceDataResponse.deserialize(response.json())<EOL>

Fetches API reference data.

:param modified_since: The response will be empty if no
changes have been made to the reference data since this
timestamp, otherwise all reference data will be returned.


def test_func_decorator(test_func):<EOL><INDENT>test_func._provider_method = provider_method<EOL>test_func._provider_name_suffix = first_param_name_suffix<EOL>return test_func<EOL><DEDENT>return test_func_decorator<EOL>

A method decorator for unittest.TestCase classes that configured a
static method to be used to provide multiple sets of test data to a single
test

:param provider_method:
    The name of the staticmethod of the class to use as the data provider

:param first_param_name_suffix:
    If the first parameter for each set should be appended to the method
    name to generate the name of the test. Otherwise integers are used.

:return:
    The decorated function


def generate_test_func(name, original_function, num, params):<EOL><INDENT>if original_function._provider_name_suffix:<EOL><INDENT>data_name = params[<NUM_LIT:0>]<EOL>params = params[<NUM_LIT:1>:]<EOL><DEDENT>else:<EOL><INDENT>data_name = num<EOL><DEDENT>expanded_name = '<STR_LIT>' % (name, data_name)<EOL>def generated_test_function(self):<EOL><INDENT>original_function(self, *params)<EOL><DEDENT>setattr(cls, expanded_name, generated_test_function)<EOL><DEDENT>for name in dir(cls):<EOL><INDENT>func = getattr(cls, name)<EOL>if hasattr(func, '<STR_LIT>'):<EOL><INDENT>num = <NUM_LIT:1><EOL>for params in getattr(cls, func._provider_method)():<EOL><INDENT>generate_test_func(name, func, num, params)<EOL>num += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>return cls<EOL>

A class decorator that works with the @provider decorator to generate test
method from a data provider


if not self.socket:<EOL><INDENT>return<EOL><DEDENT>self.socket.close()<EOL>self.socket = None<EOL>


self.setup_connection(url, timeout)<EOL>tries = <NUM_LIT:0><EOL>while tries < <NUM_LIT:2>:<EOL><INDENT>tries += <NUM_LIT:1><EOL>try:<EOL><INDENT>self.ensure_connected()<EOL>req_headers = OrderedDict()<EOL>req_headers['<STR_LIT>'] = self.url_info[<NUM_LIT:0>]<EOL>if self.url_info[<NUM_LIT:1>] != <NUM_LIT>:<EOL><INDENT>req_headers['<STR_LIT>'] += '<STR_LIT>' % self.url_info[<NUM_LIT:1>]<EOL><DEDENT>req_headers['<STR_LIT>'] = '<STR_LIT>' if self.keep_alive else '<STR_LIT>'<EOL>req_headers["<STR_LIT>"] = '<STR_LIT>' % version.__version__<EOL>request = '<STR_LIT>'<EOL>url_info = urlparse(url)<EOL>path = '<STR_LIT:/>' if not url_info.path else url_info.path<EOL>if url_info.query:<EOL><INDENT>path += '<STR_LIT:?>' + url_info.query<EOL><DEDENT>request += path + '<STR_LIT>'<EOL>self.write_request(request, req_headers)<EOL>response = self.read_headers()<EOL>if not response:<EOL><INDENT>self.close()<EOL>continue<EOL><DEDENT>v, code, message, resp_headers = response<EOL>data = self.read_body(code, resp_headers, timeout)<EOL>if code == <NUM_LIT>:<EOL><INDENT>location = resp_headers.get('<STR_LIT:location>')<EOL>if not isinstance(location, str_cls):<EOL><INDENT>raise HttpsClientError('<STR_LIT>')<EOL><DEDENT>if not re.match(r'<STR_LIT>', location):<EOL><INDENT>if not location.startswith('<STR_LIT:/>'):<EOL><INDENT>location = os.path.dirname(url_info.path) + location<EOL><DEDENT>location = url_info.scheme + '<STR_LIT>' + url_info.netloc + location<EOL><DEDENT>return self.download(location, timeout)<EOL><DEDENT>if code != <NUM_LIT:200>:<EOL><INDENT>raise HttpsClientError('<STR_LIT>' % (code, url))<EOL><DEDENT>else:<EOL><INDENT>return data<EOL><DEDENT><DEDENT>except (oscrypto_errors.TLSGracefulDisconnectError):<EOL><INDENT>self.close()<EOL>continue<EOL><DEDENT><DEDENT>

Downloads a URL and returns the contents

:param url:
    The URL to download

:param timeout:
    The int number of seconds to set the timeout to

:return:
    The string contents of the URL


def setup_connection(self, url, timeout):

url_info = urlparse(url)<EOL>if url_info.scheme == '<STR_LIT:http>':<EOL><INDENT>raise HttpsClientException('<STR_LIT>')<EOL><DEDENT>hostname = url_info.hostname<EOL>port = url_info.port<EOL>if not port:<EOL><INDENT>port = <NUM_LIT><EOL><DEDENT>if self.socket and self.url_info != (hostname, port):<EOL><INDENT>self.close()<EOL><DEDENT>self.timeout = timeout<EOL>self.url_info = (hostname, port)<EOL>return self.ensure_connected()<EOL>

:param url:
    The URL to download

:param timeout:
    The int number of seconds to set the timeout to

:return:
    A boolean indicating if the connection was reused


if self.socket:<EOL><INDENT>return True<EOL><DEDENT>host, port = self.url_info<EOL>session = tls.TLSSession()<EOL>self.socket = tls.TLSSocket(host, port, timeout=self.timeout, session=session)<EOL>return False<EOL>

Make sure a valid tls.TLSSocket() is open to the server

:return:
    A boolean indicating if the connection was reused


def write_request(self, request, headers):

lines = [request]<EOL>for header, value in headers.items():<EOL><INDENT>lines.append('<STR_LIT>' % (header, value))<EOL><DEDENT>lines.extend(['<STR_LIT>', '<STR_LIT>'])<EOL>request = '<STR_LIT:\r\n>'.join(lines).encode('<STR_LIT>')<EOL>self.socket.write(request)<EOL>

:param request:
    A unicode string of the first line of the HTTP request

:param headers:
    An OrderedDict of the request headers


version = None<EOL>code = None<EOL>text = None<EOL>headers = OrderedDict()<EOL>data = self.socket.read_until(b'<STR_LIT>')<EOL>string = data.decode('<STR_LIT>')<EOL>first = False<EOL>for line in string.split('<STR_LIT:\r\n>'):<EOL><INDENT>line = line.strip()<EOL>if first is False:<EOL><INDENT>if line == '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>match = re.match(r'<STR_LIT>', line)<EOL>if not match:<EOL><INDENT>return None<EOL><DEDENT>version = tuple(map(int, match.group(<NUM_LIT:1>).split('<STR_LIT:.>')))<EOL>code = int(match.group(<NUM_LIT:2>))<EOL>text = match.group(<NUM_LIT:3>)<EOL>first = True<EOL><DEDENT>else:<EOL><INDENT>if not len(line):<EOL><INDENT>continue<EOL><DEDENT>parts = line.split('<STR_LIT::>', <NUM_LIT:1>)<EOL>if len(parts) == <NUM_LIT:2>:<EOL><INDENT>name = parts[<NUM_LIT:0>].strip().lower()<EOL>value = parts[<NUM_LIT:1>].strip()<EOL>if name in headers:<EOL><INDENT>if isinstance(headers[name], tuple):<EOL><INDENT>headers[name] = headers[name] + (value,)<EOL><DEDENT>else:<EOL><INDENT>headers[name] = (headers[name], value)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>headers[name] = value<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return (version, code, text, headers)<EOL>

Reads the HTTP response headers from the socket

:return:
    On error, None, otherwise a 4-element tuple:
      0: A 2-element tuple of integers representing the HTTP version
      1: An integer representing the HTTP response code
      2: A unicode string of the HTTP response code name
      3: An OrderedDict of HTTP headers with lowercase unicode key and unicode values


content_length = headers.get('<STR_LIT>')<EOL>if isinstance(content_length, str_cls) and len(content_length) > <NUM_LIT:0>:<EOL><INDENT>content_length = int(content_length)<EOL><DEDENT>return content_length<EOL>

Returns the content-length from a dict of headers

:return:
    An integer of the content length


def _assert_regexp_matches(self, text, expected_regexp, msg=None):

if isinstance(expected_regexp, str_cls):<EOL><INDENT>expected_regexp = re.compile(expected_regexp)<EOL><DEDENT>if not expected_regexp.search(text):<EOL><INDENT>msg = msg or "<STR_LIT>"<EOL>msg = '<STR_LIT>' % (msg, expected_regexp.pattern, text)<EOL>self.fail(msg)<EOL><DEDENT>

Fail the test unless the text matches the regular expression.


global _local_module<EOL>if _local_module:<EOL><INDENT>return _local_module<EOL><DEDENT>module_name = '<STR_LIT>'<EOL>src_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '<STR_LIT:..>')<EOL>module_info = imp.find_module(module_name, [src_dir])<EOL>_local_module = imp.load_module(module_name, *module_info)<EOL>if os.environ.get('<STR_LIT>'):<EOL><INDENT>paths = os.environ.get('<STR_LIT>').split('<STR_LIT:U+002C>')<EOL>if len(paths) != <NUM_LIT:2>:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>_local_module.use_openssl(*paths)<EOL><DEDENT>elif os.environ.get('<STR_LIT>'):<EOL><INDENT>_local_module.use_winlegacy()<EOL><DEDENT>return _local_module<EOL>

Make sure oscrypto is initialized and the backend is selected via env vars

:return:
    The oscrypto module


loader = unittest.TestLoader()<EOL>suite = unittest.TestSuite()<EOL>for test_class in test_classes():<EOL><INDENT>tests = loader.loadTestsFromTestCase(test_class)<EOL>suite.addTests(tests)<EOL><DEDENT>return suite<EOL>

Constructs a unittest.TestSuite() of all tests for the package. For use
with setuptools.

:return:
    A unittest.TestSuite() object


setup_file = os.path.join(package_root, '<STR_LIT>')<EOL>git_wc_proc = subprocess.Popen(<EOL>['<STR_LIT>', '<STR_LIT:status>', '<STR_LIT>', '<STR_LIT>'],<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.STDOUT,<EOL>cwd=package_root<EOL>)<EOL>git_wc_status, _ = git_wc_proc.communicate()<EOL>if len(git_wc_status) > <NUM_LIT:0>:<EOL><INDENT>print(git_wc_status.decode('<STR_LIT:utf-8>').rstrip(), file=sys.stderr)<EOL>print('<STR_LIT>', file=sys.stderr)<EOL>return False<EOL><DEDENT>git_tag_proc = subprocess.Popen(<EOL>['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'],<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE,<EOL>cwd=package_root<EOL>)<EOL>tag, tag_error = git_tag_proc.communicate()<EOL>if len(tag_error) > <NUM_LIT:0>:<EOL><INDENT>print(tag_error.decode('<STR_LIT:utf-8>').rstrip(), file=sys.stderr)<EOL>print('<STR_LIT>', file=sys.stderr)<EOL>return False<EOL><DEDENT>if len(tag) == <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>', file=sys.stderr)<EOL>return False<EOL><DEDENT>tag = tag.decode('<STR_LIT:ascii>').strip()<EOL>setuptools.sandbox.run_setup(<EOL>setup_file,<EOL>['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>)<EOL>twine.cli.dispatch(['<STR_LIT>', '<STR_LIT>' % (package_name, tag)])<EOL>setuptools.sandbox.run_setup(<EOL>setup_file,<EOL>['<STR_LIT>']<EOL>)<EOL>

Creates a sdist .tar.gz and a bdist_wheel --univeral .whl and uploads
them to pypi

:return:
    A bool - if the packaging and upload process was successful


def _get_func_info(docstring, def_lineno, code_lines, prefix):

def_index = def_lineno - <NUM_LIT:1><EOL>definition = code_lines[def_index]<EOL>definition = definition.rstrip()<EOL>while not definition.endswith('<STR_LIT::>'):<EOL><INDENT>def_index += <NUM_LIT:1><EOL>definition += '<STR_LIT:\n>' + code_lines[def_index].rstrip()<EOL><DEDENT>definition = textwrap.dedent(definition).rstrip('<STR_LIT::>')<EOL>definition = definition.replace('<STR_LIT:\n>', '<STR_LIT:\n>' + prefix)<EOL>description = '<STR_LIT>'<EOL>found_colon = False<EOL>params = '<STR_LIT>'<EOL>for line in docstring.splitlines():<EOL><INDENT>if line and line[<NUM_LIT:0>] == '<STR_LIT::>':<EOL><INDENT>found_colon = True<EOL><DEDENT>if not found_colon:<EOL><INDENT>if description:<EOL><INDENT>description += '<STR_LIT:\n>'<EOL><DEDENT>description += line<EOL><DEDENT>else:<EOL><INDENT>if params:<EOL><INDENT>params += '<STR_LIT:\n>'<EOL><DEDENT>params += line<EOL><DEDENT><DEDENT>description = description.strip()<EOL>description_md = '<STR_LIT>'<EOL>if description:<EOL><INDENT>description_md = "<STR_LIT>" % (prefix, description.replace('<STR_LIT:\n>', '<STR_LIT:\n>' + prefix))<EOL>description_md = re.sub('<STR_LIT>', '<STR_LIT>', description_md)<EOL><DEDENT>params = params.strip()<EOL>if params:<EOL><INDENT>definition += ('<STR_LIT>' % (prefix, prefix))<EOL>definition += params.replace('<STR_LIT:\n>', '<STR_LIT>' % prefix)<EOL>definition += ('<STR_LIT>' % prefix)<EOL>definition = re.sub('<STR_LIT>', '<STR_LIT>', definition)<EOL><DEDENT>for search, replace in definition_replacements.items():<EOL><INDENT>definition = definition.replace(search, replace)<EOL><DEDENT>return (definition, description_md)<EOL>

Extracts the function signature and description of a Python function

:param docstring:
    A unicode string of the docstring for the function

:param def_lineno:
    An integer line number that function was defined on

:param code_lines:
    A list of unicode string lines from the source file the function was
    defined in

:param prefix:
    A prefix to prepend to all output lines

:return:
    A 2-element tuple:

     - [0] A unicode string of the function signature with a docstring of
           parameter info
     - [1] A markdown snippet of the function description


def _find_sections(md_ast, sections, last, last_class, total_lines=None):

def child_walker(node):<EOL><INDENT>for child, entering in node.walker():<EOL><INDENT>if child == node:<EOL><INDENT>continue<EOL><DEDENT>yield child, entering<EOL><DEDENT><DEDENT>for child, entering in child_walker(md_ast):<EOL><INDENT>if child.t == '<STR_LIT>':<EOL><INDENT>start_line = child.sourcepos[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>if child.level == <NUM_LIT:2>:<EOL><INDENT>if last:<EOL><INDENT>sections[(last['<STR_LIT>'], last['<STR_LIT>'])] = (last['<STR_LIT>'], start_line - <NUM_LIT:1>)<EOL>last.clear()<EOL><DEDENT><DEDENT>if child.level in set([<NUM_LIT:3>, <NUM_LIT:5>]):<EOL><INDENT>heading_elements = []<EOL>for heading_child, _ in child_walker(child):<EOL><INDENT>heading_elements.append(heading_child)<EOL><DEDENT>if len(heading_elements) != <NUM_LIT:2>:<EOL><INDENT>continue<EOL><DEDENT>first = heading_elements[<NUM_LIT:0>]<EOL>second = heading_elements[<NUM_LIT:1>]<EOL>if first.t != '<STR_LIT:code>':<EOL><INDENT>continue<EOL><DEDENT>if second.t != '<STR_LIT:text>':<EOL><INDENT>continue<EOL><DEDENT>type_name = second.literal.strip()<EOL>identifier = first.literal.strip().replace('<STR_LIT>', '<STR_LIT>').lstrip('<STR_LIT:.>')<EOL>if last:<EOL><INDENT>sections[(last['<STR_LIT>'], last['<STR_LIT>'])] = (last['<STR_LIT>'], start_line - <NUM_LIT:1>)<EOL>last.clear()<EOL><DEDENT>if type_name == '<STR_LIT>':<EOL><INDENT>if child.level != <NUM_LIT:3>:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>if type_name == '<STR_LIT:class>':<EOL><INDENT>if child.level != <NUM_LIT:3>:<EOL><INDENT>continue<EOL><DEDENT>last_class.append(identifier)<EOL><DEDENT>if type_name in set(['<STR_LIT>', '<STR_LIT>']):<EOL><INDENT>if child.level != <NUM_LIT:5>:<EOL><INDENT>continue<EOL><DEDENT>identifier = last_class[-<NUM_LIT:1>] + '<STR_LIT:.>' + identifier<EOL><DEDENT>last.update({<EOL>'<STR_LIT>': type_name,<EOL>'<STR_LIT>': identifier,<EOL>'<STR_LIT>': start_line,<EOL>})<EOL><DEDENT><DEDENT>elif child.t == '<STR_LIT>':<EOL><INDENT>find_sections(child, sections, last, last_class)<EOL><DEDENT><DEDENT>if last:<EOL><INDENT>sections[(last['<STR_LIT>'], last['<STR_LIT>'])] = (last['<STR_LIT>'], total_lines)<EOL><DEDENT>

Walks through a CommonMark AST to find section headers that delineate
content that should be updated by this script

:param md_ast:
    The AST of the markdown document

:param sections:
    A dict to store the start and end lines of a section. The key will be
    a two-element tuple of the section type ("class", "function",
    "method" or "attribute") and identifier. The values are a two-element
    tuple of the start and end line number in the markdown document of the
    section.

:param last:
    A dict containing information about the last section header seen.
    Includes the keys "type_name", "identifier", "start_line".

:param last_class:
    A unicode string of the name of the last class found - used when
    processing methods and attributes.

:param total_lines:
    An integer of the total number of lines in the markdown document -
    used to work around a bug in the API of the Python port of CommonMark


def walk_ast(node, code_lines, sections, md_chunks):

if isinstance(node, _ast.FunctionDef):<EOL><INDENT>key = ('<STR_LIT>', node.name)<EOL>if key not in sections:<EOL><INDENT>return<EOL><DEDENT>docstring = ast.get_docstring(node)<EOL>def_lineno = node.lineno + len(node.decorator_list)<EOL>definition, description_md = _get_func_info(docstring, def_lineno, code_lines, '<STR_LIT>')<EOL>md_chunk = textwrap.dedent("""<STR_LIT>""").strip() % (<EOL>node.name,<EOL>definition,<EOL>description_md<EOL>) + "<STR_LIT:\n>"<EOL>md_chunks[key] = md_chunk.replace('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>elif isinstance(node, _ast.ClassDef):<EOL><INDENT>if ('<STR_LIT:class>', node.name) not in sections:<EOL><INDENT>return<EOL><DEDENT>for subnode in node.body:<EOL><INDENT>if isinstance(subnode, _ast.FunctionDef):<EOL><INDENT>node_id = node.name + '<STR_LIT:.>' + subnode.name<EOL>method_key = ('<STR_LIT>', node_id)<EOL>is_method = method_key in sections<EOL>attribute_key = ('<STR_LIT>', node_id)<EOL>is_attribute = attribute_key in sections<EOL>is_constructor = subnode.name == '<STR_LIT>'<EOL>if not is_constructor and not is_attribute and not is_method:<EOL><INDENT>continue<EOL><DEDENT>docstring = ast.get_docstring(subnode)<EOL>def_lineno = subnode.lineno + len(subnode.decorator_list)<EOL>if not docstring:<EOL><INDENT>continue<EOL><DEDENT>if is_method or is_constructor:<EOL><INDENT>definition, description_md = _get_func_info(docstring, def_lineno, code_lines, '<STR_LIT>')<EOL>if is_constructor:<EOL><INDENT>key = ('<STR_LIT:class>', node.name)<EOL>class_docstring = ast.get_docstring(node) or '<STR_LIT>'<EOL>class_description = textwrap.dedent(class_docstring).strip()<EOL>if class_description:<EOL><INDENT>class_description_md = "<STR_LIT>" % (class_description.replace("<STR_LIT:\n>", "<STR_LIT>"))<EOL><DEDENT>else:<EOL><INDENT>class_description_md = '<STR_LIT>'<EOL><DEDENT>md_chunk = textwrap.dedent("""<STR_LIT>""").strip() % (<EOL>node.name,<EOL>class_description_md,<EOL>definition,<EOL>description_md<EOL>)<EOL>md_chunk = md_chunk.replace('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>key = method_key<EOL>md_chunk = textwrap.dedent("""<STR_LIT>""").strip() % (<EOL>subnode.name,<EOL>definition,<EOL>description_md<EOL>)<EOL><DEDENT>if md_chunk[-<NUM_LIT:5>:] == '<STR_LIT>':<EOL><INDENT>md_chunk = md_chunk[<NUM_LIT:0>:-<NUM_LIT:5>]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>key = attribute_key<EOL>description = textwrap.dedent(docstring).strip()<EOL>description_md = "<STR_LIT>" % (description.replace("<STR_LIT:\n>", "<STR_LIT>"))<EOL>md_chunk = textwrap.dedent("""<STR_LIT>""").strip() % (<EOL>subnode.name,<EOL>description_md<EOL>)<EOL><DEDENT>md_chunks[key] = re.sub('<STR_LIT>', '<STR_LIT:\n>', md_chunk.rstrip())<EOL><DEDENT><DEDENT><DEDENT>elif isinstance(node, _ast.If):<EOL><INDENT>for subast in node.body:<EOL><INDENT>walk_ast(subast, code_lines, sections, md_chunks)<EOL><DEDENT>for subast in node.orelse:<EOL><INDENT>walk_ast(subast, code_lines, sections, md_chunks)<EOL><DEDENT><DEDENT>

A callback used to walk the Python AST looking for classes, functions,
methods and attributes. Generates chunks of markdown markup to replace
the existing content.

:param node:
    An _ast module node object

:param code_lines:
    A list of unicode strings - the source lines of the Python file

:param sections:
    A dict of markdown document sections that need to be updated. The key
    will be a two-element tuple of the section type ("class", "function",
    "method" or "attribute") and identifier. The values are a two-element
    tuple of the start and end line number in the markdown document of the
    section.

:param md_chunks:
    A dict with keys from the sections param and the values being a unicode
    string containing a chunk of markdown markup.


print('<STR_LIT>')<EOL>md_files = []<EOL>for root, _, filenames in os.walk(os.path.join(package_root, '<STR_LIT>')):<EOL><INDENT>for filename in filenames:<EOL><INDENT>if not filename.endswith('<STR_LIT>'):<EOL><INDENT>continue<EOL><DEDENT>md_files.append(os.path.join(root, filename))<EOL><DEDENT><DEDENT>parser = CommonMark.Parser()<EOL>for md_file in md_files:<EOL><INDENT>md_file_relative = md_file[len(package_root) + <NUM_LIT:1>:]<EOL>if md_file_relative in md_source_map:<EOL><INDENT>py_files = md_source_map[md_file_relative]<EOL>py_paths = [os.path.join(package_root, py_file) for py_file in py_files]<EOL><DEDENT>else:<EOL><INDENT>py_files = [os.path.basename(md_file).replace('<STR_LIT>', '<STR_LIT>')]<EOL>py_paths = [os.path.join(package_root, package_name, py_files[<NUM_LIT:0>])]<EOL>if not os.path.exists(py_paths[<NUM_LIT:0>]):<EOL><INDENT>continue<EOL><DEDENT><DEDENT>with open(md_file, '<STR_LIT:rb>') as f:<EOL><INDENT>markdown = f.read().decode('<STR_LIT:utf-8>')<EOL><DEDENT>original_markdown = markdown<EOL>md_lines = list(markdown.splitlines())<EOL>md_ast = parser.parse(markdown)<EOL>last_class = []<EOL>last = {}<EOL>sections = OrderedDict()<EOL>find_sections(md_ast, sections, last, last_class, markdown.count("<STR_LIT:\n>") + <NUM_LIT:1>)<EOL>md_chunks = {}<EOL>for index, py_file in enumerate(py_files):<EOL><INDENT>py_path = py_paths[index]<EOL>with open(os.path.join(py_path), '<STR_LIT:rb>') as f:<EOL><INDENT>code = f.read().decode('<STR_LIT:utf-8>')<EOL>module_ast = ast.parse(code, filename=py_file)<EOL>code_lines = list(code.splitlines())<EOL><DEDENT>for node in ast.iter_child_nodes(module_ast):<EOL><INDENT>walk_ast(node, code_lines, sections, md_chunks)<EOL><DEDENT><DEDENT>added_lines = <NUM_LIT:0><EOL>def _replace_md(key, sections, md_chunk, md_lines, added_lines):<EOL><INDENT>start, end = sections[key]<EOL>start -= <NUM_LIT:1><EOL>start += added_lines<EOL>end += added_lines<EOL>new_lines = md_chunk.split('<STR_LIT:\n>')<EOL>added_lines += len(new_lines) - (end - start)<EOL>if start > <NUM_LIT:0> and md_lines[start][<NUM_LIT:0>:<NUM_LIT:4>] == '<STR_LIT>' and md_lines[start - <NUM_LIT:1>][<NUM_LIT:0>:<NUM_LIT:1>] == '<STR_LIT:>>':<EOL><INDENT>added_lines += <NUM_LIT:1><EOL>new_lines.insert(<NUM_LIT:0>, '<STR_LIT>')<EOL><DEDENT>md_lines[start:end] = new_lines<EOL>return added_lines<EOL><DEDENT>for key in sections:<EOL><INDENT>if key not in md_chunks:<EOL><INDENT>raise ValueError('<STR_LIT>' % key[<NUM_LIT:1>])<EOL><DEDENT>added_lines = _replace_md(key, sections, md_chunks[key], md_lines, added_lines)<EOL><DEDENT>markdown = '<STR_LIT:\n>'.join(md_lines).strip() + '<STR_LIT:\n>'<EOL>if original_markdown != markdown:<EOL><INDENT>with open(md_file, '<STR_LIT:wb>') as f:<EOL><INDENT>f.write(markdown.encode('<STR_LIT:utf-8>'))<EOL><DEDENT><DEDENT><DEDENT>

Looks through the docs/ dir and parses each markdown document, looking for
sections to update from Python docstrings. Looks for section headers in
the format:

 - ### `ClassName()` class
 - ##### `.method_name()` method
 - ##### `.attribute_name` attribute
 - ### `function_name()` function

The markdown content following these section headers up until the next
section header will be replaced by new markdown generated from the Python
docstrings of the associated source files.

By default maps docs/{name}.md to {modulename}/{name}.py. Allows for
custom mapping via the md_source_map variable.


return '<STR_LIT>' if hasattr(sys, '<STR_LIT>') else '<STR_LIT>'<EOL>

:return:
    A 2 character unicode string of the implementation - 'cp' for cpython
    or 'pp' for PyPy


if hasattr(sys, '<STR_LIT>'):<EOL><INDENT>return (sys.version_info[<NUM_LIT:0>], sys.pypy_version_info.major,<EOL>sys.pypy_version_info.minor)<EOL><DEDENT>else:<EOL><INDENT>return (sys.version_info[<NUM_LIT:0>], sys.version_info[<NUM_LIT:1>])<EOL><DEDENT>

:return:
    A tuple of integers representing the Python version number


try:<EOL><INDENT>import _manylinux<EOL>return bool(_manylinux.manylinux1_compatible)<EOL><DEDENT>except (ImportError, AttributeError):<EOL><INDENT>pass<EOL><DEDENT>try:<EOL><INDENT>proc = ctypes.CDLL(None)<EOL>gnu_get_libc_version = proc.gnu_get_libc_version<EOL>gnu_get_libc_version.restype = ctypes.c_char_p<EOL>ver = gnu_get_libc_version()<EOL>if not isinstance(ver, str_cls):<EOL><INDENT>ver = ver.decode('<STR_LIT:ascii>')<EOL><DEDENT>match = re.match(r'<STR_LIT>', ver)<EOL>return match and match.group(<NUM_LIT:1>) == '<STR_LIT:2>' and int(match.group(<NUM_LIT:2>)) >= <NUM_LIT:5><EOL><DEDENT>except (AttributeError):<EOL><INDENT>return False<EOL><DEDENT>

:return:
    A boolean indicating if the machine can use manylinux1 packages


try:<EOL><INDENT>soabi = sysconfig.get_config_var('<STR_LIT>')<EOL>if soabi:<EOL><INDENT>if soabi.startswith('<STR_LIT>'):<EOL><INDENT>return '<STR_LIT>' % soabi.split('<STR_LIT:->')[<NUM_LIT:1>]<EOL><DEDENT>return soabi.replace('<STR_LIT:.>', '<STR_LIT:_>').replace('<STR_LIT:->', '<STR_LIT:_>')<EOL><DEDENT><DEDENT>except (IOError, NameError):<EOL><INDENT>pass<EOL><DEDENT>impl = _pep425_implementation()<EOL>suffix = '<STR_LIT>'<EOL>if impl == '<STR_LIT>':<EOL><INDENT>suffix += '<STR_LIT:m>'<EOL><DEDENT>if sys.maxunicode == <NUM_LIT> and sys.version_info < (<NUM_LIT:3>, <NUM_LIT:3>):<EOL><INDENT>suffix += '<STR_LIT:u>'<EOL><DEDENT>return '<STR_LIT>' % (impl, '<STR_LIT>'.join(map(str_cls, _pep425_version())), suffix)<EOL>

:return:
    A unicode string of the system abi. Will be something like: "cp27m",
    "cp33m", etc.


tags = []<EOL>versions = []<EOL>version_info = _pep425_version()<EOL>major = version_info[:-<NUM_LIT:1>]<EOL>for minor in range(version_info[-<NUM_LIT:1>], -<NUM_LIT:1>, -<NUM_LIT:1>):<EOL><INDENT>versions.append('<STR_LIT>'.join(map(str, major + (minor,))))<EOL><DEDENT>impl = _pep425_implementation()<EOL>abis = []<EOL>abi = _pep425_get_abi()<EOL>if abi:<EOL><INDENT>abis.append(abi)<EOL><DEDENT>abi3 = _pep425_implementation() == '<STR_LIT>' and sys.version_info >= (<NUM_LIT:3>,)<EOL>if abi3:<EOL><INDENT>abis.append('<STR_LIT>')<EOL><DEDENT>abis.append('<STR_LIT:none>')<EOL>if sys.platform == '<STR_LIT>':<EOL><INDENT>plat_ver = platform.mac_ver()<EOL>ver_parts = plat_ver[<NUM_LIT:0>].split('<STR_LIT:.>')<EOL>minor = int(ver_parts[<NUM_LIT:1>])<EOL>arch = plat_ver[<NUM_LIT:2>]<EOL>if sys.maxsize == <NUM_LIT>:<EOL><INDENT>arch = '<STR_LIT>'<EOL><DEDENT>arches = []<EOL>while minor > <NUM_LIT:5>:<EOL><INDENT>arches.append('<STR_LIT>' % (minor, arch))<EOL>arches.append('<STR_LIT>' % (minor,))<EOL>arches.append('<STR_LIT>' % (minor,))<EOL>minor -= <NUM_LIT:1><EOL><DEDENT><DEDENT>else:<EOL><INDENT>if sys.platform == '<STR_LIT:win32>':<EOL><INDENT>if '<STR_LIT>' in sys.version.lower():<EOL><INDENT>arches = ['<STR_LIT>']<EOL><DEDENT>arches = [sys.platform]<EOL><DEDENT>elif hasattr(os, '<STR_LIT>'):<EOL><INDENT>(plat, _, _, _, machine) = os.uname()<EOL>plat = plat.lower().replace('<STR_LIT:/>', '<STR_LIT>')<EOL>machine.replace('<STR_LIT:U+0020>', '<STR_LIT:_>').replace('<STR_LIT:/>', '<STR_LIT:_>')<EOL>if plat == '<STR_LIT>' and sys.maxsize == <NUM_LIT>:<EOL><INDENT>machine = '<STR_LIT>'<EOL><DEDENT>arch = '<STR_LIT>' % (plat, machine)<EOL>if _pep425_supports_manylinux():<EOL><INDENT>arches = [arch.replace('<STR_LIT>', '<STR_LIT>'), arch]<EOL><DEDENT>else:<EOL><INDENT>arches = [arch]<EOL><DEDENT><DEDENT><DEDENT>for abi in abis:<EOL><INDENT>for arch in arches:<EOL><INDENT>tags.append(('<STR_LIT>' % (impl, versions[<NUM_LIT:0>]), abi, arch))<EOL><DEDENT><DEDENT>if abi3:<EOL><INDENT>for version in versions[<NUM_LIT:1>:]:<EOL><INDENT>for arch in arches:<EOL><INDENT>tags.append(('<STR_LIT>' % (impl, version), '<STR_LIT>', arch))<EOL><DEDENT><DEDENT><DEDENT>for arch in arches:<EOL><INDENT>tags.append(('<STR_LIT>' % (versions[<NUM_LIT:0>][<NUM_LIT:0>]), '<STR_LIT:none>', arch))<EOL><DEDENT>tags.append(('<STR_LIT>' % (impl, versions[<NUM_LIT:0>]), '<STR_LIT:none>', '<STR_LIT>'))<EOL>tags.append(('<STR_LIT>' % (impl, versions[<NUM_LIT:0>][<NUM_LIT:0>]), '<STR_LIT:none>', '<STR_LIT>'))<EOL>for i, version in enumerate(versions):<EOL><INDENT>tags.append(('<STR_LIT>' % (version,), '<STR_LIT:none>', '<STR_LIT>'))<EOL>if i == <NUM_LIT:0>:<EOL><INDENT>tags.append(('<STR_LIT>' % (version[<NUM_LIT:0>]), '<STR_LIT:none>', '<STR_LIT>'))<EOL><DEDENT><DEDENT>tags.append(('<STR_LIT>', '<STR_LIT:none>', '<STR_LIT>'))<EOL>return tags<EOL>

:return:
    A list of 3-element tuples with unicode strings or None:
     [0] implementation tag - cp33, pp27, cp26, py2, py2.py3
     [1] abi tag - cp26m, None
     [2] arch tag - linux_x86_64, macosx_10_10_x85_64, etc


print('<STR_LIT>' % flake8.__version__)<EOL>flake8_style = get_style_guide(config_file=os.path.join(package_root, '<STR_LIT>'))<EOL>paths = []<EOL>for _dir in [package_name, '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>for root, _, filenames in os.walk(_dir):<EOL><INDENT>for filename in filenames:<EOL><INDENT>if not filename.endswith('<STR_LIT>'):<EOL><INDENT>continue<EOL><DEDENT>paths.append(os.path.join(root, filename))<EOL><DEDENT><DEDENT><DEDENT>report = flake8_style.check_files(paths)<EOL>success = report.total_errors == <NUM_LIT:0><EOL>if success:<EOL><INDENT>print('<STR_LIT:OK>')<EOL><DEDENT>return success<EOL>

Runs flake8 lint

:return:
    A bool - if flake8 did not find any errors


xml_report_path = os.path.join(package_root, '<STR_LIT>')<EOL>if os.path.exists(xml_report_path):<EOL><INDENT>os.unlink(xml_report_path)<EOL><DEDENT>cov = coverage.Coverage(include='<STR_LIT>' % package_name)<EOL>cov.start()<EOL>from .tests import run as run_tests<EOL>result = run_tests()<EOL>print()<EOL>if ci:<EOL><INDENT>suite = unittest.TestSuite()<EOL>loader = unittest.TestLoader()<EOL>for other_package in other_packages:<EOL><INDENT>for test_class in _load_package_tests(other_package):<EOL><INDENT>suite.addTest(loader.loadTestsFromTestCase(test_class))<EOL><DEDENT><DEDENT>if suite.countTestCases() > <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>')<EOL>sys.stdout.flush()<EOL>runner_result = unittest.TextTestRunner(stream=sys.stdout, verbosity=<NUM_LIT:1>).run(suite)<EOL>result = runner_result.wasSuccessful() and result<EOL>print()<EOL>sys.stdout.flush()<EOL><DEDENT><DEDENT>cov.stop()<EOL>cov.save()<EOL>cov.report(show_missing=False)<EOL>print()<EOL>sys.stdout.flush()<EOL>if ci:<EOL><INDENT>cov.xml_report()<EOL><DEDENT>if ci and result and os.path.exists(xml_report_path):<EOL><INDENT>_codecov_submit()<EOL>print()<EOL><DEDENT>return result<EOL>

Runs the tests while measuring coverage

:param ci:
    If coverage is being run in a CI environment - this triggers trying to
    run the tests for the rest of modularcrypto and uploading coverage data

:return:
    A bool - if the tests ran successfully


proc = subprocess.Popen(<EOL>['<STR_LIT>'] + params,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.STDOUT,<EOL>cwd=cwd<EOL>)<EOL>stdout, stderr = proc.communicate()<EOL>code = proc.wait()<EOL>if code != <NUM_LIT:0>:<EOL><INDENT>e = OSError('<STR_LIT>')<EOL>e.stdout = stdout<EOL>raise e<EOL><DEDENT>return stdout.decode('<STR_LIT:utf-8>').strip()<EOL>

Executes a git command, returning the output

:param params:
    A list of the parameters to pass to git

:param cwd:
    The working directory to execute git in

:return:
    A 2-element tuple of (stdout, stderr)


output = {}<EOL>for line in data.splitlines():<EOL><INDENT>line = line.strip()<EOL>if not line or '<STR_LIT:=>' not in line:<EOL><INDENT>continue<EOL><DEDENT>parts = line.split('<STR_LIT:=>')<EOL>if len(parts) != <NUM_LIT:2>:<EOL><INDENT>continue<EOL><DEDENT>name = parts[<NUM_LIT:0>]<EOL>value = parts[<NUM_LIT:1>]<EOL>if len(value) > <NUM_LIT:1>:<EOL><INDENT>if value[<NUM_LIT:0>] == '<STR_LIT:">' and value[-<NUM_LIT:1>] == '<STR_LIT:">':<EOL><INDENT>value = value[<NUM_LIT:1>:-<NUM_LIT:1>]<EOL><DEDENT><DEDENT>output[name] = value<EOL><DEDENT>return output<EOL>

Parses a basic VAR="value data" file contents into a dict

:param data:
    A unicode string of the file data

:return:
    A dict of parsed name/value data


if sys.platform == '<STR_LIT>':<EOL><INDENT>version = _plat.mac_ver()[<NUM_LIT:0>]<EOL>_plat_ver_info = tuple(map(int, version.split('<STR_LIT:.>')))<EOL>if _plat_ver_info < (<NUM_LIT:10>, <NUM_LIT:12>):<EOL><INDENT>name = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>name = '<STR_LIT>'<EOL><DEDENT>return '<STR_LIT>' % (name, version)<EOL><DEDENT>elif sys.platform == '<STR_LIT:win32>':<EOL><INDENT>_win_ver = sys.getwindowsversion()<EOL>_plat_ver_info = (_win_ver[<NUM_LIT:0>], _win_ver[<NUM_LIT:1>])<EOL>return '<STR_LIT>' % _plat.win32_ver()[<NUM_LIT:0>]<EOL><DEDENT>elif sys.platform in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>if os.path.exists('<STR_LIT>'):<EOL><INDENT>with open('<STR_LIT>', '<STR_LIT:r>', encoding='<STR_LIT:utf-8>') as f:<EOL><INDENT>pairs = _parse_env_var_file(f.read())<EOL>if '<STR_LIT>' in pairs and '<STR_LIT>' in pairs:<EOL><INDENT>return '<STR_LIT>' % (pairs['<STR_LIT>'], pairs['<STR_LIT>'])<EOL>version = pairs['<STR_LIT>']<EOL><DEDENT>elif '<STR_LIT>' in pairs:<EOL><INDENT>return pairs['<STR_LIT>']<EOL><DEDENT>elif '<STR_LIT>' in pairs:<EOL><INDENT>return pairs['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>elif os.path.exists('<STR_LIT>'):<EOL><INDENT>with open('<STR_LIT>', '<STR_LIT:r>', encoding='<STR_LIT:utf-8>') as f:<EOL><INDENT>pairs = _parse_env_var_file(f.read())<EOL>if '<STR_LIT>' in pairs:<EOL><INDENT>return pairs['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return '<STR_LIT>' % (_plat.system(), _plat.release())<EOL><DEDENT>

Returns information about the current operating system and version

:return:
    A unicode string containing the OS name and version


dir_patterns, file_patterns = _gitignore(root)<EOL>paths = []<EOL>prefix = os.path.abspath(root) + os.sep<EOL>for base, dirs, files in os.walk(root):<EOL><INDENT>for d in dirs:<EOL><INDENT>for dir_pattern in dir_patterns:<EOL><INDENT>if fnmatch(d, dir_pattern):<EOL><INDENT>dirs.remove(d)<EOL>break<EOL><DEDENT><DEDENT><DEDENT>for f in files:<EOL><INDENT>skip = False<EOL>for file_pattern in file_patterns:<EOL><INDENT>if fnmatch(f, file_pattern):<EOL><INDENT>skip = True<EOL>break<EOL><DEDENT><DEDENT>if skip:<EOL><INDENT>continue<EOL><DEDENT>full_path = os.path.join(base, f)<EOL>if full_path[:len(prefix)] == prefix:<EOL><INDENT>full_path = full_path[len(prefix):]<EOL><DEDENT>paths.append(full_path)<EOL><DEDENT><DEDENT>return sorted(paths)<EOL>

Lists all of the files in a directory, taking into account any .gitignore
file that is present

:param root:
    A unicode filesystem path

:return:
    A list of unicode strings, containing paths of all files not ignored
    by .gitignore with root, using relative paths


gitignore_path = os.path.join(root, '<STR_LIT>')<EOL>dir_patterns = ['<STR_LIT>']<EOL>file_patterns = []<EOL>if not os.path.exists(gitignore_path):<EOL><INDENT>return (dir_patterns, file_patterns)<EOL><DEDENT>with open(gitignore_path, '<STR_LIT:r>', encoding='<STR_LIT:utf-8>') as f:<EOL><INDENT>for line in f.readlines():<EOL><INDENT>line = line.strip()<EOL>if not line:<EOL><INDENT>continue<EOL><DEDENT>if line.startswith('<STR_LIT:#>'):<EOL><INDENT>continue<EOL><DEDENT>if '<STR_LIT>' in line:<EOL><INDENT>raise NotImplementedError('<STR_LIT>')<EOL><DEDENT>if line.startswith('<STR_LIT:!>'):<EOL><INDENT>raise NotImplementedError('<STR_LIT>')<EOL><DEDENT>if line.startswith('<STR_LIT:/>'):<EOL><INDENT>raise NotImplementedError('<STR_LIT>')<EOL><DEDENT>if line.startswith('<STR_LIT>'):<EOL><INDENT>line = '<STR_LIT:#>' + line[<NUM_LIT:2>:]<EOL><DEDENT>if line.startswith('<STR_LIT>'):<EOL><INDENT>line = '<STR_LIT:!>' + line[<NUM_LIT:2>:]<EOL><DEDENT>if line.endswith('<STR_LIT:/>'):<EOL><INDENT>dir_patterns.append(line[:-<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>file_patterns.append(line)<EOL><DEDENT><DEDENT><DEDENT>return (dir_patterns, file_patterns)<EOL>

Parses a .gitignore file and returns patterns to match dirs and files.
Only basic gitignore patterns are supported. Pattern negation, ** wildcards
and anchored patterns are not currently implemented.

:param root:
    A unicode string of the path to the git repository

:return:
    A 2-element tuple:
     - 0: a list of unicode strings to match against dirs
     - 1: a list of unicode strings to match against dirs and files


def _do_request(method, url, headers, data=None, query_params=None, timeout=<NUM_LIT:20>):

if query_params:<EOL><INDENT>url += '<STR_LIT:?>' + urlencode(query_params).replace('<STR_LIT:+>', '<STR_LIT>')<EOL><DEDENT>if isinstance(data, dict):<EOL><INDENT>data_bytes = {}<EOL>for key in data:<EOL><INDENT>data_bytes[key.encode('<STR_LIT:utf-8>')] = data[key].encode('<STR_LIT:utf-8>')<EOL><DEDENT>data = urlencode(data_bytes)<EOL>headers['<STR_LIT:Content-Type>'] = '<STR_LIT>'<EOL><DEDENT>if isinstance(data, str_cls):<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>try:<EOL><INDENT>tempfd, tempf_path = tempfile.mkstemp('<STR_LIT>')<EOL>os.write(tempfd, data or b'<STR_LIT>')<EOL>os.close(tempfd)<EOL>if sys.platform == '<STR_LIT:win32>':<EOL><INDENT>powershell_exe = os.path.join('<STR_LIT>')<EOL>code = "<STR_LIT>"<EOL>code += "<STR_LIT>"<EOL>for key in headers:<EOL><INDENT>code += "<STR_LIT>" % (key, headers[key])<EOL><DEDENT>code += "<STR_LIT>" % (url, method, tempf_path)<EOL>code += "<STR_LIT>"<EOL>code += "<STR_LIT>"<EOL>stdout, stderr = _execute([powershell_exe, '<STR_LIT>', code], os.getcwd())<EOL>if stdout[-<NUM_LIT:2>:] == b'<STR_LIT:\r\n>' and b'<STR_LIT>' in stdout:<EOL><INDENT>stdout = stdout[<NUM_LIT:0>:-<NUM_LIT:2>]<EOL>parts = stdout.split(b'<STR_LIT>', <NUM_LIT:1>)<EOL>if len(parts) == <NUM_LIT:2>:<EOL><INDENT>stdout = parts[<NUM_LIT:0>] + b'<STR_LIT>' + codecs.decode(parts[<NUM_LIT:1>].replace(b'<STR_LIT:->', b'<STR_LIT>'), '<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>args = [<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>method,<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>'<EOL>]<EOL>for key in headers:<EOL><INDENT>args.append('<STR_LIT>')<EOL>args.append("<STR_LIT>" % (key, headers[key]))<EOL><DEDENT>args.append('<STR_LIT>')<EOL>args.append('<STR_LIT>' % tempf_path)<EOL>args.append(url)<EOL>stdout, stderr = _execute(args, os.getcwd())<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>if tempf_path and os.path.exists(tempf_path):<EOL><INDENT>os.remove(tempf_path)<EOL><DEDENT><DEDENT>if len(stderr) > <NUM_LIT:0>:<EOL><INDENT>raise URLError("<STR_LIT>" % (method, url, stderr))<EOL><DEDENT>parts = stdout.split(b'<STR_LIT>', <NUM_LIT:1>)<EOL>if len(parts) != <NUM_LIT:2>:<EOL><INDENT>raise URLError("<STR_LIT>" % (method, url, stdout))<EOL><DEDENT>header_block, body = parts<EOL>content_type_header = None<EOL>content_len_header = None<EOL>for hline in header_block.decode('<STR_LIT>').splitlines():<EOL><INDENT>hline_parts = hline.split('<STR_LIT::>', <NUM_LIT:1>)<EOL>if len(hline_parts) != <NUM_LIT:2>:<EOL><INDENT>continue<EOL><DEDENT>name, val = hline_parts<EOL>name = name.strip().lower()<EOL>val = val.strip()<EOL>if name == '<STR_LIT>':<EOL><INDENT>content_type_header = val<EOL><DEDENT>if name == '<STR_LIT>':<EOL><INDENT>content_len_header = val<EOL><DEDENT><DEDENT>if content_type_header is None and content_len_header != '<STR_LIT:0>':<EOL><INDENT>raise URLError("<STR_LIT>" % (method, url, stdout))<EOL><DEDENT>if content_type_header is None:<EOL><INDENT>content_type = '<STR_LIT>'<EOL>encoding = '<STR_LIT:utf-8>'<EOL><DEDENT>else:<EOL><INDENT>content_type, params = cgi.parse_header(content_type_header)<EOL>encoding = params.get('<STR_LIT>')<EOL><DEDENT>return (content_type, encoding, body)<EOL>

Performs an HTTP request

:param method:
    A unicode string of 'POST' or 'PUT'

:param url;
    A unicode string of the URL to request

:param headers:
    A dict of unicode strings, where keys are header names and values are
    the header values.

:param data:
    A dict of unicode strings (to be encoded as
    application/x-www-form-urlencoded), or a byte string of data.

:param query_params:
    A dict of unicode keys and values to pass as query params

:param timeout:
    An integer number of seconds to use as the timeout

:return:
    A 3-element tuple:
     - 0: A unicode string of the response content-type
     - 1: A unicode string of the response encoding, or None
     - 2: A byte string of the response body


proc = subprocess.Popen(<EOL>params,<EOL>stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE,<EOL>cwd=cwd<EOL>)<EOL>stdout, stderr = proc.communicate()<EOL>code = proc.wait()<EOL>if code != <NUM_LIT:0>:<EOL><INDENT>e = OSError('<STR_LIT>' % (params, code, stderr))<EOL>e.stdout = stdout<EOL>e.stderr = stderr<EOL>raise e<EOL><DEDENT>return (stdout, stderr)<EOL>

Executes a subprocess

:param params:
    A list of the executable and arguments to pass to it

:param cwd:
    The working directory to execute the command in

:return:
    A 2-element tuple of (stdout, stderr)


print('<STR_LIT>' + sys.version.replace('<STR_LIT:\n>', '<STR_LIT>'))<EOL>try:<EOL><INDENT>oscrypto_tests_module_info = imp.find_module('<STR_LIT>', [os.path.join(build_root, '<STR_LIT>')])<EOL>oscrypto_tests = imp.load_module('<STR_LIT>', *oscrypto_tests_module_info)<EOL>oscrypto = oscrypto_tests.local_oscrypto()<EOL>print('<STR_LIT>' % oscrypto.backend())<EOL><DEDENT>except (ImportError):<EOL><INDENT>pass<EOL><DEDENT>if run_lint:<EOL><INDENT>print('<STR_LIT>')<EOL>lint_result = run_lint()<EOL><DEDENT>else:<EOL><INDENT>lint_result = True<EOL><DEDENT>if run_coverage:<EOL><INDENT>print('<STR_LIT>')<EOL>sys.stdout.flush()<EOL>tests_result = run_coverage(ci=True)<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>')<EOL>sys.stdout.flush()<EOL>tests_result = run_tests()<EOL><DEDENT>sys.stdout.flush()<EOL>return lint_result and tests_result<EOL>

Runs the linter and tests

:return:
    A bool - if the linter and tests ran successfully


deps_dir = os.path.join(build_root, '<STR_LIT>')<EOL>if os.path.exists(deps_dir):<EOL><INDENT>shutil.rmtree(deps_dir, ignore_errors=True)<EOL><DEDENT>os.mkdir(deps_dir)<EOL>try:<EOL><INDENT>print("<STR_LIT>")<EOL>_stage_requirements(deps_dir, os.path.join(package_root, '<STR_LIT>', '<STR_LIT>'))<EOL>print("<STR_LIT>")<EOL>for other_package in other_packages:<EOL><INDENT>pkg_url = '<STR_LIT>' % other_package<EOL>pkg_dir = os.path.join(build_root, other_package)<EOL>if os.path.exists(pkg_dir):<EOL><INDENT>print("<STR_LIT>" % other_package)<EOL>continue<EOL><DEDENT>print("<STR_LIT>" % pkg_url)<EOL>_execute(['<STR_LIT>', '<STR_LIT>', pkg_url], build_root)<EOL><DEDENT>print()<EOL><DEDENT>except (Exception):<EOL><INDENT>if os.path.exists(deps_dir):<EOL><INDENT>shutil.rmtree(deps_dir, ignore_errors=True)<EOL><DEDENT>raise<EOL><DEDENT>return True<EOL>

Installs required development dependencies. Uses git to checkout other
modularcrypto repos for more accurate coverage data.


print('<STR_LIT>' % url)<EOL>filename = os.path.basename(url)<EOL>dest_path = os.path.join(dest, filename)<EOL>if sys.platform == '<STR_LIT:win32>':<EOL><INDENT>powershell_exe = os.path.join('<STR_LIT>')<EOL>code = "<STR_LIT>"<EOL>code += "<STR_LIT>" % (url, dest_path)<EOL>_execute([powershell_exe, '<STR_LIT>', code], dest)<EOL><DEDENT>else:<EOL><INDENT>_execute(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', url], dest)<EOL><DEDENT>return dest_path<EOL>

Downloads a URL to a directory

:param url:
    The URL to download

:param dest:
    The path to the directory to save the file in

:return:
    The filesystem path to the saved file


return tuple(map(int, version_string.split('<STR_LIT:.>')))<EOL>

:param version_string:
    A unicode dotted version string

:return:
    A tuple of integers


if path.endswith('<STR_LIT>'):<EOL><INDENT>return zipfile.ZipFile(path, '<STR_LIT:r>')<EOL><DEDENT>return tarfile.open(path, '<STR_LIT:r>')<EOL>

:param path:
    A unicode string of the filesystem path to the archive

:return:
    An archive object


if isinstance(archive, zipfile.ZipFile):<EOL><INDENT>return archive.infolist()<EOL><DEDENT>return archive.getmembers()<EOL>

:param archive:
    An archive from _open_archive()

:return:
    A list of info objects to be used with _info_name() and _extract_info()


common_root = None<EOL>for info in _list_archive_members(archive):<EOL><INDENT>fn = _info_name(info)<EOL>if fn in set(['<STR_LIT:.>', '<STR_LIT:/>']):<EOL><INDENT>continue<EOL><DEDENT>sep = None<EOL>if '<STR_LIT:/>' in fn:<EOL><INDENT>sep = '<STR_LIT:/>'<EOL><DEDENT>elif '<STR_LIT:\\>' in fn:<EOL><INDENT>sep = '<STR_LIT:\\>'<EOL><DEDENT>if sep is None:<EOL><INDENT>root_dir = fn<EOL><DEDENT>else:<EOL><INDENT>root_dir, _ = fn.split(sep, <NUM_LIT:1>)<EOL><DEDENT>if common_root is None:<EOL><INDENT>common_root = root_dir<EOL><DEDENT>else:<EOL><INDENT>if common_root != root_dir:<EOL><INDENT>return None<EOL><DEDENT><DEDENT><DEDENT>return common_root<EOL>

Check if all members of the archive are in a single top-level directory

:param archive:
    An archive from _open_archive()

:return:
    None if not a single top level directory in archive, otherwise a
    unicode string of the top level directory name


if isinstance(info, zipfile.ZipInfo):<EOL><INDENT>return info.filename.replace('<STR_LIT:\\>', '<STR_LIT:/>')<EOL><DEDENT>return info.name.replace('<STR_LIT:\\>', '<STR_LIT:/>')<EOL>

Returns a normalized file path for an archive info object

:param info:
    An info object from _list_archive_members()

:return:
    A unicode string with all directory separators normalized to "/"


if isinstance(archive, zipfile.ZipFile):<EOL><INDENT>fn = info.filename<EOL>is_dir = fn.endswith('<STR_LIT:/>') or fn.endswith('<STR_LIT:\\>')<EOL>out = archive.read(info)<EOL>if is_dir and out == b'<STR_LIT>':<EOL><INDENT>return None<EOL><DEDENT>return out<EOL><DEDENT>info_file = archive.extractfile(info)<EOL>if info_file:<EOL><INDENT>return info_file.read()<EOL><DEDENT>return None<EOL>

Extracts the contents of an archive info object

;param archive:
    An archive from _open_archive()

:param info:
    An info object from _list_archive_members()

:return:
    None, or a byte string of the file contents


if pkg_path.endswith('<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>zf = None<EOL>zf = zipfile.ZipFile(pkg_path, '<STR_LIT:r>')<EOL>for zi in zf.infolist():<EOL><INDENT>if not zi.filename.startswith('<STR_LIT>'):<EOL><INDENT>continue<EOL><DEDENT>data = _extract_info(zf, zi)<EOL>if data is not None:<EOL><INDENT>dst_path = os.path.join(deps_dir, zi.filename[<NUM_LIT:8>:])<EOL>dst_dir = os.path.dirname(dst_path)<EOL>if not os.path.exists(dst_dir):<EOL><INDENT>os.makedirs(dst_dir)<EOL><DEDENT>with open(dst_path, '<STR_LIT:wb>') as f:<EOL><INDENT>f.write(data)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>finally:<EOL><INDENT>if zf:<EOL><INDENT>zf.close()<EOL><DEDENT><DEDENT>return<EOL><DEDENT>if pkg_path.endswith('<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>zf = None<EOL>zf = zipfile.ZipFile(pkg_path, '<STR_LIT:r>')<EOL>zf.extractall(deps_dir)<EOL><DEDENT>finally:<EOL><INDENT>if zf:<EOL><INDENT>zf.close()<EOL><DEDENT><DEDENT>return<EOL><DEDENT>try:<EOL><INDENT>ar = None<EOL>ar = _open_archive(pkg_path)<EOL>pkg_name = None<EOL>base_path = _archive_single_dir(ar) or '<STR_LIT>'<EOL>if len(base_path):<EOL><INDENT>if '<STR_LIT:->' in base_path:<EOL><INDENT>pkg_name, _ = base_path.split('<STR_LIT:->', <NUM_LIT:1>)<EOL><DEDENT>base_path += '<STR_LIT:/>'<EOL><DEDENT>base_pkg_path = None<EOL>if pkg_name is not None:<EOL><INDENT>base_pkg_path = base_path + pkg_name + '<STR_LIT:/>'<EOL><DEDENT>src_path = base_path + '<STR_LIT>'<EOL>members = []<EOL>for info in _list_archive_members(ar):<EOL><INDENT>fn = _info_name(info)<EOL>if base_pkg_path is not None and fn.startswith(base_pkg_path):<EOL><INDENT>dst_path = fn[len(base_pkg_path) - len(pkg_name) - <NUM_LIT:1>:]<EOL>members.append((info, dst_path))<EOL>continue<EOL><DEDENT>if fn.startswith(src_path):<EOL><INDENT>members.append((info, fn[len(src_path):]))<EOL>continue<EOL><DEDENT><DEDENT>for info, path in members:<EOL><INDENT>info_data = _extract_info(ar, info)<EOL>if info_data is not None:<EOL><INDENT>dst_path = os.path.join(deps_dir, path)<EOL>dst_dir = os.path.dirname(dst_path)<EOL>if not os.path.exists(dst_dir):<EOL><INDENT>os.makedirs(dst_dir)<EOL><DEDENT>with open(dst_path, '<STR_LIT:wb>') as f:<EOL><INDENT>f.write(info_data)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>finally:<EOL><INDENT>if ar:<EOL><INDENT>ar.close()<EOL><DEDENT><DEDENT>

Extract a .whl, .zip, .tar.gz or .tar.bz2 into a package path to
use when running CI tasks

:param deps_dir:
    A unicode string of the directory the package should be extracted to

:param pkg_path:
    A unicode string of the path to the archive


valid_tags = _pep425tags()<EOL>exe_suffix = None<EOL>if sys.platform == '<STR_LIT:win32>' and _pep425_implementation() == '<STR_LIT>':<EOL><INDENT>win_arch = '<STR_LIT:win32>' if sys.maxsize == <NUM_LIT> else '<STR_LIT>'<EOL>version_info = sys.version_info<EOL>exe_suffix = '<STR_LIT>' % (win_arch, version_info[<NUM_LIT:0>], version_info[<NUM_LIT:1>])<EOL><DEDENT>packages = _parse_requires(path)<EOL>for p in packages:<EOL><INDENT>pkg = p['<STR_LIT>']<EOL>if p['<STR_LIT:type>'] == '<STR_LIT:url>':<EOL><INDENT>if pkg.endswith('<STR_LIT>') or pkg.endswith('<STR_LIT>') or pkg.endswith('<STR_LIT>') or pkg.endswith('<STR_LIT>'):<EOL><INDENT>url = pkg<EOL><DEDENT>else:<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>pypi_json_url = '<STR_LIT>' % pkg<EOL>json_dest = _download(pypi_json_url, deps_dir)<EOL>with open(json_dest, '<STR_LIT:rb>') as f:<EOL><INDENT>pkg_info = json.loads(f.read().decode('<STR_LIT:utf-8>'))<EOL><DEDENT>if os.path.exists(json_dest):<EOL><INDENT>os.remove(json_dest)<EOL><DEDENT>latest = pkg_info['<STR_LIT:info>']['<STR_LIT:version>']<EOL>if p['<STR_LIT:type>'] == '<STR_LIT>':<EOL><INDENT>if _tuple_from_ver(p['<STR_LIT>']) > _tuple_from_ver(latest):<EOL><INDENT>raise Exception('<STR_LIT>' % (p['<STR_LIT>'], pkg, latest))<EOL><DEDENT>version = latest<EOL><DEDENT>elif p['<STR_LIT:type>'] == '<STR_LIT>':<EOL><INDENT>if p['<STR_LIT>'] not in pkg_info['<STR_LIT>']:<EOL><INDENT>raise Exception('<STR_LIT>' % (p['<STR_LIT>'], pkg))<EOL><DEDENT>version = p['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>version = latest<EOL><DEDENT>wheels = {}<EOL>whl = None<EOL>tar_bz2 = None<EOL>tar_gz = None<EOL>exe = None<EOL>for download in pkg_info['<STR_LIT>'][version]:<EOL><INDENT>if exe_suffix and download['<STR_LIT:url>'].endswith(exe_suffix):<EOL><INDENT>exe = download['<STR_LIT:url>']<EOL><DEDENT>if download['<STR_LIT:url>'].endswith('<STR_LIT>'):<EOL><INDENT>parts = os.path.basename(download['<STR_LIT:url>']).split('<STR_LIT:->')<EOL>tag_impl = parts[-<NUM_LIT:3>]<EOL>tag_abi = parts[-<NUM_LIT:2>]<EOL>tag_arch = parts[-<NUM_LIT:1>].split('<STR_LIT:.>')[<NUM_LIT:0>]<EOL>wheels[(tag_impl, tag_abi, tag_arch)] = download['<STR_LIT:url>']<EOL><DEDENT>if download['<STR_LIT:url>'].endswith('<STR_LIT>'):<EOL><INDENT>tar_bz2 = download['<STR_LIT:url>']<EOL><DEDENT>if download['<STR_LIT:url>'].endswith('<STR_LIT>'):<EOL><INDENT>tar_gz = download['<STR_LIT:url>']<EOL><DEDENT><DEDENT>for tag in valid_tags:<EOL><INDENT>if tag in wheels:<EOL><INDENT>whl = wheels[tag]<EOL>break<EOL><DEDENT><DEDENT>if exe_suffix and exe:<EOL><INDENT>url = exe<EOL><DEDENT>elif whl:<EOL><INDENT>url = whl<EOL><DEDENT>elif tar_bz2:<EOL><INDENT>url = tar_bz2<EOL><DEDENT>elif tar_gz:<EOL><INDENT>url = tar_gz<EOL><DEDENT>else:<EOL><INDENT>raise Exception('<STR_LIT>' % pkg)<EOL><DEDENT><DEDENT>local_path = _download(url, deps_dir)<EOL>_extract_package(deps_dir, local_path)<EOL>os.remove(local_path)<EOL><DEDENT>

Installs requirements without using Python to download, since
different services are limiting to TLS 1.2, and older version of
Python do not support that

:param deps_dir:
    A unicode path to a temporary diretory to use for downloads

:param path:
    A unicoe filesystem path to a requirements file