ragout/scaffolder/output_generator.py
class OutputGenerator:
    def __init__(self, fragments_fasta, scaffolds):
        self.fragments_fasta = fragments_fasta
        self.scaffolds = scaffolds

        self.unplaced_fasta = None
        self.scaffolds_fasta = None
        self.used_fragments_len = None
        self.introduced_gap_len = None

    def make_output(self, out_dir, out_prefix):
        """
        Makes full output to the given directory
        """
        out_links = os.path.join(out_dir, out_prefix + "_scaffolds.links")
        out_agp = os.path.join(out_dir, out_prefix + "_scaffolds.agp")
        out_chr = os.path.join(out_dir, out_prefix + "_scaffolds.fasta")
        out_unplaced = os.path.join(out_dir, out_prefix + "_unplaced.fasta")

        self._make_unplaced_fasta()
        write_fasta_dict(self.unplaced_fasta, out_unplaced)

        self._fix_gaps()
        self._make_scaffolds_fasta()
        write_fasta_dict(self.scaffolds_fasta, out_chr)

        self._output_agp(out_agp, out_prefix)
        self._print_statistics()
        output_links(self.scaffolds, out_links)

    def _fix_gaps(self):
        """
        Handles negative gaps, ensures that gap values are
        within deined range
        """
        def get_seq(contig):
            seq_name, seg_start, seg_end = contig.name_with_coords()
            cont_seq = self.fragments_fasta[seq_name][seg_start:seg_end]
            if contig.sign < 0:
                cont_seq = reverse_complement(cont_seq)
            return cont_seq

        def count_ns(cnt_1, cnt_2):
            seq_1, seq_2 = get_seq(cnt_1), get_seq(cnt_2)

            left_ns, right_ns = 0, 0
            for i in range(len(seq_1) - 1, 0, -1):
                if seq_1[i].upper() != "N":
                    break
                left_ns += 1
            for i in range(len(seq_2) - 1):
                if seq_2[i].upper() != "N":
                    break
                right_ns += 1
            return left_ns, right_ns

        for scf in self.scaffolds:
            for cnt_1, cnt_2 in zip(scf.contigs[:-1], scf.contigs[1:]):
                if cnt_1.link.supporting_assembly:
                    cnt_1.trim_right(max(0, -cnt_1.link.gap))
                    cnt_1.link.gap = max(0, cnt_1.link.gap)
                    continue

                left_ns, right_ns = count_ns(cnt_1, cnt_2)

                cnt_1.trim_right(left_ns)
                cnt_2.trim_left(right_ns)
                cnt_1.link.gap += left_ns + right_ns
                cnt_1.link.gap = max(cnt_1.link.gap,
                                     config.vals["min_scaffold_gap"])
                #cnt_1.link.gap = min(cnt_1.link.gap,
                #                     config.vals["max_scaffold_gap"])

    def _output_agp(self, out_agp, assembly_name):
        """
        Output file in NCBI AGP format
        """
        SHIFT = 1
        with open(out_agp, "w") as f:
            f.write("##agp-version  2.0\n")
            f.write("#ASSEMBLY NAME: {0}\n".format(assembly_name))
            f.write("#DESCRIPTION: Pseudochromosome assembly\n")
            f.write("#PROGRAM: Ragout v{0}\n".format(__version__))
            for scf in sorted(self.scaffolds, key=lambda s: s.name):
                chr_pos = 0
                for contig_id, contig in enumerate(scf.contigs):
                    chr_start = chr_pos
                    chr_end = chr_pos + contig.length()
                    chr_pos = chr_end + contig.link.gap
                    cont_name, cont_start, cont_end = contig.name_with_coords()
                    strand = "+" if contig.sign > 0 else "-"
                    support = _support_to_string(contig.link)

                    contig_num = 2 * contig_id + 1
                    gap_num = 2 * contig_id + 2
                    cont_fields = [scf.name, chr_start + SHIFT, chr_end,
                                   contig_num, "W", cont_name, cont_start + SHIFT,
                                   cont_end, strand]
                    f.write("\t".join(map(str, cont_fields)) + "\n")
                    if contig.link.gap > 0:
                        gap_fields = [scf.name, chr_end + SHIFT, chr_pos, gap_num,
                                      "N", contig.link.gap,
                                      "scaffold", "yes", support]
                        f.write("\t".join(map(str, gap_fields)) + "\n")

    def _make_unplaced_fasta(self):
        """
        Creates unplaced (not used in scaffolds) sequences in FASTA format
        """
        used_ranges_by_seq = defaultdict(list)
        for scf in self.scaffolds:
            for ctg in scf.contigs:
                seq_name, seq_start, seq_end = ctg.name_with_coords()
                used_ranges_by_seq[seq_name].append((seq_start, seq_end))
        for seq_name in self.fragments_fasta:
            seq_len = len(self.fragments_fasta[seq_name])
            used_ranges_by_seq[seq_name].append((0, 0))
            used_ranges_by_seq[seq_name].append((seq_len, seq_len))
            used_ranges_by_seq[seq_name].sort()

        unused_ranges_by_seq = defaultdict(list)
        for seq_name in self.fragments_fasta:
            for range_1, range_2 in zip(used_ranges_by_seq[seq_name][:-1],
                                        used_ranges_by_seq[seq_name][1:]):
                if range_1[1] < range_2[0]:
                    unused_ranges_by_seq[seq_name].append((range_1[1],
                                                           range_2[0]))

        unplaced_fasta = {}
        for seq_name, unused_ranges in unused_ranges_by_seq.items():
            for ur in unused_ranges:
                if ur[0] == 0 and ur[1] == len(self.fragments_fasta[seq_name]):
                    fragment_name = seq_name
                else:
                    fragment_name = seq_name + "[{0}:{1}]".format(ur[0], ur[1])
                unplaced_fasta[fragment_name] = \
                                    self.fragments_fasta[seq_name][ur[0]:ur[1]]

        self.unplaced_fasta = unplaced_fasta

    def _make_scaffolds_fasta(self):
        """
        Creates FASTA sequences of scaffolds
        """
        logger.info("Generating FASTA output")
        scaffolds_fasta = {}

        self.used_fragments_len = 0
        self.introduced_gap_len = 0
        for scf in self.scaffolds:
            scf_seqs = []
            for contig in scf.contigs:
                seq_name, seq_start, seq_end = contig.name_with_coords()
                cont_seq = self.fragments_fasta[seq_name][seq_start:seq_end]
                if contig.sign < 0:
                    cont_seq = reverse_complement(cont_seq)

                scf_seqs.append(cont_seq)
                if contig.link.gap > 0:
                    scf_seqs.append("N" * contig.link.gap)
                    self.introduced_gap_len += contig.link.gap

                self.used_fragments_len += len(cont_seq)

            scf_seq = "".join(scf_seqs)
            scaffolds_fasta[scf.name] = scf_seq

        self.scaffolds_fasta = scaffolds_fasta

    def _print_statistics(self):
        """
        Computes and prints some useful statistics
        """
        unplaced_len = sum(map(len, list(self.unplaced_fasta.values())))
        fragments_len = sum(map(len, list(self.fragments_fasta.values())))
        output_len = self.used_fragments_len + self.introduced_gap_len

        #used_perc = 100 * float(self.used_fragments_len) / fragments_len
        unplaced_perc = 100 * float(unplaced_len) / fragments_len
        gap_perc = 100 * float(self.introduced_gap_len) / output_len

        unplaced_count = len(self.unplaced_fasta)
        used_fragments_num = 0
        for scf in self.scaffolds:
            used_fragments_num += len(scf.contigs)

        contigs_len = [len(c) for c in self.fragments_fasta.values()]
        scaffolds_len = [len(c) for c in self.scaffolds_fasta.values()]
        contigs_n50 = _calc_n50(contigs_len, fragments_len)
        scaffolds_n50 = _calc_n50(scaffolds_len, output_len)

        logger.info("Assembly statistics:\n\n"
                    "\tScaffolds:\t\t{0}\n"
                    "\tUsed fragments:\t\t{1}\n"
                    "\tScaffolds length:\t{2}\n\n"
                    "\tUnplaced fragments:\t{3}\n"
                    "\tUnplaced length:\t{4} ({5:2.2f}%)\n"
                    "\tIntroduced Ns length:\t{6} ({7:2.2f}%)\n\n"
                    "\tFragments N50:\t\t{8}\n"
                    "\tAssembly N50:\t\t{9}\n"
                    .format(len(self.scaffolds), used_fragments_num,
                            output_len, unplaced_count, unplaced_len,
                            unplaced_perc, self.introduced_gap_len, gap_perc,
                            contigs_n50, scaffolds_n50))
ragout/phylogeny/phylogeny.py
class Phylogeny:
    """
    Represents phylogenetic tree and scores it with
    given half-breakpoint states
    """
    def __init__(self, tree):
        self.tree = tree
        self.tree_string = str(tree)
        self._scale_branches()

    @classmethod
    def from_newick(phylo, newick_str):
        return phylo(parse_tree(newick_str))

    @classmethod
    def from_permutations(phylo, perm_container):
        ti = TreeInferer(perm_container)
        return phylo(ti.build())

    def _scale_branches(self):
        """
        Fits mu coefficient according to branch lengths
        to avoid underflows/overflows
        """
        def tree_length(node):
            if node.terminal:
                return []

            lengths = []
            for node, _bootstrap, length in node.get_edges():
                assert length is not None
                lengths.append(length)
                lengths.extend(tree_length(node))

            return lengths

        lengths = tree_length(self.tree)
        assert len(lengths)
        self.mu = float(1) / _median(lengths)
        logger.debug("Branch lengths: %s, mu = %f", str(lengths), self.mu)

    def estimate_tree(self, leaf_states):
        """
        Scores the tree with weighted parsimony procedure
        """
        all_states = set(leaf_states.values())

        #score of a tree branch
        def branch_score(parent, child, branch):
            if parent == child or child is None:
                return 0.0
            else:
                #prevent underflow
                length = max(branch, 0.0000001)
                #adding one to counter possibly small exp value 
                return 1.0 + math.exp(-self.mu * length)

        #recursive
        def rec_helper(root):
            if root.terminal:
                leaf_score = (lambda s: 0.0 if s == leaf_states[root.identifier]
                                            else float("inf"))
                return {s : leaf_score(s) for s in all_states}

            nodes_scores = {}
            for node, _bootstrap, _length  in root.get_edges():
                nodes_scores[node] = rec_helper(node)

            root_scores = defaultdict(float)
            for root_state in all_states:
                for node, _bootstrap, branch_length in root.edges:
                    min_score = float("inf")
                    for child_state in all_states:
                        score = (nodes_scores[node][child_state] +
                                branch_score(root_state, child_state,
                                             branch_length))
                        min_score = min(min_score, score)
                    root_scores[root_state] += min_score

            return root_scores

        return min(rec_helper(self.tree).values())

    def terminals_dfs_order(self):
        """
        Returns terminal nodes' names in dfs order
        """
        def get_labels(root):
            if root.terminal:
                return [root.identifier]

            edges = sorted(root.get_edges(), key=lambda e: e[2], reverse=True)
            edges = sorted(edges, key=lambda e: e[0].terminal)
            #return list(chain(*map(lambda e: get_labels(e[0]), edges)))
            return list(chain(*[get_labels(e[0]) for e in edges]))

        return get_labels(self.tree)

    def leaves_by_distance(self, genome):
        """
        Returns leaves names sorted by the distance from
        the given genome.
        """
        graph = nx.Graph()
        start = [None]
        def rec_helper(root):
            if root.terminal and root.identifier == genome:
                start[0] = root
            if root.terminal:
                return
            for node, _bootstrap, branch_length in root.edges:
                graph.add_edge(root, node, weight=branch_length)
                rec_helper(node)

        rec_helper(self.tree)
        distances = nx.single_source_dijkstra_path_length(graph, start[0])
        leaves = [g for g in distances.keys()
                  if g.terminal and g.identifier != genome]

        #return list(map(lambda n: n.identifier,
        #                sorted(leaves, key=distances.get)))
        return [n.identifier for n in
                sorted(leaves, key=distances.get)]
ragout/parsers/recipe_parser.py
def parse_ragout_recipe(filename):
    if not os.path.exists(filename):
        raise RecipeException("Can't open recipe file")

    prefix = os.path.dirname(filename)

    recipe_dict = {"genomes" : {}}
    known_params = ["tree", "target", "blocks", "maf", "hal", "fasta",
                    "draft", "references", "naming_ref"]
    deprecated = ["circular"]
    required_params = ["references", "target"]

    cast_bool = ["circular", "draft"]
    fix_path = ["fasta", "maf", "hal"]

    defaults = {"circular" : False,
                "draft" : False}

    param_matcher = re.compile(r"([^\s]+)\s*=\s*([^\s].*)$")
    with open(filename, "r") as f:
        for lineno, line in enumerate(f):
            line = line.strip()
            if not line or line.startswith("#"):
                continue

            m = param_matcher.match(line)
            if not m or not "." in m.group(1):
                raise RecipeException("Error parsing recipe on line {0}"
                                      .format(lineno + 1))

            (obj, param_name), value = m.group(1).rsplit(".", 1), m.group(2)
            if param_name in deprecated:
                logger.warning("Recipe parameter '%s' is deprecated", param_name)
                continue
            if param_name not in known_params:
                raise RecipeException("Unknown recipe parameter '{0}' on line {1}"
                                      .format(param_name, lineno))

            #checking values, casting
            if param_name in cast_bool:
                if value.lower() in ["true", "1"]:
                    value = True
                elif value.lower() in ["false", "0"]:
                    value = False
                else:
                    raise RecipeException("Error parsing recipe on line "
                                          "{0}: wrong value '{1}' for bool param"
                                          .format(lineno, value))
            if param_name == "blocks":
                if value not in config.vals["blocks"]:
                    try:
                        value = list(map(int, value.split(",")))
                    except Exception:
                        raise RecipeException("Can't parse block size set: {0}"
                                              .format(value))
            if param_name == "references":
                value = [s.strip() for s in value.split(",")]
            if param_name in fix_path:
                value = os.path.expanduser(value)
                value = os.path.join(prefix, value)
            ###

            if obj == "":
                recipe_dict[param_name] = value
            elif obj == "*":
                defaults[param_name] = value
            else:
                recipe_dict["genomes"].setdefault(obj, {})[param_name] = value

    for param in required_params:
        if param not in recipe_dict:
            raise RecipeException("Required parameter '{0}' not found in recipe"
                                  .format(param))

    genomes = recipe_dict["references"] + [recipe_dict["target"]]
    if "tree" in recipe_dict:
        try:
            leaves = get_leaves_names(recipe_dict["tree"])
            if set(leaves) != set(genomes):
                raise RecipeException("The tree does not agree with "
                                      "the specified set of genomes")
        except PhyloException as e:
            raise RecipeException(e)

    for g in recipe_dict["genomes"]:
        if g not in genomes:
            raise RecipeException("Recipe error: genome '{0}' is not in "
                                  "specified as reference or target".format(g))

    for g in genomes:
        recipe_dict["genomes"].setdefault(g, {})

    for g, g_params in recipe_dict["genomes"].items():
        for def_key, def_val in defaults.items():
            g_params.setdefault(def_key, def_val)

    return recipe_dict
ragout/parsers/phylogeny_parser.py
class PhyloException(Exception):
    pass
ragout/overlap/overlap.py
class OverlapException(Exception):
    pass
ragout/breakpoint_graph/breakpoint_graph.py
class BreakpointGraph(object):
    """
    Breakpoint graph implementation
    """
    def __init__(self, perm_container=None):
        self.bp_graph = nx.MultiGraph()
        self.target = None
        self.references = []
        self.debug_nodes = set()
        if perm_container is not None:
            self.build_from(perm_container)

    def connected_component_subgraphs(self,G):
        for c in nx.connected_components(G):
            yield G.subgraph(c)

    def build_from(self, perm_container):
        """
        Builds breakpoint graph from permutations
        """
        for perm in perm_container.ref_perms:
            if perm.genome_name not in self.references:
                self.references.append(perm.genome_name)
        self.target = perm_container.target_perms[0].genome_name

        self.contig_ends = []
        for perm in perm_container.target_perms:
            self.contig_ends.append((perm.blocks[0].signed_id(),
                                     -perm.blocks[-1].signed_id()))

        for perm in chain(perm_container.ref_perms,
                          perm_container.target_perms):
            assert perm.blocks
            for prev_block, next_block in perm.iter_pairs():
                self.bp_graph.add_node(-prev_block.signed_id())
                self.bp_graph.add_node(next_block.signed_id())

                self.bp_graph.add_edge(-prev_block.signed_id(),
                                       next_block.signed_id(),
                                       genome_id=perm.genome_name,
                                       chr_name=perm.chr_name,
                                       start=prev_block.end,
                                       end=next_block.start,
                                       infinity=False)

            if perm.genome_name in self.references and not perm.draft:
                self.bp_graph.add_edge(-perm.blocks[-1].signed_id(),
                                       perm.blocks[0].signed_id(),
                                       genome_id=perm.genome_name,
                                       chr_name=perm.chr_name,
                                       infinity=True)

        logger.debug("Built breakpoint graph with %d nodes", len(self.bp_graph))

    def connected_components(self):
        #subgraphs = nx.connected_component_subgraphs(self.bp_graph)
        subgraphs = self.connected_component_subgraphs(self.bp_graph)
        bp_graphs = []
        for subgr in subgraphs:
            bg = BreakpointGraph()
            bg.target = self.target
            bg.references = copy(self.references)
            bg.bp_graph = subgr
            bp_graphs.append(bg)
        return bp_graphs

    def genomes_chrs_support(self, node_1, node_2):
        if not self.bp_graph.has_edge(node_1, node_2):
            return []
        return [GenChrPair(e["genome_id"], e["chr_name"]) for e in
                self.bp_graph[node_1][node_2].values()]

    def genomes_support(self, node_1, node_2):
        return [gp.genome for gp in self.genomes_chrs_support(node_1, node_2)]

    def to_weighted_graph(self, phylogeny):
        """
        Converts a breakpoint graph into a weighted adjacency graph
        using half-breakpoint state parsimony problem
        """
        assert len(self.bp_graph) >= 2
        g = nx.Graph()
        g.add_nodes_from(self.bp_graph.nodes)

        for node in self.bp_graph.nodes:
            adjacencies = {}
            for neighbor in self.bp_graph.neighbors(node):
                for edge in self.bp_graph[node][neighbor].values():
                    adjacencies[edge["genome_id"]] = neighbor

            for ref_id in self.references:
                if ref_id not in adjacencies:
                    adjacencies[ref_id] = None  #"void" state in paper

            break_weights = {}
            for neighbor in self.bp_graph.neighbors(node):
                adjacencies[self.target] = neighbor
                break_weights[neighbor] = phylogeny.estimate_tree(adjacencies)

            #normalization
            total_weights = sum(break_weights.values())
            for neighbor in self.bp_graph.neighbors(node):
                weight = (break_weights[neighbor] / total_weights
                          if total_weights != 0 else 0)
                _update_edge(g, node, neighbor, weight)

        return g

    #def add_debug_node(self, node):
    #    self.debug_nodes.add(node)

    def alternating_cycle(self, node_1, node_2):
        """
        Determines if there is a cycle of alternating colors
        that goes through the given red-supported (!) edge
        """
        def get_genome_ids(xxx_todo_changeme):
            (u, v) = xxx_todo_changeme
            return self.genomes_support(u, v)

        good_path = False
        path = None
        for path in self._alternating_paths(node_1, node_2):
            assert len(path) % 2 == 0
            if len(path) == 2:
                continue

            edges = list(zip(path[:-1], path[1:]))
            even_colors = list(map(get_genome_ids, edges[1::2]))
            even_good = all([set(e) == set([self.target]) for e in even_colors])
            if not even_good:
                continue

            odd_colors = [get_genome_ids(e) for e in edges[0::2]]
            common_genomes = set(odd_colors[0])
            for edge_colors in odd_colors:
                common_genomes = common_genomes.intersection(edge_colors)

            if common_genomes:
                #self._check_distances(path)
                good_path = True
                break

        return len(path) // 2 if good_path else None

    #def _check_distances(self, path):
    #    assert len(path) % 2 == 0
    #    path.append(path[0])
    #    edges = list(zip(path[:-1], path[1:]))
    #    even_dist = list(map(lambda (n1, n2): self.get_distance(n1, n2),
    #                         edges[1::2]))
    #    odd_dist = list(map(lambda (n1, n2): self.get_distance(n1, n2),
    #                        edges[0::2]))
    #    diff = abs(sum(even_dist) - sum(odd_dist))
    #    coeff = float(diff) / (sum(even_dist) + sum(odd_dist))
    #    logger.debug(coeff)

    def is_infinity(self, node_1, node_2):
        if not self.bp_graph.has_edge(node_1, node_2):
            return False

        for edge_data in self.bp_graph[node_1][node_2].values():
            if edge_data["infinity"]:
                return True
        return False

    def get_distance(self, node_1, node_2, phylogeny):
        """
        Tries to guess the distance between synteny blocks
        in a target genome
        """
        DEFAULT_DISTANCE = 0
        if not self.bp_graph.has_edge(node_1, node_2):
            return DEFAULT_DISTANCE
        distances = {e["genome_id"] : e["end"] - e["start"]
                     for e in self.bp_graph[node_1][node_2].values()}

        genomes_order = phylogeny.leaves_by_distance(self.target)
        for g in genomes_order:
            if g in distances:
                return distances[g]

        raise Exception("Distance function error")

    def debug_output(self):
        if not debugger.debugging:
            return

        graph_out = os.path.join(debugger.debug_dir, "breakpoint_graph.dot")
        _output_graph(self.bp_graph, graph_out)

    def _alternating_paths(self, src, dst):
        """
        Finds a path of alternating colors between two nodes
        """
        completed_paths = []
        visited = set()
        dfs_stack = [(src, True, [src])]

        #def rec_helper(node, colored):
        while dfs_stack:
            node, colored, cur_path = dfs_stack.pop()

            if node == dst:
                completed_paths.append(cur_path)
                continue
                #return [[dst]]

            visited.add(node)
            #paths = []
            for neighbor in self.bp_graph.neighbors(node):
                if neighbor in visited:
                    continue

                ##
                genomes = self.genomes_support(node, neighbor)
                #non_target = set(filter(lambda g: g != self.target, genomes))
                non_target = set([g for g in genomes if g != self.target])
                if colored and len(non_target) == 0:
                    continue
                if not colored and self.target not in genomes:
                    continue
                ##

                #far_paths = rec_helper(neighbor, not colored)
                dfs_stack.append((neighbor, not colored, cur_path + [neighbor]))
                #map(lambda p: p.append(node), far_paths)
                #paths.extend(far_paths)
            #visited.remove(node)
            #return paths

        #paths = list(map(lambda p: p[::-1], rec_helper(src, True)))
        return completed_paths
ragout/shared/debug.py
class DebugConfig():
    """
    Singleton providing global debug configuration
    """
    instance = None

    def __init__(self):
        self.debug_dir = None
        self.debugging = False

    def set_debugging(self, debugging):
        self.debugging = debugging

    def set_debug_dir(self, debug_dir):
        if not self.debugging:
            return
        self.debug_dir = debug_dir
        if not os.path.isdir(debug_dir):
            os.mkdir(debug_dir)

    def clear_debug_dir(self):
        if not self.debugging:
            return
        if os.path.isdir(self.debug_dir):
            shutil.rmtree(self.debug_dir)
            os.mkdir(self.debug_dir)

    @staticmethod
    def get_instance():
        if not DebugConfig.instance:
            DebugConfig.instance = DebugConfig()
        return DebugConfig.instance
ragout/breakpoint_graph/permutation.py
class PermException(Exception):
    pass
ragout/synteny_backend/synteny_backend.py
class BackendException(Exception):
    pass
ragout/breakpoint_graph/inferer.py
class AdjacencyInferer(object):
    def __init__(self, breakpoint_graph, phylogeny):
        self.main_graph = breakpoint_graph
        self.phylogeny = phylogeny
        self.orphans_count = 0
        self.guessed_count = 0
        self.trimmed_count = 0

    def connected_component_subgraphs(self,G):
        for c in nx.connected_components(G):
            yield G.subgraph(c)

    def infer_adjacencies(self):
        """
        Infers missing adjacencies by recovering perfect matching
        """
        logger.info("Inferring missing adjacencies")

        subgraphs = self.main_graph.connected_components()
        logger.debug("Found %d connected components", len(subgraphs))

        chosen_edges = []

        for subgraph in subgraphs:
            chosen_edges.extend(self._process_component(subgraph))

        logger.debug("Inferred %d adjacencies", len(chosen_edges))
        logger.debug("%d orphaned nodes", self.orphans_count)
        logger.debug("%d guessed edges", self.guessed_count)
        logger.debug("%d trimmed edges", self.trimmed_count)

        adjacencies = {}
        for node_1, node_2 in chosen_edges:
            distance = 0
            supporting_genomes = []
            infinity = self.main_graph.is_infinity(node_1, node_2)
            if not infinity:
                distance = self.main_graph.get_distance(node_1, node_2,
                                                        self.phylogeny)
                supporting_genomes = self.main_graph \
                                        .genomes_chrs_support(node_1, node_2)
                assert abs(node_1) != abs(node_2)

            adjacencies[node_1] = Adjacency(node_2, distance,
                                            supporting_genomes, infinity)
            adjacencies[node_2] = Adjacency(node_1, distance,
                                            supporting_genomes, infinity)

        self.main_graph.debug_output()
        self._debug_output(chosen_edges)

        return adjacencies

    def _process_component(self, subgraph):
        """
        Processes a connected component of the breakpoint graph
        """
        adjacency = subgraph.to_weighted_graph(self.phylogeny)
        trimmed_graph = self._trim_known_edges(adjacency)
        unused_nodes = set(trimmed_graph.nodes)

        chosen_edges = []
        #for trim_subgraph in nx.connected_component_subgraphs(trimmed_graph):
        for trim_subgraph in self.connected_component_subgraphs(trimmed_graph):
            if len(trim_subgraph) < 2:
                continue

            if len(trim_subgraph) == 2:
                chosen_edges.append(tuple(trim_subgraph.nodes))
                for n in trim_subgraph.nodes:
                    unused_nodes.remove(n)
                continue

            matching_edges = _min_weight_matching(trim_subgraph)

            for edge in matching_edges:
                for n in edge:
                    unused_nodes.remove(n)

            chosen_edges.extend(matching_edges)

        #predicting target-specific rearrangement
        #NO!
        #if len(unused_nodes) == 2:
        #    node_1, node_2 = tuple(unused_nodes)
        #    cycle = subgraph.alternating_cycle(node_1, node_2)
        #    if (abs(node_1) != abs(node_2) and cycle in [2, 3]):
        #        self.guessed_count += 1
        #        chosen_edges.append((node_1, node_2))
        #        unused_nodes.clear()
        self.orphans_count += len(unused_nodes)

        return chosen_edges

    def _trim_known_edges(self, graph):
        """
        Removes edges with known target adjacencies (red edges from paper)
        """
        trimmed_graph = graph.copy()
        for v1, v2 in graph.edges:
            if not trimmed_graph.has_node(v1) or not trimmed_graph.has_node(v2):
                continue

            genome_ids = set(self.main_graph.genomes_support(v1, v2))
            if self.main_graph.target in genome_ids:
                for node in [v1, v2]:
                    trimmed_graph.remove_node(node)
                self.trimmed_count += 1

        return trimmed_graph

    def _debug_output(self, chosen_edges):
        if not debugger.debugging:
            return

        phylo_out = os.path.join(debugger.debug_dir, "phylogeny.txt")
        edges_out = os.path.join(debugger.debug_dir, "predicted_edges.dot")
        _output_edges(chosen_edges, edges_out)
        _output_phylogeny(self.phylogeny.tree_string, self.main_graph.target,
                          phylo_out)
ragout/breakpoint_graph/chimera_detector.py
class ChimeraDetector(object):
    def __init__(self, breakpoint_graphs, run_stages, target_seqs):
        logger.info("Detecting chimeric adjacencies")
        self.bp_graphs = breakpoint_graphs
        self.run_stages = run_stages
        self.target_seqs = target_seqs
        self._make_hierarchical_breaks()

    def _make_hierarchical_breaks(self):
        """
        Determines where and at what synteny blocks scale to break each contig
        """
        seq_cuts = defaultdict(lambda : defaultdict(list))

        #extracting and grouping by sequence
        for stage in self.run_stages:
            logger.debug(">>With block size: %d", stage.block_size)
            breaks = self._get_contig_breaks(self.bp_graphs[stage])
            for br in breaks:
                seq_cuts[br.seq_name][stage].append(br)

        hierarchical_cuts = defaultdict(lambda : defaultdict(list))
        for seq_name in seq_cuts:
            for i in range(len(self.run_stages)):
                top_stage = self.run_stages[i]

                for top_break in seq_cuts[seq_name][top_stage]:
                    if top_break.good:
                        continue

                    adjusted_break = (top_break.begin, top_break.end)
                    for j in range(i + 1, len(self.run_stages)):
                        lower_stage = self.run_stages[j]
                        #check if there is overlapping cut
                        for lower_break in seq_cuts[seq_name][lower_stage]:
                            ovlp_left = max(adjusted_break[0], lower_break.begin)
                            ovlp_right = min(adjusted_break[1], lower_break.end)
                            #if so, update current and go down
                            if ovlp_right >= ovlp_left:
                                adjusted_break = (ovlp_left, ovlp_right)
                                break

                    break_pos = self._optimal_break(seq_name, *adjusted_break)
                    hierarchical_cuts[seq_name][top_stage].append(break_pos)
        self.hierarchical_cuts = hierarchical_cuts

    def _optimal_break(self, seq_name, break_start, break_end):
        """
        Finds the longet run of Ns within given range
        """
        seq = self.target_seqs[seq_name]
        cur_pos = break_start
        cur_len = 0
        max_pos = break_start
        max_len = 0
        for i in range(break_start, break_end):
            if seq[i].upper() == "N":
                cur_len += 1
            if seq[i].upper() != "N" or i == break_end - 1:
                if max_len < cur_len:
                    max_len = cur_len
                    max_pos = cur_pos
                cur_pos = i + 1
                cur_len = 0
        return max_pos + max_len // 2

    def _get_contig_breaks(self, bp_graph):
        """
        Detects chimeric adjacencies
        """
        seq_cuts = []

        subgraphs = bp_graph.connected_components()
        num_target_adj = 0
        num_unique_adj = 0
        num_removed_adj = 0
        for subgr in subgraphs:
            if len(subgr.bp_graph) > 100:
                logger.debug("Processing component of size %d",
                             len(subgr.bp_graph))

            for (u, v, data) in subgr.bp_graph.edges(data=True):
                if data["genome_id"] != subgr.target:
                    continue

                num_target_adj += 1
                genomes = set(subgr.genomes_support(u, v))
                if (genomes != set([bp_graph.target])
                    and not subgr.is_infinity(u, v)):
                    continue

                num_unique_adj += 1
                seq_name, start, end = data["chr_name"], data["start"], data["end"]
                if subgr.alternating_cycle(u, v) is not None:
                    seq_cuts.append(ContigBreak(seq_name, start, end, True))
                    continue

                seq_cuts.append(ContigBreak(seq_name, start, end, False))
                num_removed_adj += 1

        #bp_graph.debug_output()
        logger.debug("Checking %d target adjacencies", num_target_adj)
        logger.debug("Found %d target-specific adjacencies, "
                     "%d broken as chimeric", num_unique_adj, num_removed_adj)
        return seq_cuts

    def _valid_2break(self, bp_graph, red_edge):
        """
        Checks if there is a valid 2-break through the given red edge
        """
        assert len(bp_graph) == 4
        red_1, red_2 = red_edge
        cand_1, cand_2 = tuple(set(bp_graph.nodes) - set(red_edge))
        if abs(cand_1) == abs(cand_2):
            return False

        if bp_graph.has_edge(red_1, cand_1):
            if not bp_graph.has_edge(red_2, cand_2):
                return False
            known_1 = red_1, cand_1
            known_2 = red_2, cand_2
        elif bp_graph.has_edge(red_1, cand_2):
            if not bp_graph.has_edge(red_2, cand_1):
                return False
            known_1 = red_1, cand_2
            known_2 = red_2, cand_1
        else:
            return False

        chr_1 = {}
        for data in bp_graph[known_1[0]][known_1[1]].values():
            chr_1[data["genome_id"]] = data["chr_name"]
        chr_2 = {}
        for data in bp_graph[known_2[0]][known_2[1]].values():
            chr_2[data["genome_id"]] = data["chr_name"]
        common_genomes = set(chr_1.keys()).intersection(list(chr_2.keys()))
        for genome in common_genomes:
            if chr_1[genome] != chr_2[genome]:
                return False

        return True

    def break_contigs(self, perm_container, block_sizes):
        """
        Breaks contigs in inferred cut positions
        """
        logger.info("Removing chimeric adjacencies")

        new_container = deepcopy(perm_container)
        new_target_perms = []
        num_breaks = 0
        num_chimeras = 0

        for perm in new_container.target_perms:
            break_points = []
            for size in block_sizes:
                break_points.extend(self.hierarchical_cuts[perm.chr_name][size])
            break_points = list(set(break_points))
            num_breaks += len(break_points)
            if not break_points:
                new_target_perms.append(perm)
            else:
                num_chimeras += 1
                new_target_perms.extend(_break_permutation(perm, break_points))

        logger.debug("Chimera Detector: %d cuts made in %d sequences",
                     num_breaks, num_chimeras)
        new_container.target_perms = new_target_perms
        return new_container
ragout/parsers/fasta_parser.py
class FastaError(Exception):
    pass
ragout/breakpoint_graph/permutation.py
class PermutationContainer:
    def __init__(self, block_coords_file, recipe,
                 resolve_repeats, allow_ref_indels, phylogeny):
        """
        Parses permutation files referenced from recipe and filters duplications
        """
        self.ref_perms = []
        self.target_perms = []
        self.recipe = recipe

        logging.info("Reading %s", block_coords_file)
        permutations = _parse_blocks_coords(block_coords_file)

        has_sequences = set()
        draft_names = set()
        for p in permutations:
            if p.genome_name not in recipe["genomes"]:
                continue
            p.draft = recipe["genomes"][p.genome_name]["draft"]
            if p.draft:
                draft_names.add(p.genome_name)

            has_sequences.add(p.genome_name)
            if p.genome_name == recipe["target"]:
                self.target_perms.append(p)
            else:
                self.ref_perms.append(p)

        for genome in recipe["genomes"]:
            if genome not in has_sequences:
                raise PermException("No sequences read for genome {0}. Check "
                                    "recipe for correctness.".format(genome))
        _check_coverage(self.ref_perms + self.target_perms)

        logger.debug("Read %d reference sequences", len(self.ref_perms))
        if not len(self.ref_perms):
            raise PermException("No synteny blocks found in "
                                "reference sequences")
        logger.debug("Read %d target sequences", len(self.target_perms))
        if not len(self.target_perms):
            raise PermException("No synteny blocks found in "
                                "target sequences")

        self._filter_indels(allow_ref_indels)
        logger.debug("%d target sequences left after indel filtering",
                     len(self.target_perms))

        repeats = _find_repeats(self.ref_perms + self.target_perms)
        ###
        if resolve_repeats:
            if phylogeny is None:
                raise PermException("Resolving repeats with "
                                    "yet unknown phylogeny")
            rr.resolve_repeats(self.ref_perms, self.target_perms,
                               repeats, phylogeny, draft_names)
        ###
        self._filter_repeats(repeats)
        logger.debug("%d target sequences left after repeat filtering",
                     len(self.target_perms))
        if not len(self.target_perms):
            raise PermException("No synteny blocks found in the target "
                                "genome after repeat/indel filtering.")

        if debugger.debugging:
            filtered_file = os.path.join(debugger.debug_dir, "filtered_contigs.txt")
            output_permutations(self.target_perms, filtered_file)

    def _filter_indels(self, allow_ref_indels):
        """
        Keep only blocks that appear in target and
        all references (or one reference, if allow_ref_indels is set)
        """
        multiplicity = defaultdict(int)
        target_blocks = set()
        reference_blocks = set()
        def process(perms, block_set):
            for perm in perms:
                for block in perm.blocks:
                    multiplicity[block.block_id] += 1
                    block_set.add(block.block_id)

        process(self.target_perms, target_blocks)
        process(self.ref_perms, reference_blocks)

        if allow_ref_indels:
            to_keep = target_blocks.intersection(reference_blocks)
        else:
            num_genomes = len(self.recipe["genomes"])
            #to_keep = set(filter(lambda b: multiplicity[b] >= num_genomes,
            #                     multiplicity))
            to_keep = set([b for b in multiplicity if multiplicity[b] >= num_genomes])

        self.ref_perms = _filter_permutations(self.ref_perms, to_keep)
        self.target_perms = _filter_permutations(self.target_perms, to_keep)


    def _filter_repeats(self, repeats):
        """
        Filters repetitive blocks
        """
        self.target_perms = _filter_permutations(self.target_perms, repeats,
                                                 inverse=True)
        self.ref_perms = _filter_permutations(self.ref_perms, repeats,
                                              inverse=True)
ragout/parsers/fasta_parser.py
def read_fasta_dict(filename):
    """
    Reads fasta file into dictionary. Also preforms some validation
    """
    logger.info("Reading contigs file")

    header = None
    seq = []
    fasta_dict = {}

    try:
        with open(filename, "rb") as f:
            for lineno, line in enumerate(f):
                line = line.strip()
                if line.startswith(b">"):
                    if header:
                        fasta_dict[_STR(header)] = _STR(b"".join(seq))
                        seq = []
                    header = line[1:].split(b" ")[0]
                else:
                    if not _validate_seq(line):
                        raise FastaError("Invalid char in \"{0}\" at line {1}"
                                         .format(filename, lineno))
                    seq.append(line)

            if header and len(seq):
                fasta_dict[_STR(header)] = _STR(b"".join(seq))

    except IOError as e:
        raise FastaError(e)

    return fasta_dict
ragout/parsers/recipe_parser.py
class RecipeException(Exception):
    pass
ragout/synteny_backend/synteny_backend.py
class SyntenyBackend:
    backends = {}
    def __init__(self):
        self.target_fasta = None
        self.threads = None
        self.blocks = None

    def make_permutations(self, recipe, blocks,
                          output_dir, overwrite, threads):
        """
        Runs backend and then prepare data for futher processing
        """
        self.target_fasta = recipe["genomes"][recipe["target"]].get("fasta")
        self.threads = threads
        self.blocks = blocks

        files = self.run_backend(recipe, output_dir, overwrite)
        assert sorted(files.keys()) == sorted(blocks)

        return files

    def run_backend(self, _recipe, _output_dir, _overwrite):
        """
        Runs backend and returns a dict with permutations files
        Indexed by block sizes
        """
        return None

    def get_target_fasta(self):
        """
        Returns a path to a fasta file with contigs
        """
        return self.target_fasta

    def infer_block_scale(self, recipe):
        """
        Infers synteny block scale based on target assembly size
        """
        target_fasta = recipe["genomes"][recipe["target"]].get("fasta")
        if not target_fasta or not os.path.exists(target_fasta):
            raise BackendException("Could not open target FASTA file "
                                   "or it is not specified")
        size = os.path.getsize(target_fasta)
        if size < config.vals["big_genome_threshold"]:
            return "small"
        else:
            return "large"

    @staticmethod
    def get_available_backends():
        return SyntenyBackend.backends

    @staticmethod
    def register_backend(name, instance):
        assert name not in SyntenyBackend.backends
        SyntenyBackend.backends[name] = instance


Question: import os
import shutil
import logging
import argparse
import ragout.assembly_graph.assembly_refine as asref
import ragout.scaffolder.scaffolder as scfldr
import ragout.scaffolder.merge_iters as merge
import ragout.maf2synteny.maf2synteny as m2s
import ragout.overlap.overlap as overlap
import ragout.shared.config as config
import ragout.synteny_backend.sibelia
import ragout.synteny_backend.maf
import ragout.synteny_backend.hal
import ragout.six as six
from collections import namedtuple
from ragout.scaffolder.output_generator import OutputGenerator
from ragout.overlap.overlap import OverlapException
from ragout.phylogeny.phylogeny import Phylogeny
from ragout.parsers.phylogeny_parser import PhyloException
from ragout.breakpoint_graph.permutation import (PermutationContainer,
                                                 PermException)
from ragout.synteny_backend.synteny_backend import (SyntenyBackend,
                                                    BackendException)
from ragout.parsers.recipe_parser import parse_ragout_recipe, RecipeException
from ragout.parsers.fasta_parser import read_fasta_dict, FastaError
from ragout.shared.debug import DebugConfig
from ragout.breakpoint_graph.breakpoint_graph import BreakpointGraph
from ragout.breakpoint_graph.inferer import AdjacencyInferer
from ragout.breakpoint_graph.chimera_detector import ChimeraDetector
from ragout.__version__ import __version__
    backends = SyntenyBackend.get_available_backends()
    if backend not in backends:
        raise BackendException("\"{0}\" is not installed.".format(backend))

    if not m2s.check_binary():
        raise BackendException("maf2synteny binary is missing, "
                               "did you run 'make'?")

    if not overlap.check_binary():
        raise BackendException("overlap binary is missing, "
                               "did you run 'make'?")


def _make_run_stages(block_sizes, resolve_repeats):
    """
    Setting parameters of run stages (iterations)
    """
    stages = []
    #general stages for structural assembly
    for block in block_sizes:
        stages.append(RunStage(name=str(block), block_size=block,
                               ref_indels=False, repeats=False,
                               rearrange=True))

    #refining stage to close assembly gaps
    stages.append(RunStage(name="refine", block_size=block_sizes[ID_SMALLEST],
                           ref_indels=True, repeats=resolve_repeats,
                           rearrange=False))
    return stages


def _get_phylogeny_and_naming_ref(recipe, permutation_file):
    """
    Retrieves phylogeny (infers if necessary) as well as
    naming reference genome
    """
    if "tree" in recipe:
        logger.info("Phylogeny is taken from the recipe")
        phylogeny = Phylogeny.from_newick(recipe["tree"])
    else:
        logger.info("Inferring phylogeny from synteny blocks data")
        perm_cont = PermutationContainer(permutation_file, recipe,
                                         resolve_repeats=False,
                                         allow_ref_indels=True,
                                         phylogeny=None)
        phylogeny = Phylogeny.from_permutations(perm_cont)
        logger.info("Inferred tree: %s", phylogeny.tree_string)

    if "naming_ref" in recipe:
        naming_ref = recipe["naming_ref"]
    else:
        leaves_sorted = phylogeny.leaves_by_distance(recipe["target"])
        naming_ref = leaves_sorted[0]
        logger.info("'%s' is chosen as a naming reference", naming_ref)

    return phylogeny, naming_ref


def _get_synteny_scale(recipe, synteny_backend):
    if "blocks" in recipe:
        if isinstance(recipe["blocks"], six.string_types):
            scale = config.vals["blocks"][recipe["blocks"]]
        else:
            scale = recipe["blocks"]
    else:
        scale = config.vals["blocks"][synteny_backend.infer_block_scale(recipe)]

    logger.info("Running withs synteny block sizes '%s'", str(scale))
    return scale


def _run_ragout(args):
    """
    Top-level logic of the program
    """
    if not os.path.isdir(args.out_dir):
        os.mkdir(args.out_dir)

    debug_root = os.path.join(args.out_dir, "debug")
    debugger.set_debugging(args.debug)
    debugger.set_debug_dir(debug_root)
    debugger.clear_debug_dir()

    out_log = os.path.join(args.out_dir, "ragout.log")
    _enable_logging(out_log, args.debug)
    logger.info("Starting Ragout v%s", str(__version__))

    #parsing recipe, preparing synteny backend
    _check_extern_modules(args.synteny_backend)
    synteny_backend = SyntenyBackend.get_available_backends() \
                                        [args.synteny_backend]
    recipe = parse_ragout_recipe(args.recipe)
    synteny_sizes = _get_synteny_scale(recipe, synteny_backend)

    #Running synteny backend to get synteny blocks
    perm_files = synteny_backend.make_permutations(recipe, synteny_sizes,
                                                   args.out_dir, args.overwrite,
                                                   args.threads)

    #setting up phylogenetic tree
    phylo_perm_file = perm_files[synteny_sizes[ID_SMALLEST]]
    phylogeny, naming_ref = _get_phylogeny_and_naming_ref(recipe,
                                                          phylo_perm_file)

    #parsing permutation files, apply filters and build breakpoint graph
    logger.info("Processing permutation files")
    raw_bp_graphs = {}
    stage_perms = {}
    run_stages = _make_run_stages(synteny_sizes, args.resolve_repeats)
    for stage in run_stages:
        debugger.set_debug_dir(os.path.join(debug_root, stage.name))
        stage_perms[stage] = PermutationContainer(perm_files[stage.block_size],
                                                  recipe, stage.repeats,
                                                  stage.ref_indels, phylogeny)
        raw_bp_graphs[stage] = BreakpointGraph(stage_perms[stage])

    #initializing chimera detector
    target_sequences = read_fasta_dict(synteny_backend.get_target_fasta())
    chim_detect = None
    if not args.solid_scaffolds:

Answer:         chim_detect = ChimeraDetector(raw_bp_graphs, run_stages, target_sequences)