camsa/core/data_structures.py
def merge_assembly_points(assembly_points_by_source):
    unique_assembly_points = defaultdict(list)

    for assembly_points in assembly_points_by_source.values():
        for assembly_point in assembly_points:
            seq1, seq2 = assembly_point.seq1, assembly_point.seq2
            seq1_or, seq2_or = assembly_point.seq1_or, assembly_point.seq2_or
            if seq1 < seq2:
                entry = (seq1, seq1_or, seq2, seq2_or)
            else:
                entry = (seq2, inverse_orientation(seq2_or), seq1, inverse_orientation(seq1_or))
            unique_assembly_points[entry].append(assembly_point)
    result = []
    for (seq1, seq1_or, seq2, seq2_or), children in unique_assembly_points.items():
        sources = sorted(set(source for ap in children for source in ap.sources))
        weight = sum(ap.cw for ap in children)
        children_ids = [ap.self_id for ap in children]
        merged_assembly_point = AssemblyPoint(seq1=seq1, seq2=seq2, seq1_or=seq1_or, seq2_or=seq2_or,
                                              sources=sources, cw=weight, children_ids=children_ids)
        result.append(merged_assembly_point)
    return result
camsa/core/data_structures.py
def assign_ids_to_assembly_points(assembly_points, id_prefix="", id_generator=None, skip_existing=True, sort=False):
    assembly_points_by_ids = {}
    if id_generator is None:
        id_generator = itertools.count()
    existing_ids = set()
    if not skip_existing:
        for ap in assembly_points:
            if ap.self_id is not None and ap.self_id != "?":
                existing_ids.add(ap.self_id)
    if sort:
        s_assembly_points = sorted(assembly_points, key=lambda entry: (entry.seq1, entry.seq2, entry.seq1_or, entry.seq2_or, entry.sources))
    else:
        s_assembly_points = assembly_points
    for assembly_point in s_assembly_points:
        if skip_existing:
            new_id = id_prefix + str(six.next(id_generator))
            assembly_point.self_id = new_id
        elif assembly_point.self_id is None or assembly_point.self_id == "?":
            new_id = id_prefix + str(six.next(id_generator))
            while new_id in existing_ids:
                new_id = id_prefix + str(six.next(id_generator))
            assembly_point.self_id = new_id
        assembly_points_by_ids[assembly_point.self_id] = assembly_point
    return assembly_points_by_ids
camsa/core/comparative_analysis.py
def compute_and_update_assembly_points_conflicts(assembly_points_by_ids):
    """

    :param assembly_points_by_ids: a dictionary, where key is the assembly point id, and value is merged assembly points
    :return:
    """
    sag = ScaffoldAssemblyGraph.from_assembly_points(assembly_points=assembly_points_by_ids.values(),
                                                     add_scaffold_edges=False)
    for ap in assembly_points_by_ids.values():
        conflicted_assembly_points = get_conflicting_assembly_points(sag=sag, assembly_point=ap,
                                                                     assembly_points_by_ids=assembly_points_by_ids)
        for c_ap in conflicted_assembly_points:
            conflict_type = get_conflict_type(ap1=ap, ap2=c_ap)
            assert conflict_type != Conflicts.non_conflicted
            if conflict_type == Conflicts.conflicted:
                update_assembly_points_as_conflicted(ap1=ap, ap2=c_ap)
            elif conflict_type == Conflicts.semi_conflicted:
                update_assembly_points_as_semi_conflicted(ap1=ap, ap2=c_ap)
camsa/core/data_structures.py
def to_json(value):
    if isinstance(value, Sequence):
        return json.dumps({"seq_id": value.name, "length": value.length})
    result = {}
    for key, values in value.items():
        if isinstance(values, str):
            result[key] = values
        else:
            result[key] = sorted(values)
    return json.dumps(result)
camsa/core/data_structures.py
class OrderGraph(object):
    def __init__(self, graph=None):
        self.graph = graph if graph is not None else networkx.Graph()

    def add_ap(self, ap):
        seq1, seq2 = ap.seq1, ap.seq2
        if self.graph.has_edge(u=seq1, v=seq2):
            self.graph[seq1][seq2]["ids"].append(ap.self_id)
            self.graph[seq1][seq2]["sources"].extend(ap.sources)
        else:
            self.graph.add_edge(seq1, seq2, ids=[ap.self_id], sources=[source for source in ap.sources])

    @classmethod
    def from_aps(cls, aps):
        result = cls()
        for ap in aps:
            result.add_ap(ap=ap)
        return result
camsa/core/merging.py
class MergingStrategies(enum.Enum):
    greedy_merging = "greedy"
    maximal_matching = "maximal-matching"
camsa/core/data_structures.py
class Sequence(object):
    def __init__(self, name, length=None, parent_seq_id=None, start=None, end=None, strand="+", annotation=None, seq_group_id="Default"):
        self.name = name
        self._length = length
        self._parent_seq_id = parent_seq_id
        self._start = start
        self._end = end
        self._seq_group_id = seq_group_id
        self.strand = strand
        self._annotation = annotation

    @property
    def length(self):
        if self._length is not None:
            return self._length
        if self.start is not None and self.end is not None:
            return self.end - self.start
        return None

    @property
    def parent_seq_id(self):
        if self._parent_seq_id is None:
            return self.name
        return self._parent_seq_id

    @property
    def start(self):
        if self._start is None:
            return 0
        return int(self._start)

    @property
    def end(self):
        if self._end is None:
            return sys.maxsize
        return int(self._end)

    @property
    def seq_group_id(self):
        return str(self._seq_group_id)

    @property
    def annotation(self):
        return str(self._annotation)
camsa/core/data_structures.py
class AssemblyPoint(object):
    def __init__(self, seq1, seq2, seq1_or, seq2_or, sources, cw=None, parent_id=None, children_ids=None, self_id=None,
                 gap_size=None):
        self.seq1 = seq1
        self.seq2 = seq2
        self.seq1_or = seq1_or
        self.seq2_or = seq2_or
        self.sources = sorted(sources)
        self.cw = cw
        self.gap_size = gap_size
        self.self_id = self_id
        self.parent_id = parent_id
        self.children_ids = children_ids if children_ids is not None else []
        #### comparative metrics
        self.participates_in_merged = False
        self.in_conflicted = defaultdict(set)
        self.in_semi_conflicted = defaultdict(set)
        self.out_conflicted = defaultdict(set)
        self.out_semi_conflicted = defaultdict(set)
        self.seq1_par_or = None
        self.seq2_par_or = None
        #### ref metrics
        self.ref_metrics = RefMetrics()
        self.extra_data = {}

    @property
    def orientation_as_word(self):
        if self.is_unoriented:
            return "U"
        if self.is_semi_oriented:
            return "SO"
        return "O"

    @property
    def is_unoriented(self):
        return [self.seq1_or, self.seq2_or].count("?") == 2

    @property
    def is_semi_oriented(self):
        return [self.seq1_or, self.seq2_or].count("?") == 1

    @property
    def is_oriented(self):
        return [self.seq1_or, self.seq2_or].count("?") == 0

    def is_in_semi_conflicted_for(self, source_name):
        return source_name in self.in_semi_conflicted

    def is_in_conflicted_for(self, source_name):
        return source_name in self.in_conflicted

    def is_out_semi_conflicted_for(self, source_name):
        return len(self.out_semi_conflicted) > 0

    def is_out_conflicted_for(self, source_name):
        return len(self.out_conflicted) > 0

    @property
    def is_out_conflicted(self):
        return len(self.out_conflicted) > 0

    @property
    def is_out_semi_conflicted(self):
        return len(self.out_semi_conflicted) > 0

    @property
    def is_non_conflicted(self):
        return len(self.in_conflicted) == 0 and len(self.out_conflicted) == 0 and len(self.in_semi_conflicted) == 0 and len(
            self.out_semi_conflicted) == 0

    def get_edges(self, weight=False, sort=True):
        result = []
        for ap in self.get_all_all_possible_realizations():
            head = ap.seq1 + ("h" if ap.seq1_or == "+" else "t")
            tail = ap.seq2 + ("t" if ap.seq2_or == "+" else "h")
            hv = head
            tv = tail
            if sort:
                hv, tv = (hv, tv) if hv < tv else (tv, hv)
            if weight:
                result.append((hv, tv, self.cw))
            else:
                result.append((hv, tv))
        return result

    def get_all_all_possible_realizations(self):
        result = []
        if self.seq1_or == "?":
            seq1_choices = ["+", "-"]
        else:
            seq1_choices = [self.seq1_or]
        if self.seq2_or == "?":
            seq2_choices = ["+", "-"]
        else:
            seq2_choices = [self.seq2_or]
        for ctg1_or, ctg2_or in itertools.product(seq1_choices, seq2_choices):
            result.append(AssemblyPoint(seq1=self.seq1, seq2=self.seq2, seq1_or=ctg1_or, seq2_or=ctg2_or,
                                        sources=self.sources))
        return result

    @property
    def is_ambiguous(self):
        return self.seq1_or == "?" or self.seq2_or == "?"
camsa/core/merging.py
def update_assembly_points_with_merged_assembly(original_assembly_points_by_ids, merged_assembly_points_by_ids, merged_assembly_graph):
    for ap in merged_assembly_points_by_ids.values():
        for u, v in ap.get_edges():
            if merged_assembly_graph.has_edge(u=u, v=v):
                par_or_1 = "+" if u.endswith("h") else "-"
                par_or_2 = "+" if v.endswith("t") else "-"
                forward = u[:-1] == ap.seq1
                if not forward:
                    par_or_1, par_or_2 = inverse_orientation(par_or_2), inverse_orientation(par_or_1)
                ap.seq1_par_or = par_or_1
                ap.seq2_par_or = par_or_2
                ap.participates_in_merged = True
                for child_id in ap.children_ids:
                    child = original_assembly_points_by_ids[child_id]
                    child.participates_in_merged = True
                    child.seq1_par_or = par_or_1
                    child.seq2_par_or = par_or_2
                break
camsa/core/data_structures.py
class Assembly(object):
    def __init__(self, name, aps):
        self.name = name
        self.aps = aps
        self.total_cnt = 0
        self.max_non_conflicting_aps = []

    @property
    def entries_names(self):
        return self.get_all_entries_names()

    def get_all_entries_names(self):
        return sorted(set(name for ap in self.aps for name in ap.sources))

    def sort_aps(self):
        self.aps = sorted(self.aps,
                          key=lambda ap: (ap.contig_1, ap.contig_1_orientation, ap.contig_2, ap.contig_2_orientation))

    @property
    def unoriented_ap_cnt(self):
        return sum(ap.is_unoriented for ap in self.aps)

    @property
    def semi_oriented_aps_cnt(self):
        return sum(ap.is_semi_oriented for ap in self.aps)

    @property
    def oriented_aps_cnt(self):
        return sum(ap.is_oriented for ap in self.aps)

    @property
    def in_conflicted_cnt(self):
        cnt = 0
        for ap in self.aps:
            if isinstance(self.name, tuple):
                to_check = self.name
            else:
                to_check = [self.name]
            for name in to_check:
                if ap.is_in_conflicted_for(name):
                    cnt += 1
                    break
        return cnt

    @property
    def in_semi_conflicted_cnt(self):
        cnt = 0
        for ap in self.aps:
            if isinstance(self.name, tuple):
                to_check = self.name
            else:
                to_check = [self.name]
            for name in to_check:
                if ap.is_in_semi_conflicted_for(name):
                    cnt += 1
                    break
        return cnt

    @property
    def out_conflicted_cnt(self):
        return sum(ap.is_out_conflicted for ap in self.aps)

    @property
    def out_semi_conflicted_cnt(self):
        return sum(ap.is_out_semi_conflicted for ap in self.aps)

    @property
    def non_conflicted_cnt(self):
        return len([ap for ap in self.aps if ap.is_non_conflicted])
camsa/core/merging.py
def update_gap_sizes_in_merged_assembly(original_assembly_points_by_ids, merged_assembly_points_by_ids):
    aps_in_merged_assembly = [ap for ap in merged_assembly_points_by_ids.values() if ap.participates_in_merged]
    for ap in aps_in_merged_assembly:
        children_aps = [original_assembly_points_by_ids[c_id] for c_id in ap.children_ids]
        cumulative_gap_size = 0.0
        gap_size_sources_cnt = 0.0
        for c_ap in children_aps:
            if isinstance(c_ap.gap_size, numbers.Number):
                gap_size_sources_cnt += 1
                cumulative_gap_size += c_ap.gap_size
        if gap_size_sources_cnt == 0:
            inferred_gap_size = "?"
        else:
            inferred_gap_size = cumulative_gap_size / gap_size_sources_cnt
        ap.gap_size = inferred_gap_size
camsa/core/merging.py
class MergingStrategies(enum.Enum):
def get_un_oriented_assembly_points(assembly_points):
def merge_greedily(assembly_points_by_sources, acyclic=True, min_cw=0.0):
    def get_redundant_edges_from_assembly_points(e, points_by_edges, processed_points):
def maximal_matching(assembly_points_by_sources, acyclic=True, min_cw=0.0):
def get_edges_from_matching(matching):
def update_assembly_points_with_merged_assembly(original_assembly_points_by_ids, merged_assembly_points_by_ids, merged_assembly_graph):
def update_gap_sizes_in_merged_assembly(original_assembly_points_by_ids, merged_assembly_points_by_ids):
camsa/core/data_structures.py
def assign_parents_to_children(children_assembly_points_by_ids, parent_assembly_points_by_ids):
    for p_assembly_point in parent_assembly_points_by_ids.values():
        for child_id in p_assembly_point.children_ids:
            children_assembly_points_by_ids[child_id].parent_id = p_assembly_point.self_id
camsa/core/io.py
def get_fn_relations_for_column_names(fieldnames, aliases):
def extract_nullable_value(field, row, fn_relations, default="?"):
def extract_nullable_numerical_value(field, row, fn_relations, default="?"):
def read_pairs(source, delimiter="\t", destination=None, default_cw_eae=1, default_cw_cae=0.9, read_ids=False, read_extra_data=False):
def read_assembly_points_from_input_sources(sources, delimiter="\t", default_cw_eae=1, default_cw_cae=0.75, read_ids=False, read_extra_data=False):
def read_lengths(source, delimiter="\t", destination=None):
def get_header_and_extract_list(settings):
def write_assembly_points(assembly_points, destination, output_setup, delimiter="\t"):
def write_seqi(sequences, destination, output_setup, delimiter="\t"):
def read_seqi_from_input_sources(source, delimiter="\t", destination=None):
def remove_dir(dir_path):
PAIRS_COLUMN_ALIASES = {
    ########################
    "species": "origin",
    "organisms": "origin",
    "origin": "origin",
    ########################
    "ctg1": "seq1",
    "contig1": "seq1",
    "seq1": "seq1",
    ########################
    "ctg2": "seq2",
    "contig2": "seq2",
    "seq2": "seq2",
    ########################
    "orientation_ctg1": "seq1_or",
    "orientation_contig1": "seq1_or",
    "ctg1_orientation": "seq1_or",
    "ctg1_or": "seq1_or",
    "seq1_or": "seq1_or",
    ########################
    "ctg2_orientation": "seq2_or",
    "orientation_ctg2": "seq2_or",
    "orientation_contig2": "seq2_or",
    "ctg2_or": "seq2_or",
    "seq2_or": "seq2_or",
    ########################
    "distance": "gap_size",
    "ctg1-ctg2_gap": "gap_size",
    "gap_size": "gap_size",
    ########################
    "score": "cw",
    "cw": "cw",
    ########################
    "self_id": "self_id",
}
LENGTHS_COLUMN_ALIASES = {
    ########################
    "ctg_id": "seq_id",
    ########################
    "ctg_length": "seq_length"
}
class OrientationChoice(Enum):


Question: import datetime
import itertools
import logging
import os
import shutil
import sys
import configargparse
import six
import camsa
from collections import defaultdict
from jinja2 import FileSystemLoader
from jinja2.environment import Environment
from camsa.core import io as camsa_io
from camsa.core import merging
from camsa.core.comparative_analysis import compute_and_update_assembly_points_conflicts
from camsa.core.data_structures import Assembly, assign_ids_to_assembly_points, merge_assembly_points, assign_parents_to_children, to_json, Sequence, OrderGraph, AssemblyPoint
from camsa.core.merging import MergingStrategies, update_assembly_points_with_merged_assembly, update_gap_sizes_in_merged_assembly
    #                     help="")
    parser.add_argument("--c-merging-cw-min", type=float,
                        help="A threshold for the minimum cumulative confidence weight for merged assembly edges in MSAG.\nEdges with confidence weight below are not considered in the \"merged\" assembly construction.\nDEFAULT: 0.0")
    parser.add_argument("--c-merging-strategy", choices=[MergingStrategies.greedy_merging.value, MergingStrategies.maximal_matching.value],
                        default=MergingStrategies.maximal_matching.value,
                        help="A strategy to produced a merged assembly from the given ones.\nDEFAULT: maximal-matching")
    parser.add_argument("--c-merging-cycles", dest="allow_cycles", action="store_true", default=False,
                        help="Whether to allow cycles in the produced merged assembly.\nDEFAULT: False")
    parser.add_argument("--version", action="version", version=camsa.VERSION)
    parser.add_argument("--i-delimiter", default="\t", type=str,
                        help="String used as a delimiter in the input files with CAMSA assembly points")
    parser.add_argument("-o", "--o-dir",
                        help="A directory, where CAMSA will store all of the produced output (report, assets, etc).\nDEFAULT: camsa_{date}")
    # parser.add_argument("--o-interactive-disable", action="store_false", dest="o_interactive", default=True,
    #                     help="")
    parser.add_argument("--o-merged-format", type=str,
                        help="The CAMSA-out formatting for the merged scaffold assemblies in a form of CAMSA points.")
    parser.add_argument("--o-subgroups-format", type=str,
                        help="The CAMSA-out formatting for the subgrouped assembly points form of CAMSA points.")
    parser.add_argument("--o-subgroups-uo-format", type=str,
                        help="The CAMSA-out formatting for the subgrouped unoriented assembly points in a form of CAMSA points.")
    parser.add_argument("--o-collapsed-format", type=str,
                        help="The CAMSA-out formatting for the collapsed assembly points and their computed conflicts.")
    parser.add_argument("--o-original-format", type=str,
                        help="The CAMSA-out formatting for the non-collapsed assembly points and their computed conflicts.")
    parser.add_argument("--c-logging-level", default=logging.INFO, type=int,
                        choices=[logging.NOTSET, logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL],
                        help="Logging level for CAMSA.\nDEFAULT: {info}".format(info=logging.INFO))
    parser.add_argument("--c-logging-formatter-entry",
                        help="Format string for python logger.")
    args = parser.parse_args()

    start_time = datetime.datetime.now()

    #######################################
    #           logging setup             #
    #######################################
    if args.o_dir is None:
        args.o_dir = os.path.join(os.getcwd(), "camsa_{date}".format(
            date=datetime.datetime.now().strftime("%b_%d_%Y__%H_%M")))

    args.output_dir = os.path.abspath(os.path.expanduser(args.o_dir))
    if not os.path.exists(args.output_dir):
        os.makedirs(args.output_dir)

    logger = logging.getLogger("CAMSA.main")
    ch = logging.StreamHandler()
    logger.setLevel(args.c_logging_level)
    logger.addHandler(ch)
    fh = logging.FileHandler(os.path.join(args.output_dir, "camsa.log"), "wt")
    fh.setLevel(logging.DEBUG)
    logger.addHandler(fh)
    logger.info(full_description)
    logger.info(parser.format_values())
    ch.setFormatter(logging.Formatter(args.c_logging_formatter_entry))
    fh.setFormatter(logging.Formatter(args.c_logging_formatter_entry))
    logger.info("Starting the analysis")

    #######################################
    #           input stage               #
    #######################################
    logger.info("Processing input")

    # key is the origin of assembly point; values is the list of all points from that source
    logger.info("Reading assembly points")
    assembly_points_by_sources = camsa_io.read_assembly_points_from_input_sources(sources=args.points,
                                                                                  default_cw_eae=args.c_cw_exact,
                                                                                  default_cw_cae=args.c_cw_candidate,
                                                                                  delimiter=args.i_delimiter)
    or_seqi = defaultdict(list)
    if args.seqi is not None:
        args.seqi = os.path.abspath(os.path.expanduser(args.seqi))
        logger.info("Reading sequences' info")
        with open(args.seqi, "rt") as source:
            camsa_io.read_seqi_from_input_sources(source=source, delimiter=args.seqi_delimiter, destination=or_seqi)
    seqi = {}
    for seq_name in list(or_seqi.keys()):
        entry_list = or_seqi[seq_name]
        seqi[seq_name] = entry_list[0]
    if len(seqi) > 0:
        sequences_ids_from_assembly_points = set()
        for source, aps in assembly_points_by_sources.items():
            for ap in aps:
                sequences_ids_from_assembly_points.add(ap.seq1)
                sequences_ids_from_assembly_points.add(ap.seq2)
        sequences_ids_from_lengths = set(seqi.keys())
        difference = sequences_ids_from_assembly_points - sequences_ids_from_lengths
        if len(difference) > 0:
            logger.warning("Some sequences, that participate in assembly points do not have lengths associated with them,"
                           "while others do.")
            logger.warning("Problematic sequences: {seqs}".format(seqs=",".join(sorted(difference))))
            if args.seqi_ensure:
                logger.error("A flag \"--seqi-ensure-all\" was set, so the program terminates")
            else:
                logger.warning("Assigning lengths of -1 to all problematic sequences")
                for seq_id in difference:
                    seqi[seq_id] = Sequence(name=seq_id, length=-1)

    #######################################
    #       extracting reference          #
    #######################################
    reference_assembly = None
    if args.reference and args.reference_name != "":
        try:
            reference_aps = assembly_points_by_sources.pop(args.reference_name)
            reference_assembly = Assembly(name=args.reference_name, aps=reference_aps)
        except KeyError:
            logger.critical("Supplied reference \"{reference_name}\" was not found among assembly sources [{avail_sources}]".format(reference_name=args.reference_name, avail_sources=",".join(assembly_points_by_sources.keys())))
            exit(1)

    id_generator = itertools.count()
    original_assembly_points = [or_ap for aps in assembly_points_by_sources.values() for or_ap in aps]
    # for assembly_points in assembly_points_by_sources.values():
    original_assembly_points_by_ids = assign_ids_to_assembly_points(assembly_points=original_assembly_points, id_prefix="or_",
                                                                    id_generator=id_generator, sort=True)

    #######################################
    #       assembly points merging       #
    #######################################
    logger.info("Merging assembly points from different sources into a set of unique ones.")

Answer:     merged_assembly_points = merge_assembly_points(assembly_points_by_source=assembly_points_by_sources)