gene_storage = {}<EOL>json_dic = {"<STR_LIT>": []}<EOL>logger.info("<STR_LIT>")<EOL>for key, entry in self.storage.items():<EOL><INDENT>logger.debug("<STR_LIT>".format(<EOL>entry["<STR_LIT>"]))<EOL>sample_id = re.match("<STR_LIT>", entry["<STR_LIT>"]).groups()[<NUM_LIT:0>]<EOL>database = entry["<STR_LIT>"]<EOL>if sample_id not in gene_storage:<EOL><INDENT>gene_storage[sample_id] = {}<EOL><DEDENT>if database not in gene_storage[sample_id]:<EOL><INDENT>gene_storage[sample_id][database] = []<EOL><DEDENT>gene_storage[sample_id][database].append(<EOL>entry["<STR_LIT>"].replace("<STR_LIT:'>", "<STR_LIT>").replace('<STR_LIT:">', '<STR_LIT>')<EOL>)<EOL><DEDENT>for sample, table_data in gene_storage.items():<EOL><INDENT>json_dic["<STR_LIT>"].append({<EOL>"<STR_LIT>": sample,<EOL>"<STR_LIT:data>": []<EOL>})<EOL>for db, gene_list in table_data.items():<EOL><INDENT>ind_json = {<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": db,<EOL>"<STR_LIT:value>": len(gene_list),<EOL>"<STR_LIT>": gene_list<EOL>}<EOL>json_dic["<STR_LIT>"][-<NUM_LIT:1>]["<STR_LIT:data>"].append(ind_json)<EOL><DEDENT><DEDENT>return json_dic<EOL>


json_plot = self.get_plot_data()<EOL>json_table = self.get_table_data()<EOL>json_dic = {**json_plot, **json_table}<EOL>with open("<STR_LIT>", "<STR_LIT:w>") as json_report:<EOL><INDENT>json_report.write(json.dumps(json_dic, separators=("<STR_LIT:U+002C>", "<STR_LIT::>")))<EOL><DEDENT>


logger.info("<STR_LIT>")<EOL>warnings = []<EOL>fails = "<STR_LIT>"<EOL>bowtie_info = Bowtie(sample_id, bowite_log)<EOL>print(bowtie_info.overall_rate)<EOL>with open("<STR_LIT>", "<STR_LIT:w>") as json_report:<EOL><INDENT>json_dic = {<EOL>"<STR_LIT>": [{<EOL>"<STR_LIT>": sample_id,<EOL>"<STR_LIT:data>": [<EOL>{"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:value>": int(bowtie_info.n_reads),<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": False},<EOL>{"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:value>": int(bowtie_info.align_0x),<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": False},<EOL>{"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:value>": int(bowtie_info.align_1x),<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": False},<EOL>{"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:value>": int(bowtie_info.align_mt1x),<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": False},<EOL>{"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:value>": float(bowtie_info.overall_rate),<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": False}<EOL>]<EOL>}],<EOL>}<EOL>if warnings:<EOL><INDENT>json_dic["<STR_LIT>"] = [{<EOL>"<STR_LIT>": sample_id,<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:value>": warnings<EOL>}]<EOL><DEDENT>if fails:<EOL><INDENT>json_dic["<STR_LIT>"] = [{<EOL>"<STR_LIT>": sample_id,<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:value>": [fails]<EOL>}]<EOL><DEDENT>json_report.write(json.dumps(json_dic, separators=("<STR_LIT:U+002C>", "<STR_LIT::>")))<EOL><DEDENT>with open("<STR_LIT>", "<STR_LIT:w>") as status_fh:<EOL><INDENT>status_fh.write("<STR_LIT>")<EOL><DEDENT>

Main executor of the process_mapping template.

    Parameters
    ----------
    sample_id : str
        Sample Identification string.
    boetie_log: str
        Path to the log file generated by bowtie.


print("<STR_LIT>")<EOL>regexes = {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': r"<STR_LIT>",<EOL>'<STR_LIT>': r"<STR_LIT>",<EOL>'<STR_LIT>': r"<STR_LIT>"<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': r"<STR_LIT>",<EOL>'<STR_LIT>': r"<STR_LIT>",<EOL>'<STR_LIT>': r"<STR_LIT>",<EOL>'<STR_LIT>': r"<STR_LIT>",<EOL>'<STR_LIT>': r"<STR_LIT>",<EOL>'<STR_LIT>': r"<STR_LIT>",<EOL>'<STR_LIT>': r"<STR_LIT>",<EOL>'<STR_LIT>': r"<STR_LIT>"<EOL>}<EOL>}<EOL>with open(bowtie_log, "<STR_LIT:r>") as f:<EOL><INDENT>for l in f:<EOL><INDENT>print(l)<EOL>total = re.search(r"<STR_LIT>", l)<EOL>print(total)<EOL>if total:<EOL><INDENT>print(total)<EOL>self.set_n_reads(total.group(<NUM_LIT:1>))<EOL><DEDENT>paired = re.search(r"<STR_LIT>", l)<EOL>if paired:<EOL><INDENT>paired_total = int(paired.group(<NUM_LIT:1>))<EOL>paired_numbers = {}<EOL>l = f.readline()<EOL>while l.startswith('<STR_LIT:U+0020>'):<EOL><INDENT>for k, r in regexes['<STR_LIT>'].items():<EOL><INDENT>match = re.search(r, l)<EOL>if match:<EOL><INDENT>paired_numbers[k] = int(match.group(<NUM_LIT:1>))<EOL><DEDENT><DEDENT>l = f.readline()<EOL><DEDENT>align_zero_times = paired_numbers['<STR_LIT>'] + paired_numbers['<STR_LIT>']<EOL>if align_zero_times:<EOL><INDENT>self.set_align_0x(align_zero_times)<EOL><DEDENT>align_one_time = paired_numbers['<STR_LIT>'] + paired_numbers['<STR_LIT>']<EOL>if align_one_time:<EOL><INDENT>self.set_align_1x(align_one_time)<EOL><DEDENT>align_more_than_one_time = paired_numbers['<STR_LIT>'] + paired_numbers['<STR_LIT>']<EOL>if align_more_than_one_time:<EOL><INDENT>self.set_align_mt1x(align_more_than_one_time)<EOL><DEDENT><DEDENT>overall = re.search(r"<STR_LIT>", l)<EOL>if overall:<EOL><INDENT>self.overall_rate = float(overall.group(<NUM_LIT:1>))<EOL><DEDENT><DEDENT><DEDENT>

Parse a bowtie log file.

        This is a bowtie log parsing method that populates the
        :py:attr:`self.n_reads, self.align_0x, self.align_1x, self.align_mt1x and self.overall_rate` attributes with
        data from the log file.

        Disclamer: THIS METHOD IS HORRIBLE BECAUSE THE BOWTIE LOG IS HORRIBLE.

        The insertion of data on the attribytes is done by the
        :py:meth:`set_attribute method.

        Parameters
        ----------
        bowtie_log : str
            Path to the boetie log file.


<EOL>depth_dic_coverage = {}<EOL>for line in depth_file:<EOL><INDENT>tab_split = line.split()  <EOL>reference = "<STR_LIT:_>".join(tab_split[<NUM_LIT:0>].strip().split("<STR_LIT:_>")[<NUM_LIT:0>:<NUM_LIT:3>])  <EOL>position = tab_split[<NUM_LIT:1>]<EOL>num_reads_align = float(tab_split[<NUM_LIT:2>].rstrip())<EOL>if reference not in depth_dic_coverage:<EOL><INDENT>depth_dic_coverage[reference] = {}<EOL><DEDENT>depth_dic_coverage[reference][position] = num_reads_align<EOL><DEDENT>logger.info("<STR_LIT>")<EOL>depth_file.close()<EOL>logger.debug("<STR_LIT>".format(<EOL>asizeof(depth_dic_coverage)/<NUM_LIT>))<EOL>return depth_dic_coverage<EOL>

Function that parse samtools depth file and creates 3 dictionaries that
will be useful to make the outputs of this script, both the tabular file
and the json file that may be imported by pATLAS

Parameters
----------
depth_file: textIO
    the path to depth file for each sample

Returns
-------
depth_dic_coverage: dict
        dictionary with the coverage per position for each plasmid


def generate_jsons(depth_dic_coverage, plasmid_length, cutoff):

<EOL>percentage_bases_covered = {}<EOL>dict_cov = {}<EOL>for ref in depth_dic_coverage:<EOL><INDENT>perc_value_per_ref = float(len(depth_dic_coverage[ref])) /float(plasmid_length[ref])<EOL>if perc_value_per_ref >= cutoff:<EOL><INDENT>percentage_bases_covered[ref] = round(perc_value_per_ref, <NUM_LIT:2>)<EOL>interval = round(int(plasmid_length[ref]) * <NUM_LIT>,<EOL>ndigits=<NUM_LIT:0>)<EOL>if interval < <NUM_LIT:1>:<EOL><INDENT>interval = <NUM_LIT:1><EOL><DEDENT>dict_cov[ref] = {<EOL>"<STR_LIT>": int(plasmid_length[ref]),<EOL>"<STR_LIT>": int(interval),<EOL>"<STR_LIT>": []<EOL>}<EOL>array_of_cov = []<EOL>reset_counter = <NUM_LIT:0><EOL>logger.info("<STR_LIT>".format(ref))<EOL>for i in range(int(plasmid_length[ref])):<EOL><INDENT>try:<EOL><INDENT>array_of_cov.append(int(depth_dic_coverage[ref][str(i)]))<EOL><DEDENT>except KeyError:<EOL><INDENT>array_of_cov.append(<NUM_LIT:0>)<EOL><DEDENT>if reset_counter == interval:<EOL><INDENT>dict_cov[ref]["<STR_LIT>"].append(<EOL>int(sum(array_of_cov)/len(array_of_cov))<EOL>)<EOL>reset_counter = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>reset_counter += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>logger.info("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>logger.debug("<STR_LIT>".format(<EOL>asizeof(percentage_bases_covered)/<NUM_LIT>))<EOL>logger.debug("<STR_LIT>".format(asizeof(dict_cov)/<NUM_LIT>))<EOL>return percentage_bases_covered, dict_cov<EOL>

Parameters
----------
depth_dic_coverage: dict
     dictionary with the coverage per position for each plasmid

Returns
-------
percentage_bases_covered: dict
dict_cov:  dict


@MainWrapper<EOL>def main(depth_file, json_dict, cutoff, sample_id):

<EOL>logger.debug("<STR_LIT>".format(cutoff, type(cutoff)))<EOL>try:<EOL><INDENT>cutoff_val = float(cutoff)<EOL>if cutoff_val < <NUM_LIT>:<EOL><INDENT>logger.warning("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT>except ValueError:<EOL><INDENT>logger.error("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(cutoff))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>plasmid_length = json.load(open(json_dict))<EOL>if plasmid_length:<EOL><INDENT>logger.info("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>logger.error("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>depth_file_in = open(depth_file)<EOL>logger.info("<STR_LIT>")<EOL>depth_dic_coverage = depth_file_reader(depth_file_in)<EOL>percentage_bases_covered, dict_cov = generate_jsons(depth_dic_coverage,<EOL>plasmid_length,<EOL>cutoff_val)<EOL>if percentage_bases_covered and dict_cov:<EOL><INDENT>logger.info("<STR_LIT>".format(<EOL>str(len(percentage_bases_covered))))<EOL>logger.info("<STR_LIT>".format(str(len(dict_cov))))<EOL><DEDENT>else:<EOL><INDENT>logger.error("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>logger.info("<STR_LIT>".format("<STR_LIT>".format(depth_file)))<EOL>with open("<STR_LIT>".format(depth_file), "<STR_LIT:w>") as output_json:<EOL><INDENT>output_json.write(json.dumps(percentage_bases_covered))<EOL><DEDENT>json_dic = {<EOL>"<STR_LIT>": [{<EOL>"<STR_LIT>": sample_id,<EOL>"<STR_LIT:data>": [{<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": percentage_bases_covered,<EOL>"<STR_LIT:value>": len(percentage_bases_covered)<EOL>}]<EOL>}],<EOL>"<STR_LIT>": sample_id,<EOL>"<STR_LIT>": percentage_bases_covered,<EOL>"<STR_LIT>": [{<EOL>"<STR_LIT>": sample_id,<EOL>"<STR_LIT:data>": {<EOL>"<STR_LIT>": dict_cov<EOL>},<EOL>}]<EOL>}<EOL>logger.debug("<STR_LIT>".format(asizeof(json_dic)/<NUM_LIT>))<EOL>logger.info("<STR_LIT>")<EOL>with open("<STR_LIT>", "<STR_LIT:w>") as json_report:<EOL><INDENT>json_report.write(json.dumps(json_dic, separators=("<STR_LIT:U+002C>", "<STR_LIT::>")))<EOL><DEDENT>

Function that handles the inputs required to parse depth files from bowtie
and dumps a dict to a json file that can be imported into pATLAS.

Parameters
----------
depth_file: str
     the path to depth file for each sample
json_dict: str
    the file that contains the dictionary with keys and values for
    accessions
    and their respective lengths
cutoff: str
    the cutoff used to trim the unwanted matches for the minimum coverage
    results from mapping. This value may range between 0 and 1.
sample_id: str
    the id of the sample being parsed


if not magic_dict:<EOL><INDENT>magic_dict = MAGIC_DICT<EOL><DEDENT>max_len = max(len(x) for x in magic_dict)<EOL>with open(file_path, "<STR_LIT:rb>") as f:<EOL><INDENT>file_start = f.read(max_len)<EOL><DEDENT>logger.debug("<STR_LIT>".format(file_start))<EOL>for magic, file_type in magic_dict.items():<EOL><INDENT>if file_start.startswith(magic):<EOL><INDENT>return file_type<EOL><DEDENT><DEDENT>return None<EOL>

Guesses the compression of an input file.

    This function guesses the compression of a given file by checking for
    a binary signature at the beginning of the file. These signatures are
    stored in the :py:data:`MAGIC_DICT` dictionary. The supported compression
    formats are gzip, bzip2 and zip. If none of the signatures in this
    dictionary are found at the beginning of the file, it returns ``None``.

    Parameters
    ----------
    file_path : str
        Path to input file.
    magic_dict : dict, optional
        Dictionary containing the signatures of the compression types. The
        key should be the binary signature and the value should be the
        compression format. If left ``None``, it falls back to
        :py:data:`MAGIC_DICT`.

    Returns
    -------
    file_type : str or None
        If a compression type is detected, returns a string with the format.
        If not, returns ``None``.


vals = [ord(c) for c in qual_str]<EOL>return min(vals), max(vals)<EOL>

Get range of the Unicode encode range for a given string of characters.

    The encoding is determined from the result of the :py:func:`ord` built-in.

    Parameters
    ----------
    qual_str : str
        Arbitrary string.

    Returns
    -------
    x : tuple
        (Minimum Unicode code, Maximum Unicode code).


valid_encodings = []<EOL>valid_phred = []<EOL>for encoding, (phred, (emin, emax)) in RANGES.items():<EOL><INDENT>if rmin >= emin and rmax <= emax:<EOL><INDENT>valid_encodings.append(encoding)<EOL>valid_phred.append(phred)<EOL><DEDENT><DEDENT>return valid_encodings, valid_phred<EOL>

Returns the valid encodings for a given encoding range.

    The encoding ranges are stored in the :py:data:`RANGES` dictionary, with
    the encoding name as a string and a list as a value containing the
    phred score and a tuple with the encoding range. For a given encoding
    range provided via the two first arguments, this function will return
    all possible encodings and phred scores.

    Parameters
    ----------
    rmin : int
        Minimum Unicode code in range.
    rmax : int
        Maximum Unicode code in range.

    Returns
    -------
    valid_encodings : list
        List of all possible encodings for the provided range.
    valid_phred : list
        List of all possible phred scores.


@MainWrapper<EOL>def main(sample_id, fastq_pair, gsize, minimum_coverage, opts):

logger.info("<STR_LIT>")<EOL>if "<STR_LIT>" in opts:<EOL><INDENT>skip_encoding = True<EOL><DEDENT>else:<EOL><INDENT>skip_encoding = False<EOL><DEDENT>gmin, gmax = <NUM_LIT>, <NUM_LIT:0><EOL>encoding = []<EOL>phred = None<EOL>chars = <NUM_LIT:0><EOL>nreads = <NUM_LIT:0><EOL>max_read_length = <NUM_LIT:0><EOL>file_objects = []<EOL>for fastq in fastq_pair:<EOL><INDENT>logger.info("<STR_LIT>".format(fastq))<EOL>logger.info("<STR_LIT>".format(fastq))<EOL>ftype = guess_file_compression(fastq)<EOL>if ftype:<EOL><INDENT>logger.info("<STR_LIT>".format(<EOL>fastq, ftype))<EOL>file_objects.append(COPEN[ftype](fastq, "<STR_LIT>"))<EOL><DEDENT>else:<EOL><INDENT>logger.info("<STR_LIT>"<EOL>"<STR_LIT>".format(fastq))<EOL>file_objects.append(open(fastq))<EOL><DEDENT><DEDENT>logger.info("<STR_LIT>")<EOL>with open("<STR_LIT>".format(sample_id), "<STR_LIT:w>") as enc_fh,open("<STR_LIT>".format(sample_id), "<STR_LIT:w>") as phred_fh,open("<STR_LIT>".format(sample_id), "<STR_LIT:w>") as cov_fh,open("<STR_LIT>".format(sample_id), "<STR_LIT:w>") as cov_rep,open("<STR_LIT>".format(sample_id), "<STR_LIT:w>") as len_fh,open("<STR_LIT>", "<STR_LIT:w>") as json_report,open("<STR_LIT>", "<STR_LIT:w>") as status_fh,open("<STR_LIT>", "<STR_LIT:w>") as fail_fh:<EOL><INDENT>try:<EOL><INDENT>for i, line in enumerate(chain(*file_objects)):<EOL><INDENT>if (i + <NUM_LIT:1>) % <NUM_LIT:4> == <NUM_LIT:0> and not skip_encoding:<EOL><INDENT>lmin, lmax = get_qual_range(line.strip())<EOL>if lmin < gmin or lmax > gmax:<EOL><INDENT>gmin, gmax = min(lmin, gmin), max(lmax, gmax)<EOL>encoding, phred = get_encodings_in_range(gmin, gmax)<EOL>logger.debug(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(<EOL>i, [lmin, lmax], encoding, phred))<EOL><DEDENT><DEDENT>if (i + <NUM_LIT:3>) % <NUM_LIT:4> == <NUM_LIT:0>:<EOL><INDENT>read_len = len(line.strip())<EOL>chars += read_len<EOL>nreads += <NUM_LIT:1><EOL>if read_len > max_read_length:<EOL><INDENT>logger.debug("<STR_LIT>"<EOL>"<STR_LIT>".format(i, read_len))<EOL>max_read_length = read_len<EOL><DEDENT><DEDENT><DEDENT>logger.info("<STR_LIT>")<EOL>exp_coverage = round(chars / (gsize * <NUM_LIT>), <NUM_LIT:2>)<EOL>if "<STR_LIT>" not in opts:<EOL><INDENT>json_dic = {<EOL>"<STR_LIT>": [{<EOL>"<STR_LIT>": sample_id,<EOL>"<STR_LIT:data>": [<EOL>{"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:value>": chars,<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": True},<EOL>{"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:value>": nreads,<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": True},<EOL>{"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:value>": exp_coverage,<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT>": minimum_coverage<EOL>}<EOL>]<EOL>}],<EOL>"<STR_LIT>": [{<EOL>"<STR_LIT>": sample_id,<EOL>"<STR_LIT:data>": {<EOL>"<STR_LIT>": chars<EOL>}<EOL>}],<EOL>}<EOL><DEDENT>else:<EOL><INDENT>json_dic = {<EOL>"<STR_LIT>": [{<EOL>"<STR_LIT>": sample_id,<EOL>"<STR_LIT:data>": [<EOL>{"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:value>": exp_coverage,<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT>": minimum_coverage<EOL>}<EOL>],<EOL>}],<EOL>}<EOL><DEDENT>if len(encoding) > <NUM_LIT:0>:<EOL><INDENT>encoding = set(encoding)<EOL>phred = set(phred)<EOL>enc = "<STR_LIT:{}>".format("<STR_LIT:U+002C>".join([x for x in encoding]))<EOL>phred = "<STR_LIT:{}>".format("<STR_LIT:U+002C>".join(str(x) for x in phred))<EOL>logger.info("<STR_LIT>".format(enc))<EOL>logger.info("<STR_LIT>".format(enc))<EOL>enc_fh.write(enc)<EOL>phred_fh.write(phred)<EOL><DEDENT>else:<EOL><INDENT>if not skip_encoding:<EOL><INDENT>encoding_msg = "<STR_LIT>""<STR_LIT>"<EOL>logger.warning(encoding_msg)<EOL>json_dic["<STR_LIT>"] = [{<EOL>"<STR_LIT>": sample_id,<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:value>": [encoding_msg]<EOL>}]<EOL>enc_fh.write("<STR_LIT:None>")<EOL>phred_fh.write("<STR_LIT:None>")<EOL><DEDENT><DEDENT>logger.info("<STR_LIT>"<EOL>"<STR_LIT:{}>".format(gsize))<EOL>logger.info("<STR_LIT>".format(exp_coverage))<EOL>if exp_coverage >= minimum_coverage:<EOL><INDENT>cov_rep.write("<STR_LIT>".format(<EOL>sample_id, str(exp_coverage), "<STR_LIT>"))<EOL>cov_fh.write(str(exp_coverage))<EOL>status_fh.write("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>fail_msg = "<STR_LIT>""<STR_LIT>".format(exp_coverage, minimum_coverage)<EOL>logger.error(fail_msg)<EOL>fail_fh.write(fail_msg)<EOL>cov_fh.write("<STR_LIT>")<EOL>status_fh.write("<STR_LIT>")<EOL>cov_rep.write("<STR_LIT>".format(<EOL>sample_id, str(exp_coverage), "<STR_LIT>"))<EOL>json_dic["<STR_LIT>"] = [{<EOL>"<STR_LIT>": sample_id,<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT:value>": [fail_msg]<EOL>}]<EOL><DEDENT>json_report.write(json.dumps(json_dic, separators=("<STR_LIT:U+002C>", "<STR_LIT::>")))<EOL>len_fh.write("<STR_LIT:{}>".format(max_read_length))<EOL><DEDENT>except EOFError:<EOL><INDENT>logger.error("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>for fh in [enc_fh, phred_fh, cov_fh, cov_rep, len_fh]:<EOL><INDENT>fh.write("<STR_LIT>")<EOL>status_fh.write("<STR_LIT>")<EOL>fail_fh.write("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>

Main executor of the integrity_coverage template.

    Parameters
    ----------
    sample_id : str
        Sample Identification string.
    fastq_pair : list
        Two element list containing the paired FastQ files.
    gsize : float or int
        Estimate of genome size in Mb.
    minimum_coverage : float or int
        Minimum coverage required for a sample to pass the coverage check
    opts : list
        List of arbitrary options. See `Expected input`_.


with open("<STR_LIT>", "<STR_LIT:w>") as status_fh:<EOL><INDENT>status_fh.write("<STR_LIT:error>")<EOL><DEDENT>

Nextflow specific function that logs an error upon unexpected failing


version_storage = []<EOL>template_version = self.context.get("<STR_LIT>", None)<EOL>template_program = self.context.get("<STR_LIT>", None)<EOL>template_build = self.context.get("<STR_LIT>", None)<EOL>if template_version and template_program and template_build:<EOL><INDENT>if self.logger:<EOL><INDENT>self.logger.debug("<STR_LIT>"<EOL>"<STR_LIT:{}>".format(template_program,<EOL>template_version,<EOL>template_build))<EOL><DEDENT>version_storage.append({<EOL>"<STR_LIT>": template_program,<EOL>"<STR_LIT:version>": template_version,<EOL>"<STR_LIT>": template_build<EOL>})<EOL><DEDENT>for var, obj in self.context.items():<EOL><INDENT>if var.startswith("<STR_LIT>"):<EOL><INDENT>ver = obj()<EOL>version_storage.append(ver)<EOL>if self.logger:<EOL><INDENT>self.logger.debug("<STR_LIT>"<EOL>"<STR_LIT:{}>".format(ver))<EOL><DEDENT><DEDENT><DEDENT>with open("<STR_LIT>", "<STR_LIT:w>") as fh:<EOL><INDENT>fh.write(json.dumps(version_storage, separators=("<STR_LIT:U+002C>", "<STR_LIT::>")))<EOL><DEDENT>

Writes versions JSON for a template file

        This method creates the JSON file ``.versions`` based on the metadata
        and specific functions that are present in a given template script.

        It starts by fetching the template metadata, which can be specified
        via the ``__version__``, ``__template__`` and ``__build__``
        attributes. If all of these attributes exist, it starts to populate
        a JSON/dict array (Note that the absence of any one of them will
        prevent the version from being written).

        Then, it will search the
        template scope for functions that start with the substring
        ``__set_version`` (For example ``def __set_version_fastqc()`).
        These functions should gather the version of
        an arbitrary program and return a JSON/dict object with the following
        information::

            {
                "program": <program_name>,
                "version": <version>
                "build": <build>
            }

        This JSON/dict object is then written in the ``.versions`` file.


for fq in fastq:<EOL><INDENT>rp = os.path.realpath(fq)<EOL>logger.debug("<STR_LIT>".format(rp))<EOL>if re.match("<STR_LIT>", rp):<EOL><INDENT>os.remove(rp)<EOL><DEDENT><DEDENT>

Cleans the temporary fastq files. If they are symlinks, the link
source is removed

Parameters
----------
fastq : list
    List of fastq files.


@MainWrapper<EOL>def main(sample_id, fastq_pair, clear):

logger.info("<STR_LIT>")<EOL>if "<STR_LIT>" in fastq_pair[<NUM_LIT:0>]:<EOL><INDENT>sample_id += "<STR_LIT>"<EOL><DEDENT>version = __get_version_skesa()["<STR_LIT:version>"]<EOL>output_file = "<STR_LIT>".format(sample_id, version.replace("<STR_LIT:.>", "<STR_LIT>"))<EOL>cli = [<EOL>"<STR_LIT>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT>".format(fastq_pair[<NUM_LIT:0>], fastq_pair[<NUM_LIT:1>]),<EOL>"<STR_LIT>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT>"<EOL>]<EOL>logger.debug("<STR_LIT>".format(cli))<EOL>with open(output_file, "<STR_LIT:w>") as fh:<EOL><INDENT>p = subprocess.Popen(cli, stdout=fh, stderr=PIPE)<EOL><DEDENT>stdout, stderr = p.communicate()<EOL>try:<EOL><INDENT>stderr = stderr.decode("<STR_LIT:utf8>")<EOL>stdout = stdout.decode("<STR_LIT:utf8>")<EOL><DEDENT>except (UnicodeDecodeError, AttributeError):<EOL><INDENT>stderr = str(stderr)<EOL>stdout = str(stdout)<EOL><DEDENT>logger.info("<STR_LIT>"<EOL>"<STR_LIT>".format(stdout))<EOL>logger.info("<STR_LIT>"<EOL>"<STR_LIT>".format(stderr))<EOL>logger.info("<STR_LIT>".format(<EOL>p.returncode))<EOL>if clear == "<STR_LIT:true>" and os.path.exists(output_file):<EOL><INDENT>clean_up(fastq_pair)<EOL><DEDENT>with open("<STR_LIT>", "<STR_LIT:w>") as fh:<EOL><INDENT>if p.returncode != <NUM_LIT:0>:<EOL><INDENT>fh.write("<STR_LIT:error>")<EOL>raise SystemExit(p.returncode)<EOL><DEDENT>else:<EOL><INDENT>fh.write("<STR_LIT>")<EOL><DEDENT><DEDENT>

Main executor of the skesa template.

    Parameters
    ----------
    sample_id : str
        Sample Identification string.
    fastq_pair : list
        Two element list containing the paired FastQ files.
    clear : str
        Can be either 'true' or 'false'. If 'true', the input fastq files will
        be removed at the end of the run, IF they are in the working directory


logger.info("<STR_LIT>")<EOL>print(newick)<EOL>tree = dendropy.Tree.get(file=open(newick, '<STR_LIT:r>'), schema="<STR_LIT>")<EOL>tree.reroot_at_midpoint()<EOL>to_write=tree.as_string("<STR_LIT>").strip().replace("<STR_LIT>", '<STR_LIT>').replace('<STR_LIT:U+0020>', '<STR_LIT:_>').replace("<STR_LIT:'>", "<STR_LIT>")<EOL>with open("<STR_LIT>", "<STR_LIT:w>") as json_report:<EOL><INDENT>json_dic = {<EOL>"<STR_LIT>": [{<EOL>"<STR_LIT>": [<EOL>to_write<EOL>]<EOL>}],<EOL>}<EOL>json_report.write(json.dumps(json_dic, separators=("<STR_LIT:U+002C>", "<STR_LIT::>")))<EOL><DEDENT>with open("<STR_LIT>", "<STR_LIT:w>") as status_fh:<EOL><INDENT>status_fh.write("<STR_LIT>")<EOL><DEDENT>

Main executor of the process_newick template.

    Parameters
    ----------
    newick : str
        path to the newick file.


directives = None<EOL>fields = name_str.split("<STR_LIT:=>")<EOL>process_name = fields[<NUM_LIT:0>]<EOL>if len(fields) == <NUM_LIT:2>:<EOL><INDENT>_directives = fields[<NUM_LIT:1>].replace("<STR_LIT:'>", '<STR_LIT:">')<EOL>try:<EOL><INDENT>directives = json.loads(_directives)<EOL><DEDENT>except json.decoder.JSONDecodeError:<EOL><INDENT>raise eh.ProcessError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(<EOL>process_name, name_str))<EOL><DEDENT><DEDENT>return process_name, directives<EOL>

Parses the process string and returns the process name and its
        directives

        Process strings my contain directive information with the following
        syntax::

            proc_name={'directive':'val'}

        This method parses this string and returns the process name as a
        string and the directives information as a dictionary.

        Parameters
        ----------
        name_str : str
            Raw string with process name and, potentially, directive
            information

        Returns
        -------
        str
            Process name
        dict or None
            Process directives


def _build_connections(self, process_list, ignore_dependencies,<EOL>auto_dependency):

logger.debug("<STR_LIT>")<EOL>logger.debug("<STR_LIT>")<EOL>logger.debug("<STR_LIT>")<EOL>logger.debug("<STR_LIT>".format(process_list))<EOL>for p, con in enumerate(process_list):<EOL><INDENT>logger.debug("<STR_LIT>".format(p, con))<EOL>in_lane = con["<STR_LIT:input>"]["<STR_LIT>"]<EOL>out_lane = con["<STR_LIT>"]["<STR_LIT>"]<EOL>logger.debug("<STR_LIT>".format(p, in_lane))<EOL>logger.debug("<STR_LIT>".format(p, out_lane))<EOL>if out_lane > self.lanes:<EOL><INDENT>self.lanes = out_lane<EOL><DEDENT>p_in_name, p_out_name, out_directives = self._get_process_names(<EOL>con, p)<EOL>if p_out_name not in self.process_map:<EOL><INDENT>logger.error(colored_print(<EOL>"<STR_LIT>"<EOL>.format(p_out_name), "<STR_LIT>"))<EOL>guess_process(p_out_name, self.process_map)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>out_process = self.process_map[p_out_name](template=p_out_name)<EOL>if out_directives:<EOL><INDENT>out_process.update_attributes(out_directives)<EOL><DEDENT>input_suf = "<STR_LIT>".format(in_lane, p)<EOL>output_suf = "<STR_LIT>".format(out_lane, p)<EOL>logger.debug("<STR_LIT>"<EOL>"<STR_LIT>".format(<EOL>p, input_suf, output_suf))<EOL>out_process.set_main_channel_names(input_suf, output_suf, out_lane)<EOL>if p_in_name != "<STR_LIT>":<EOL><INDENT>in_process = self.process_map[p_in_name](template=p_in_name)<EOL>logger.debug("<STR_LIT>"<EOL>"<STR_LIT>".format(p))<EOL>self._test_connection(in_process, out_process)<EOL>out_process.parent_lane = in_lane<EOL><DEDENT>else:<EOL><INDENT>out_process.parent_lane = None<EOL><DEDENT>logger.debug("<STR_LIT>".format(<EOL>p, out_process.parent_lane))<EOL>if in_lane != out_lane:<EOL><INDENT>logger.debug("<STR_LIT>"<EOL>"<STR_LIT>".format(p))<EOL>self._fork_tree[in_lane].append(out_lane)<EOL>try:<EOL><INDENT>parent_process = [<EOL>x for x in self.processes if x.lane == in_lane and<EOL>x.template == p_in_name<EOL>][<NUM_LIT:0>]<EOL>logger.debug(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(p, parent_process,<EOL>out_process.input_channel))<EOL>parent_process.update_main_forks(out_process.input_channel)<EOL><DEDENT>except IndexError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>else:<EOL><INDENT>parent_process = self.processes[-<NUM_LIT:1>]<EOL>if parent_process.lane and parent_process.lane != out_lane:<EOL><INDENT>parent_process = [x for x in self.processes[::-<NUM_LIT:1>]<EOL>if x.lane == out_lane][<NUM_LIT:0>]<EOL><DEDENT>if parent_process.output_channel:<EOL><INDENT>logger.debug(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(<EOL>p, parent_process.output_channel))<EOL>out_process.input_channel = parent_process.output_channel<EOL><DEDENT><DEDENT>if out_process.dependencies and not ignore_dependencies:<EOL><INDENT>logger.debug("<STR_LIT>"<EOL>"<STR_LIT:{}>".format(p, p_out_name,<EOL>out_process.dependencies))<EOL>parent_lanes = self._get_fork_tree(out_lane)<EOL>for dep in out_process.dependencies:<EOL><INDENT>if not self._search_tree_backwards(dep, parent_lanes):<EOL><INDENT>if auto_dependency:<EOL><INDENT>self._add_dependency(<EOL>out_process, dep, in_lane, out_lane, p)<EOL><DEDENT>elif not self.export_parameters:<EOL><INDENT>logger.error(colored_print(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(p_out_name, dep),<EOL>"<STR_LIT>"))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>self.processes.append(out_process)<EOL><DEDENT>logger.debug("<STR_LIT>".format(self.processes))<EOL>logger.debug("<STR_LIT>".format(self._fork_tree))<EOL>

Parses the process connections dictionaries into a process list

        This method is called upon instantiation of the NextflowGenerator
        class. Essentially, it sets the main input/output channel names of the
        processes so that they can be linked correctly.

        If a connection between two consecutive process is not possible due
        to a mismatch in the input/output types, it exits with an error.

        Returns
        -------


try:<EOL><INDENT>_p_in_name = con["<STR_LIT:input>"]["<STR_LIT>"]<EOL>p_in_name, _ = self._parse_process_name(_p_in_name)<EOL>logger.debug("<STR_LIT>".format(pid, p_in_name))<EOL>_p_out_name = con["<STR_LIT>"]["<STR_LIT>"]<EOL>p_out_name, out_directives = self._parse_process_name(<EOL>_p_out_name)<EOL>logger.debug("<STR_LIT>".format(pid, p_out_name))<EOL><DEDENT>except eh.ProcessError as ex:<EOL><INDENT>logger.error(colored_print(ex.value, "<STR_LIT>"))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>return p_in_name, p_out_name, out_directives<EOL>

Returns the input/output process names and output process directives

        Parameters
        ----------
        con : dict
            Dictionary with the connection information between two processes.

        Returns
        -------
        input_name : str
            Name of the input process
        output_name : str
            Name of the output process
        output_directives : dict
            Parsed directives from the output process


def _add_dependency(self, p, template, inlane, outlane, pid):

dependency_proc = self.process_map[template](template=template)<EOL>if dependency_proc.input_type != p.input_type:<EOL><INDENT>logger.error("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(<EOL>p.template, p.input_type, template,<EOL>dependency_proc.input_type))<EOL><DEDENT>input_suf = "<STR_LIT>".format(inlane, pid)<EOL>output_suf = "<STR_LIT>".format(outlane, pid)<EOL>dependency_proc.set_main_channel_names(input_suf, output_suf, outlane)<EOL>dependency_proc.input_channel = p.input_channel<EOL>p.input_channel = dependency_proc.output_channel<EOL>if not p.parent_lane:<EOL><INDENT>p.parent_lane = outlane<EOL>dependency_proc.parent_lane = None<EOL><DEDENT>else:<EOL><INDENT>dependency_proc.parent_lane = inlane<EOL>p.parent_lane = outlane<EOL><DEDENT>self.processes.append(dependency_proc)<EOL>

Automatically Adds a dependency of a process.

        This method adds a template to the process list attribute as a
        dependency. It will adapt the input lane, output lane and process
        id of the process that depends on it.

        Parameters
        ----------
        p : Process
            Process class that contains the dependency.
        template : str
            Template name of the dependency.
        inlane : int
            Input lane.
        outlane : int
            Output lane.
        pid : int
            Process ID.


def _search_tree_backwards(self, template, parent_lanes):

for p in self.processes[::-<NUM_LIT:1>]:<EOL><INDENT>if p.lane not in parent_lanes:<EOL><INDENT>continue<EOL><DEDENT>if p.template == template:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

Searches the process tree backwards in search of a provided process

        The search takes into consideration the provided parent lanes and
        searches only those

        Parameters
        ----------
        template : str
            Name of the process template attribute being searched
        parent_lanes : list
            List of integers with the parent lanes to be searched

        Returns
        -------
        bool
            Returns True when the template is found. Otherwise returns False.


logger.debug("<STR_LIT>")<EOL>logger.debug("<STR_LIT>")<EOL>logger.debug("<STR_LIT>")<EOL>self.template += hs.header<EOL>

Adds the header template to the master template string


logger.debug("<STR_LIT>")<EOL>logger.debug("<STR_LIT>")<EOL>logger.debug("<STR_LIT>")<EOL>self.template += fs.footer<EOL>

Adds the footer template to the master template string


def _update_raw_input(self, p, sink_channel=None, input_type=None):

process_input = input_type if input_type else p.input_type<EOL>process_channel = sink_channel if sink_channel else p.input_channel<EOL>logger.debug("<STR_LIT>"<EOL>"<STR_LIT>".format(p.template, process_input))<EOL>raw_in = p.get_user_channel(process_channel, process_input)<EOL>logger.debug("<STR_LIT>".format(p.template,<EOL>raw_in))<EOL>if process_input in self.main_raw_inputs:<EOL><INDENT>self.main_raw_inputs[process_input]["<STR_LIT>"].append(<EOL>raw_in["<STR_LIT>"])<EOL><DEDENT>else:<EOL><INDENT>self.main_raw_inputs[process_input] = {<EOL>"<STR_LIT>": raw_in["<STR_LIT>"],<EOL>"<STR_LIT>": "<STR_LIT>".format(<EOL>raw_in["<STR_LIT>"].format(raw_in["<STR_LIT>"]),<EOL>raw_in["<STR_LIT>"],<EOL>raw_in["<STR_LIT>"].format(raw_in["<STR_LIT>"])),<EOL>"<STR_LIT>": [raw_in["<STR_LIT>"]]<EOL>}<EOL><DEDENT>logger.debug("<STR_LIT>".format(<EOL>p.template, self.main_raw_inputs))<EOL>

Given a process, this method updates the
        :attr:`~Process.main_raw_inputs` attribute with the corresponding
        raw input channel of that process. The input channel and input type
        can be overridden if the `input_channel` and `input_type` arguments
        are provided.

        Parameters
        ----------
        p : flowcraft.Process.Process
            Process instance whose raw input will be modified
        sink_channel: str
            Sets the channel where the raw input will fork into. It overrides
            the process's `input_channel` attribute.
        input_type: str
            Sets the type of the raw input. It overrides the process's
            `input_type` attribute.


if p.extra_input:<EOL><INDENT>logger.debug("<STR_LIT>".format(<EOL>p.template, p.extra_input))<EOL>if p.extra_input == "<STR_LIT:default>":<EOL><INDENT>if p.input_type in self.main_raw_inputs:<EOL><INDENT>logger.error(colored_print(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(p.input_type, p.template), "<STR_LIT>"))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>param = p.input_type<EOL><DEDENT>else:<EOL><INDENT>param = p.extra_input<EOL><DEDENT>dest_channel = "<STR_LIT>".format(p.template, p.pid)<EOL>if param not in self.extra_inputs:<EOL><INDENT>self.extra_inputs[param] = {<EOL>"<STR_LIT>": p.input_type,<EOL>"<STR_LIT>": [dest_channel]<EOL>}<EOL><DEDENT>else:<EOL><INDENT>if self.extra_inputs[param]["<STR_LIT>"] != p.input_type:<EOL><INDENT>logger.error(colored_print(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(<EOL>p.input_type, p.template,<EOL>self.extra_inputs[param]["<STR_LIT>"]),<EOL>"<STR_LIT>"))<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>self.extra_inputs[param]["<STR_LIT>"].append(dest_channel)<EOL><DEDENT>logger.debug("<STR_LIT>"<EOL>"<STR_LIT>".format(p.template, param,<EOL>self.extra_inputs[param]))<EOL>p.update_main_input(<EOL>"<STR_LIT>".format(p.input_channel, dest_channel)<EOL>)<EOL><DEDENT>

Given a process, this method updates the
        :attr:`~Process.extra_inputs` attribute with the corresponding extra
        inputs of that process

        Parameters
        ----------
        p : flowcraft.Process.Process


parent_lanes = [lane]<EOL>while True:<EOL><INDENT>original_lane = lane<EOL>for fork_in, fork_out in self._fork_tree.items():<EOL><INDENT>if lane in fork_out:<EOL><INDENT>lane = fork_in<EOL>parent_lanes.append(fork_in)<EOL><DEDENT><DEDENT>if lane == original_lane:<EOL><INDENT>break<EOL><DEDENT><DEDENT>return parent_lanes<EOL>

Parameters
----------
p

Returns
-------


output_type = link["<STR_LIT>"].lstrip("<STR_LIT:_>")<EOL>parent_forks = self._get_fork_tree(p.lane)<EOL>fork_sink = "<STR_LIT>".format(link["<STR_LIT>"], p.pid)<EOL>for proc in self.processes[::-<NUM_LIT:1>]:<EOL><INDENT>if proc.lane not in parent_forks:<EOL><INDENT>continue<EOL><DEDENT>if proc.output_type == output_type:<EOL><INDENT>proc.update_main_forks(fork_sink)<EOL>logger.debug("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT:{}>".format(<EOL>p.template, link["<STR_LIT>"], output_type,<EOL>link["<STR_LIT>"], proc))<EOL>return<EOL><DEDENT><DEDENT>self._update_raw_input(p, fork_sink, output_type)<EOL>

Parameters
----------
p
link

Returns
-------


<EOL>if p.link_start:<EOL><INDENT>logger.debug("<STR_LIT>".format(<EOL>p.template, p.link_start))<EOL>for l in p.link_start:<EOL><INDENT>if l in self.secondary_channels:<EOL><INDENT>self.secondary_channels[l][p.lane] = {"<STR_LIT:p>": p, "<STR_LIT:end>": []}<EOL><DEDENT>else:<EOL><INDENT>self.secondary_channels[l] = {p.lane: {"<STR_LIT:p>": p, "<STR_LIT:end>": []}}<EOL><DEDENT><DEDENT><DEDENT>if p.link_end:<EOL><INDENT>logger.debug("<STR_LIT>".format(<EOL>p.template, p.link_end))<EOL>for l in p.link_end:<EOL><INDENT>parent_forks = self._get_fork_tree(p.lane)<EOL>if l["<STR_LIT>"].startswith("<STR_LIT>"):<EOL><INDENT>self._set_implicit_link(p, l)<EOL>continue<EOL><DEDENT>if l["<STR_LIT>"] not in self.secondary_channels:<EOL><INDENT>continue<EOL><DEDENT>for lane in parent_forks:<EOL><INDENT>if lane in self.secondary_channels[l["<STR_LIT>"]]:<EOL><INDENT>self.secondary_channels[<EOL>l["<STR_LIT>"]][lane]["<STR_LIT:end>"].append("<STR_LIT:{}>".format(<EOL>"<STR_LIT>".format(l["<STR_LIT>"], p.pid)))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>logger.debug("<STR_LIT>".format(<EOL>p.template, self.secondary_channels))<EOL>

Given a process, this method updates the
        :attr:`~Process.secondary_channels` attribute with the corresponding
        secondary inputs of that channel.

        The rationale of the secondary channels is the following:

            - Start storing any secondary emitting channels, by checking the
              `link_start` list attribute of each process. If there are
              channel names in the link start, it adds to the secondary
              channels dictionary.
            - Check for secondary receiving channels, by checking the
              `link_end` list attribute. If the link name starts with a
              `__` signature, it will created an implicit link with the last
              process with an output type after the signature. Otherwise,
              it will check is a corresponding link start already exists in
              the at least one process upstream of the pipeline and if so,
              it will update the ``secondary_channels`` attribute with the
              new link.

        Parameters
        ----------
        p : flowcraft.Process.Process


logger.debug("<STR_LIT>")<EOL>logger.debug("<STR_LIT>")<EOL>logger.debug("<STR_LIT>")<EOL>for i, p in enumerate(self.processes):<EOL><INDENT>logger.debug("<STR_LIT>".format(<EOL>p.template, i))<EOL>p.set_channels(pid=i)<EOL>logger.debug("<STR_LIT>".format(p.parent_lane, p.input_type, p.template))<EOL>if not p.parent_lane and p.input_type:<EOL><INDENT>self._update_raw_input(p)<EOL><DEDENT>self._update_extra_inputs(p)<EOL>self._update_secondary_channels(p)<EOL>logger.info(colored_print(<EOL>"<STR_LIT>".format(p.template)))<EOL><DEDENT>

Sets the main channels for the pipeline

        This method will parse de the :attr:`~Process.processes` attribute
        and perform the following tasks for each process:

            - Sets the input/output channels and main input forks and adds
              them to the process's
              :attr:`flowcraft.process.Process._context`
              attribute (See
              :func:`~NextflowGenerator.set_channels`).
            - Automatically updates the main input channel of the first
              process of each lane so that they fork from the user provide
              parameters (See
              :func:`~NextflowGenerator._update_raw_input`).
            - Check for the presence of secondary channels and adds them to the
              :attr:`~NextflowGenerator.secondary_channels` attribute.

        Notes
        -----
        **On the secondary channel setup**: With this approach, there can only
        be one secondary link start for each type of secondary link. For
        instance, If there are two processes that start a secondary channel
        for the ``SIDE_max_len`` channel, only the last one will be recorded,
        and all receiving processes will get the channel from the latest
        process. Secondary channels can only link if the source process if
        downstream of the sink process in its "forking" path.


Sets the main raw inputs and secondary inputs on the init process

        This method will fetch the :class:`flowcraft.process.Init` process
        instance and sets the raw input (
        :func:`flowcraft.process.Init.set_raw_inputs`) for
        that process. This will handle the connection of the user parameters
        with channels that are then consumed in the pipeline.


Sets the secondary channels for the pipeline

        This will iterate over the
        :py:attr:`NextflowGenerator.secondary_channels` dictionary that is
        populated when executing
        :func:`~NextflowGenerator._update_secondary_channels` method.


self._set_status_channels()<EOL>self._set_general_compilers()<EOL>

Wrapper method that calls functions related to compiler channels


for c, c_info in self.compilers.items():<EOL><INDENT>compiler_cls = c_info["<STR_LIT>"](template=c_info["<STR_LIT>"])<EOL>c_info["<STR_LIT>"] = []<EOL>for p in self.processes:<EOL><INDENT>if not any([isinstance(p, x) for x in self.skip_class]):<EOL><INDENT>if c in p.compiler:<EOL><INDENT>channels = ["<STR_LIT>".format(i, p.pid) for i in<EOL>p.compiler[c]]<EOL>c_info["<STR_LIT>"].extend(channels)<EOL><DEDENT><DEDENT><DEDENT>if c_info["<STR_LIT>"]:<EOL><INDENT>compiler_cls.set_compiler_channels(c_info["<STR_LIT>"],<EOL>operator="<STR_LIT>")<EOL>self.processes.append(compiler_cls)<EOL><DEDENT><DEDENT>

Adds compiler channels to the :attr:`processes` attribute.

        This method will iterate over the pipeline's processes and check
        if any process is feeding channels to a compiler process. If so, that
        compiler process is added to the pipeline and those channels are
        linked to the compiler via some operator.


status_inst = pc.StatusCompiler(template="<STR_LIT>")<EOL>report_inst = pc.ReportCompiler(template="<STR_LIT>")<EOL>status_channels = []<EOL>for p in [p for p in self.processes]:<EOL><INDENT>if not any([isinstance(p, x) for x in self.skip_class]):<EOL><INDENT>status_channels.extend(p.status_strs)<EOL><DEDENT><DEDENT>if not status_channels:<EOL><INDENT>logger.debug("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>return<EOL><DEDENT>logger.debug("<STR_LIT>".format(status_channels))<EOL>if len(status_channels) != len(set(status_channels)):<EOL><INDENT>raise eh.ProcessError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(<EOL>"<STR_LIT:U+002CU+0020>".join(status_channels)<EOL>))<EOL><DEDENT>status_inst.set_compiler_channels(status_channels)<EOL>report_channels = ["<STR_LIT>".format(x.lstrip("<STR_LIT>")) for x in<EOL>status_channels]<EOL>report_inst.set_compiler_channels(report_channels)<EOL>self.processes.extend([status_inst, report_inst])<EOL>

Compiles all status channels for the status compiler process


config_str = "<STR_LIT>"<EOL>ignore_directives = ["<STR_LIT>", "<STR_LIT:version>"]<EOL>for p, directives in res_dict.items():<EOL><INDENT>for d, val in directives.items():<EOL><INDENT>if d in ignore_directives:<EOL><INDENT>continue<EOL><DEDENT>config_str += '<STR_LIT>'.format(p, pid, d, val)<EOL><DEDENT><DEDENT>return config_str<EOL>

Returns the nextflow resources string from a dictionary object

        If the dictionary has at least on of the resource directives, these
        will be compiled for each process in the dictionary and returned
        as a string read for injection in the nextflow config file template.

        This dictionary should be::

            dict = {"processA": {"cpus": 1, "memory": "4GB"},
                    "processB": {"cpus": 2}}

        Parameters
        ----------
        res_dict : dict
            Dictionary with the resources for processes.
        pid : int
            Unique identified of the process

        Returns
        -------
        str
            nextflow config string


config_str = "<STR_LIT>"<EOL>for p, directives in cont_dict.items():<EOL><INDENT>container = "<STR_LIT>"<EOL>if "<STR_LIT>" in directives:<EOL><INDENT>container += directives["<STR_LIT>"]<EOL>if "<STR_LIT:version>" in directives:<EOL><INDENT>container += "<STR_LIT>".format(directives["<STR_LIT:version>"])<EOL><DEDENT>else:<EOL><INDENT>container += "<STR_LIT>"<EOL><DEDENT><DEDENT>if container:<EOL><INDENT>config_str += '<STR_LIT>'.format(p, pid, container)<EOL><DEDENT><DEDENT>return config_str<EOL>

Returns the nextflow containers string from a dictionary object

        If the dictionary has at least on of the container directives, these
        will be compiled for each process in the dictionary and returned
        as a string read for injection in the nextflow config file template.

        This dictionary should be::

            dict = {"processA": {"container": "asd", "version": "1.0.0"},
                    "processB": {"container": "dsd"}}

        Parameters
        ----------
        cont_dict : dict
            Dictionary with the containers for processes.
        pid : int
            Unique identified of the process

        Returns
        -------
        str
            nextflow config string


params_str = "<STR_LIT>"<EOL>for p in self.processes:<EOL><INDENT>logger.debug("<STR_LIT>".format(<EOL>p.template, p.params)<EOL>)<EOL>if p.params and p.template != "<STR_LIT>":<EOL><INDENT>p.set_param_id("<STR_LIT>".format(p.pid))<EOL>params_str += "<STR_LIT>"<EOL>params_str += "<STR_LIT>".format(p.template,<EOL>p.pid)<EOL>params_str += "<STR_LIT>".format("<STR_LIT:->" * (len(p.template) + len(p.pid) + <NUM_LIT:12>))<EOL>params_str += "<STR_LIT>"<EOL><DEDENT>for param, val in p.params.items():<EOL><INDENT>if p.template == "<STR_LIT>":<EOL><INDENT>param_id = param<EOL><DEDENT>else:<EOL><INDENT>param_id = "<STR_LIT>".format(param, p.pid)<EOL><DEDENT>params_str += "<STR_LIT>".format(param_id, val["<STR_LIT:default>"])<EOL><DEDENT><DEDENT>return params_str<EOL>

Returns the nextflow params string from a dictionary object.

        The params dict should be a set of key:value pairs with the
        parameter name, and the default parameter value::

            self.params = {
                "genomeSize": 2.1,
                "minCoverage": 15
            }

        The values are then added to the string as they are. For instance,
        a ``2.1`` float will appear as ``param = 2.1`` and a
        ``"'teste'" string will appear as ``param = 'teste'`` (Note the
        string).

        Returns
        -------
        str
            Nextflow params configuration string


Returns the merged nextflow params string from a dictionary object.

        The params dict should be a set of key:value pairs with the
        parameter name, and the default parameter value::

            self.params = {
                "genomeSize": 2.1,
                "minCoverage": 15
            }

        The values are then added to the string as they are. For instance,
        a ``2.1`` float will appear as ``param = 2.1`` and a
        ``"'teste'" string will appear as ``param = 'teste'`` (Note the
        string).

        Identical parameters in multiple processes will be merged into the same
        param.

        Returns
        -------
        str
            Nextflow params configuration string


params_temp = {}<EOL>for p in self.processes:<EOL><INDENT>logger.debug("<STR_LIT>".format(p.template,<EOL>p.params))<EOL>for param, val in p.params.items():<EOL><INDENT>params_temp[param] = val["<STR_LIT:default>"]<EOL><DEDENT><DEDENT>config_str = "<STR_LIT>" + "<STR_LIT>".join([<EOL>"<STR_LIT>".format(param, val) for param, val in params_temp.items()<EOL>])<EOL>return config_str<EOL>


help_dict = {}<EOL>help_list = []<EOL>for p in self.processes:<EOL><INDENT>for param, val in p.params.items():<EOL><INDENT>if param in help_dict:<EOL><INDENT>help_dict[param]["<STR_LIT>"].append(p.template)<EOL><DEDENT>else:<EOL><INDENT>tpl = [p.template] if p.template != "<STR_LIT>" else []<EOL>help_dict[param] = {"<STR_LIT>": tpl,<EOL>"<STR_LIT:description>": val["<STR_LIT:description>"]}<EOL><DEDENT><DEDENT><DEDENT>for p, val in help_dict.items():<EOL><INDENT>if not val["<STR_LIT>"]:<EOL><INDENT>val["<STR_LIT>"] = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>val["<STR_LIT>"] = "<STR_LIT>".format("<STR_LIT:;>".join(val["<STR_LIT>"]))<EOL><DEDENT>help_list.append("<STR_LIT>".format(<EOL>p, val["<STR_LIT:description>"], val["<STR_LIT>"]))<EOL><DEDENT>return help_list<EOL>


config_str = "<STR_LIT>"<EOL>config_str += '<STR_LIT>'.format(self.pipeline_name)<EOL>config_str += '<STR_LIT>'.format(self.nf_file)<EOL>return config_str<EOL>

Returns the nextflow manifest config string to include in the
        config file from the information on the pipeline.

        Returns
        -------
        str
            Nextflow manifest configuration string


logger.debug("<STR_LIT>")<EOL>logger.debug("<STR_LIT>")<EOL>logger.debug("<STR_LIT>")<EOL>resources = "<STR_LIT>"<EOL>containers = "<STR_LIT>"<EOL>params = "<STR_LIT>"<EOL>manifest = "<STR_LIT>"<EOL>if self.merge_params:<EOL><INDENT>params += self._get_merged_params_string()<EOL>help_list = self._get_merged_params_help()<EOL><DEDENT>else:<EOL><INDENT>params += self._get_params_string()<EOL>help_list = self._get_params_help()<EOL><DEDENT>for p in self.processes:<EOL><INDENT>if not p.directives:<EOL><INDENT>continue<EOL><DEDENT>logger.debug("<STR_LIT>".format(<EOL>p.template, p.directives))<EOL>resources += self._get_resources_string(p.directives, p.pid)<EOL>containers += self._get_container_string(p.directives, p.pid)<EOL><DEDENT>manifest = self._get_manifest_string()<EOL>self.resources = self._render_config("<STR_LIT>", {<EOL>"<STR_LIT>": resources<EOL>})<EOL>self.containers = self._render_config("<STR_LIT>", {<EOL>"<STR_LIT>": containers<EOL>})<EOL>self.params = self._render_config("<STR_LIT>", {<EOL>"<STR_LIT>": params<EOL>})<EOL>self.manifest = self._render_config("<STR_LIT>", {<EOL>"<STR_LIT>": manifest<EOL>})<EOL>self.help = self._render_config("<STR_LIT>", {<EOL>"<STR_LIT>": basename(self.nf_file),<EOL>"<STR_LIT>": help_list,<EOL>"<STR_LIT:version>": __version__,<EOL>"<STR_LIT>": "<STR_LIT:U+0020>".join([x.upper() for x in self.pipeline_name])<EOL>})<EOL>self.user_config = self._render_config("<STR_LIT>", {})<EOL>

This method will iterate over all process in the pipeline and
        populate the nextflow configuration files with the directives
        of each process in the pipeline.


def dag_to_file(self, dict_viz, output_file="<STR_LIT>"):

outfile_dag = open(os.path.join(dirname(self.nf_file), output_file)<EOL>, "<STR_LIT:w>")<EOL>outfile_dag.write(json.dumps(dict_viz))<EOL>outfile_dag.close()<EOL>

Writes dag to output file

        Parameters
        ----------
        dict_viz: dict
            Tree like dictionary that is used to export tree data of processes
            to html file and here for the dotfile .treeDag.json


dict_viz = {<EOL>"<STR_LIT:name>": "<STR_LIT:root>",<EOL>"<STR_LIT>": []<EOL>}<EOL>last_of_us = {}<EOL>f_tree = self._fork_tree if self._fork_tree else {<NUM_LIT:1>: [<NUM_LIT:1>]}<EOL>for x, (k, v) in enumerate(f_tree.items()):<EOL><INDENT>for p in self.processes[<NUM_LIT:1>:]:<EOL><INDENT>if x == <NUM_LIT:0> and p.lane not in [k] + v:<EOL><INDENT>continue<EOL><DEDENT>if x > <NUM_LIT:0> and p.lane not in v:<EOL><INDENT>continue<EOL><DEDENT>if not p.parent_lane:<EOL><INDENT>lst = dict_viz["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>lst = last_of_us[p.parent_lane]<EOL><DEDENT>tooltip = {<EOL>"<STR_LIT:name>": "<STR_LIT>".format(p.template, p.pid),<EOL>"<STR_LIT>": {<EOL>"<STR_LIT>": p.pid,<EOL>"<STR_LIT:input>": p.input_type,<EOL>"<STR_LIT>": p.output_type if p.output_type else "<STR_LIT:None>",<EOL>"<STR_LIT>": p.lane,<EOL>},<EOL>"<STR_LIT>": []<EOL>}<EOL>dir_var = "<STR_LIT>"<EOL>for k2, v2 in p.directives.items():<EOL><INDENT>dir_var += k2<EOL>for d in v2:<EOL><INDENT>try:<EOL><INDENT>directive = v2[d].replace("<STR_LIT:'>", "<STR_LIT>").replace('<STR_LIT:">', '<STR_LIT>')if isinstance(v2[d], str) else v2[d]<EOL>dir_var += "<STR_LIT>".format(d, directive)<EOL><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>if dir_var:<EOL><INDENT>tooltip["<STR_LIT>"]["<STR_LIT>"] = dir_var<EOL><DEDENT>else:<EOL><INDENT>tooltip["<STR_LIT>"]["<STR_LIT>"] = "<STR_LIT>"<EOL><DEDENT>lst.append(tooltip)<EOL>last_of_us[p.lane] = lst[-<NUM_LIT:1>]["<STR_LIT>"]<EOL><DEDENT><DEDENT>self.dag_to_file(dict_viz)<EOL>with open(os.path.join(dirname(self.nf_file),<EOL>"<STR_LIT>"), "<STR_LIT:w>") as fh:<EOL><INDENT>fh.write(json.dumps(self._fork_tree))<EOL><DEDENT>return self._render_config("<STR_LIT>", {"<STR_LIT:data>": dict_viz})<EOL>

Write pipeline attributes to json

        This function writes the pipeline and their attributes to a json file,
        that is intended to be read by resources/pipeline_graph.html to render
        a graphical output showing the DAG.


<EOL>with open(join(project_root, "<STR_LIT>"), "<STR_LIT:w>") as fh:<EOL><INDENT>fh.write(self.resources)<EOL><DEDENT>with open(join(project_root, "<STR_LIT>"), "<STR_LIT:w>") as fh:<EOL><INDENT>fh.write(self.containers)<EOL><DEDENT>with open(join(project_root, "<STR_LIT>"), "<STR_LIT:w>") as fh:<EOL><INDENT>fh.write(self.params)<EOL><DEDENT>with open(join(project_root, "<STR_LIT>"), "<STR_LIT:w>") as fh:<EOL><INDENT>fh.write(self.manifest)<EOL><DEDENT>if not exists(join(project_root, "<STR_LIT>")):<EOL><INDENT>with open(join(project_root, "<STR_LIT>"), "<STR_LIT:w>") as fh:<EOL><INDENT>fh.write(self.user_config)<EOL><DEDENT><DEDENT>lib_dir = join(project_root, "<STR_LIT>")<EOL>if not exists(lib_dir):<EOL><INDENT>os.makedirs(lib_dir)<EOL><DEDENT>with open(join(lib_dir, "<STR_LIT>"), "<STR_LIT:w>") as fh:<EOL><INDENT>fh.write(self.help)<EOL><DEDENT>pipeline_to_json = self.render_pipeline()<EOL>with open(splitext(self.nf_file)[<NUM_LIT:0>] + "<STR_LIT>", "<STR_LIT:w>") as fh:<EOL><INDENT>fh.write(pipeline_to_json)<EOL><DEDENT>

Wrapper method that writes all configuration files to the pipeline
        directory


params_json = {}<EOL>for p in self.processes[<NUM_LIT:1>:]:<EOL><INDENT>params_json[p.template] = p.params<EOL><DEDENT>sys.stdout.write(json.dumps(params_json))<EOL>

Export pipeline params as a JSON to stdout

        This run mode iterates over the pipeline processes and exports the
        params dictionary of each component as a JSON to stdout.


directives_json = {}<EOL>for p in self.processes[<NUM_LIT:1>:]:<EOL><INDENT>directives_json[p.template] = p.directives<EOL><DEDENT>sys.stdout.write(json.dumps(directives_json))<EOL>

Export pipeline directives as a JSON to stdout


<EOL>dict_of_parsed = {}<EOL>terminal_width = shutil.get_terminal_size().columns - <NUM_LIT:3><EOL>center_string = "<STR_LIT>"<EOL>tags_list = [<EOL>[<EOL>"<STR_LIT:=>" * int(terminal_width / <NUM_LIT:4>),<EOL>"<STR_LIT>".format(<EOL>"<STR_LIT:=>" * int(((terminal_width/<NUM_LIT:2> - len(center_string)) / <NUM_LIT:2>)),<EOL>center_string)<EOL>,<EOL>"<STR_LIT>".format("<STR_LIT:=>" * int(terminal_width / <NUM_LIT:4>))<EOL>],<EOL>["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"],<EOL>[<EOL>"<STR_LIT:=>" * int(terminal_width / <NUM_LIT:4>),<EOL>"<STR_LIT:=>" * int(terminal_width / <NUM_LIT:2>),<EOL>"<STR_LIT:=>" * int(terminal_width / <NUM_LIT:4>)<EOL>]<EOL>]<EOL>for p in self.processes[<NUM_LIT:1>:]:<EOL><INDENT>template = p.template<EOL>if template in dict_of_parsed:<EOL><INDENT>continue<EOL><DEDENT>dict_of_parsed[template] = {<EOL>"<STR_LIT>": []<EOL>}<EOL>for directives in p.directives.values():<EOL><INDENT>try:<EOL><INDENT>repo = directives["<STR_LIT>"]<EOL>default_version = directives["<STR_LIT:version>"]<EOL><DEDENT>except KeyError:<EOL><INDENT>repo = "<STR_LIT>"<EOL>default_version = "<STR_LIT>"<EOL><DEDENT>repo_version = repo + default_version<EOL>if repo_version not in dict_of_parsed[template]["<STR_LIT>"]:<EOL><INDENT>r = requests.get(<EOL>"<STR_LIT>"<EOL>.format(repo)<EOL>)<EOL>if r.status_code != <NUM_LIT>:<EOL><INDENT>r_content = json.loads(r.content)["<STR_LIT>"]<EOL>for version in r_content:<EOL><INDENT>printed_version = (version["<STR_LIT:name>"] + "<STR_LIT:*>")if version["<STR_LIT:name>"] == default_versionelse version["<STR_LIT:name>"]<EOL>tags_list.append([template, repo, printed_version])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>tags_list.append([template, repo, "<STR_LIT>"])<EOL><DEDENT><DEDENT>dict_of_parsed[template]["<STR_LIT>"].append(repo_version)<EOL><DEDENT><DEDENT>for x, entry in enumerate(tags_list):<EOL><INDENT>color = "<STR_LIT>" if x < <NUM_LIT:3> else("<STR_LIT>" if x % <NUM_LIT:2> != <NUM_LIT:0> else "<STR_LIT>")<EOL>final_width = [<EOL>int(terminal_width/<NUM_LIT:4>),<EOL>int(terminal_width/<NUM_LIT:2>),<EOL>int(terminal_width/<NUM_LIT:4>)<EOL>]<EOL>sys.stdout.write(<EOL>colored_print("<STR_LIT>".format(<EOL>*entry, *final_width), color)<EOL>)<EOL><DEDENT>sys.stdout.write("<STR_LIT>".format("<STR_LIT>",<EOL>terminal_width + <NUM_LIT:3>))<EOL>

Export all dockerhub tags associated with each component given by
the -t flag.


Main pipeline builder

        This method is responsible for building the
        :py:attr:`NextflowGenerator.template` attribute that will contain
        the nextflow code of the pipeline.

        First it builds the header, then sets the main channels, the
        secondary inputs, secondary channels and finally the
        status channels. When the pipeline is built, is writes the code
        to a nextflow file.


This function is bound to the SIGINT signal (like ctrl+c) to graciously
    exit the program and reset the curses options.


if not os.path.exists(self.trace_file):<EOL><INDENT>raise eh.InspectionError("<STR_LIT>"<EOL>"<STR_LIT>".format(self.trace_file))<EOL><DEDENT>if not os.path.exists(self.log_file):<EOL><INDENT>raise eh.InspectionError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>

Checks whetner the trace and log files are available


return dict(<EOL>(x.strip(), pos) for pos, x in enumerate(header.split("<STR_LIT:\t>"))<EOL>)<EOL>

Parses the trace file header and retrieves the positions of each
        column key.

        Parameters
        ----------
        header : str
            The header line of nextflow's trace file

        Returns
        -------
        dict
            Mapping the column ID to its position (e.g.: {"tag":2})


try:<EOL><INDENT>first_hash, second_hash = hash_str.split("<STR_LIT:/>")<EOL>first_hash_path = join(abspath("<STR_LIT>"), first_hash)<EOL>for l in os.listdir(first_hash_path):<EOL><INDENT>if l.startswith(second_hash):<EOL><INDENT>return join(first_hash_path, l)<EOL><DEDENT><DEDENT><DEDENT>except FileNotFoundError:<EOL><INDENT>return None<EOL><DEDENT>

Expands the hash string of a process (ae/1dasjdm) into a full
        working directory

        Parameters
        ----------
        hash_str : str
            Nextflow process hash with the beggining of the work directory

        Returns
        -------
        str
            Path to working directory of the hash string


if s == "<STR_LIT:->":<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>if s.endswith("<STR_LIT>"):<EOL><INDENT>return float(s.rstrip("<STR_LIT>")) / <NUM_LIT:1000><EOL><DEDENT>fields = list(map(float, re.split("<STR_LIT>", s)[:-<NUM_LIT:1>]))<EOL>if len(fields) == <NUM_LIT:4>:<EOL><INDENT>return fields[<NUM_LIT:0>] * <NUM_LIT> * <NUM_LIT> + fields[<NUM_LIT:1>] * <NUM_LIT> + fields[<NUM_LIT:2>] * <NUM_LIT> +fields[<NUM_LIT:3>]<EOL><DEDENT>if len(fields) == <NUM_LIT:3>:<EOL><INDENT>return fields[<NUM_LIT:0>] * <NUM_LIT> + fields[<NUM_LIT:1>] * <NUM_LIT> + fields[<NUM_LIT:2>]<EOL><DEDENT>elif len(fields) == <NUM_LIT:2>:<EOL><INDENT>return fields[<NUM_LIT:0>] * <NUM_LIT> + fields[<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>return fields[<NUM_LIT:0>]<EOL><DEDENT>

Converts a hms string into seconds.

        Parameters
        ----------
        s : str
            The hms string can be something like '20s', '1m30s' or '300ms'.

        Returns
        -------
        float
            Time in seconds.


if s.upper().endswith("<STR_LIT>"):<EOL><INDENT>return float(s.rstrip("<STR_LIT>")) / <NUM_LIT><EOL><DEDENT>elif s.upper().endswith("<STR_LIT>"):<EOL><INDENT>return float(s.rstrip("<STR_LIT:B>")) / <NUM_LIT> / <NUM_LIT><EOL><DEDENT>elif s.upper().endswith("<STR_LIT>"):<EOL><INDENT>return float(s.rstrip("<STR_LIT>"))<EOL><DEDENT>elif s.upper().endswith("<STR_LIT>"):<EOL><INDENT>return float(s.rstrip("<STR_LIT>")) * <NUM_LIT><EOL><DEDENT>elif s.upper().endswith("<STR_LIT>"):<EOL><INDENT>return float(s.rstrip("<STR_LIT>")) * <NUM_LIT> * <NUM_LIT><EOL><DEDENT>else:<EOL><INDENT>return float(s)<EOL><DEDENT>

Converts size string into megabytes

        Parameters
        ----------
        s : str
            The size string can be '30KB', '20MB' or '1GB'

        Returns
        -------
        float
            With the size in bytes


if s / <NUM_LIT> > <NUM_LIT:1>:<EOL><INDENT>return "<STR_LIT>".format(round(s / <NUM_LIT>, <NUM_LIT:1>))<EOL><DEDENT>else:<EOL><INDENT>return "<STR_LIT>".format(s)<EOL><DEDENT>


with open(self.log_file) as fh:<EOL><INDENT>for line in fh:<EOL><INDENT>if re.match("<STR_LIT>", line):<EOL><INDENT>match = re.match("<STR_LIT>", line)<EOL>process = match.group(<NUM_LIT:1>)<EOL>if any([process.startswith(x) for x in self._blacklist]):<EOL><INDENT>continue<EOL><DEDENT>if process not in self.skip_processes:<EOL><INDENT>self.processes[match.group(<NUM_LIT:1>)] = {<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": set(),<EOL>"<STR_LIT>": set(),<EOL>"<STR_LIT>": set(),<EOL>"<STR_LIT>": set(),<EOL>"<STR_LIT>": None,<EOL>"<STR_LIT>": None<EOL>}<EOL>self.process_tags[process] = {}<EOL><DEDENT><DEDENT>if re.match("<STR_LIT>", line):<EOL><INDENT>tag_match = re.match("<STR_LIT>", line)<EOL>self.pipeline_tag = tag_match.group(<NUM_LIT:1>) if tag_match else"<STR_LIT:?>"<EOL>name_match = re.match("<STR_LIT>", line)<EOL>self.pipeline_name = name_match.group(<NUM_LIT:1>) if name_matchelse "<STR_LIT:?>"<EOL><DEDENT><DEDENT><DEDENT>self.content_lines = len(self.processes)<EOL>

Parses the .nextflow.log file and retrieves the complete list
        of processes

        This method searches for specific signatures at the beginning of the
        .nextflow.log file::

             Apr-19 19:07:32.660 [main] DEBUG nextflow.processor
             TaskProcessor - Creating operator > report_corrupt_1_1 --
             maxForks: 4

        When a line with the .*Creating operator.* signature is found, the
        process name is retrieved and populates the :attr:`processes` attribute


self.trace_info = defaultdict(list)<EOL>self.process_tags = {}<EOL>self.process_stats = {}<EOL>self.samples = []<EOL>self.stored_ids = []<EOL>self.stored_log_ids = []<EOL>self.time_start = None<EOL>self.time_stop = None<EOL>self.execution_command = None<EOL>self.nextflow_version = None<EOL>self.abort_cause = None<EOL>self._c = <NUM_LIT:0><EOL>for p in self.processes.values():<EOL><INDENT>p["<STR_LIT>"] = "<STR_LIT>"<EOL>for i in ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]:<EOL><INDENT>p[i] = set()<EOL><DEDENT><DEDENT>

Clears inspect attributes when re-executing a pipeline


with open(self.log_file) as fh:<EOL><INDENT>try:<EOL><INDENT>first_line = next(fh)<EOL><DEDENT>except:<EOL><INDENT>raise eh.InspectionError("<STR_LIT>")<EOL><DEDENT>time_str = "<STR_LIT:U+0020>".join(first_line.split()[:<NUM_LIT:2>])<EOL>self.time_start = time_str<EOL>if not self.execution_command:<EOL><INDENT>try:<EOL><INDENT>self.execution_command = re.match(<EOL>"<STR_LIT>", first_line).group(<NUM_LIT:1>)<EOL><DEDENT>except AttributeError:<EOL><INDENT>self.execution_command = "<STR_LIT>"<EOL><DEDENT><DEDENT>for line in fh:<EOL><INDENT>if "<STR_LIT>" in line:<EOL><INDENT>if not self.nextflow_version:<EOL><INDENT>try:<EOL><INDENT>vline = next(fh)<EOL>self.nextflow_version = re.match(<EOL>"<STR_LIT>", vline).group(<NUM_LIT:1>)<EOL><DEDENT>except AttributeError:<EOL><INDENT>self.nextflow_version = "<STR_LIT>"<EOL><DEDENT><DEDENT><DEDENT>if "<STR_LIT>" in line:<EOL><INDENT>self.run_status = "<STR_LIT>"<EOL>try:<EOL><INDENT>self.abort_cause = re.match(<EOL>"<STR_LIT>", line).group(<NUM_LIT:1>)<EOL><DEDENT>except AttributeError:<EOL><INDENT>self.abort_cause = "<STR_LIT>"<EOL><DEDENT>time_str = "<STR_LIT:U+0020>".join(line.split()[:<NUM_LIT:2>])<EOL>self.time_stop = time_str<EOL>self.send = True<EOL>return<EOL><DEDENT>if "<STR_LIT>" in line:<EOL><INDENT>self.run_status = "<STR_LIT>"<EOL>time_str = "<STR_LIT:U+0020>".join(line.split()[:<NUM_LIT:2>])<EOL>self.time_stop = time_str<EOL>self.send = True<EOL>return<EOL><DEDENT><DEDENT><DEDENT>if self.run_status not in ["<STR_LIT>", "<STR_LIT>"]:<EOL><INDENT>self._clear_inspect()<EOL>sleep(<NUM_LIT:5>)<EOL>self._get_pipeline_processes()<EOL><DEDENT>self.run_status = "<STR_LIT>"<EOL>

Parses the .nextflow.log file for signatures of pipeline status.
        It sets the :attr:`status_info` attribute.


def _update_tag_status(self, process, vals):

good_status = ["<STR_LIT>", "<STR_LIT>"]<EOL>for v in list(vals)[::-<NUM_LIT:1>]:<EOL><INDENT>p = self.processes[process]<EOL>tag = v["<STR_LIT>"]<EOL>if tag in p["<STR_LIT>"]:<EOL><INDENT>p["<STR_LIT>"].remove(tag)<EOL>if v["<STR_LIT:status>"] in good_status:<EOL><INDENT>p["<STR_LIT>"].add(tag)<EOL><DEDENT>elif v["<STR_LIT:status>"] == "<STR_LIT>":<EOL><INDENT>if not v["<STR_LIT>"]:<EOL><INDENT>v["<STR_LIT>"] = "<STR_LIT>"<EOL><DEDENT>self.process_tags[process][tag]["<STR_LIT>"] =self._retrieve_log(join(v["<STR_LIT>"], "<STR_LIT>"))<EOL>p["<STR_LIT>"].add(tag)<EOL><DEDENT><DEDENT>elif tag in p["<STR_LIT>"]:<EOL><INDENT>if v["<STR_LIT:status>"] in good_status:<EOL><INDENT>p["<STR_LIT>"].remove(tag)<EOL>p["<STR_LIT>"].remove(tag)<EOL>del self.process_tags[process][tag]["<STR_LIT>"]<EOL><DEDENT>elif self.run_status == "<STR_LIT>":<EOL><INDENT>p["<STR_LIT>"].remove(tag)<EOL><DEDENT><DEDENT>elif v["<STR_LIT:status>"] in good_status:<EOL><INDENT>p["<STR_LIT>"].add(tag)<EOL><DEDENT>if v["<STR_LIT:status>"] not in good_status:<EOL><INDENT>if v["<STR_LIT>"] in list(p["<STR_LIT>"]) + list(p["<STR_LIT>"]):<EOL><INDENT>vals.remove(v)<EOL><DEDENT><DEDENT><DEDENT>return vals<EOL>

Updates the 'submitted', 'finished', 'failed' and 'retry' status
        of each process/tag combination.

        Process/tag combinations provided to this method already appear on
        the trace file, so their submission status is updated based on their
        execution status from nextflow.

        For instance, if a tag is successfully
        complete, it is moved from the 'submitted' to the 'finished' list.
        If not, it is moved to the 'failed' list.

        Parameters
        ----------
        process : str
            Name of the current process. Must be present in attr:`processes`
        vals : list
            List of tags for this process that have been gathered in the
            trace file.


with open(self.log_file) as fh:<EOL><INDENT>for line in fh:<EOL><INDENT>if "<STR_LIT>" in line:<EOL><INDENT>return<EOL><DEDENT>if "<STR_LIT>" in line:<EOL><INDENT>process_m = re.match("<STR_LIT>", line)<EOL>if process_m:<EOL><INDENT>process = process_m.group(<NUM_LIT:1>)<EOL>if process in self.processes:<EOL><INDENT>self.processes[process]["<STR_LIT>"] = "<STR_LIT:C>"<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>

Checks whether the channels to each process have been closed.


if not os.path.exists(path):<EOL><INDENT>return None<EOL><DEDENT>with open(path) as fh:<EOL><INDENT>return fh.readlines()<EOL><DEDENT>

Method used to retrieve the contents of a log file into a list.

        Parameters
        ----------
        path

        Returns
        -------
        list or None
            Contents of the provided file, each line as a list entry


def _update_trace_info(self, fields, hm):

process = fields[hm["<STR_LIT>"]]<EOL>if process not in self.processes:<EOL><INDENT>return<EOL><DEDENT>info = dict((column, fields[pos]) for column, pos in hm.items())<EOL>process_tag_headers = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>for h in process_tag_headers:<EOL><INDENT>if info["<STR_LIT>"] not in self.process_tags[process]:<EOL><INDENT>try:<EOL><INDENT>timestart = info["<STR_LIT:start>"].split()[<NUM_LIT:1>]<EOL><DEDENT>except KeyError:<EOL><INDENT>timestart = None<EOL><DEDENT>self.process_tags[process][info["<STR_LIT>"]] = {<EOL>"<STR_LIT>": self._expand_path(info["<STR_LIT>"]),<EOL>"<STR_LIT:start>": timestart<EOL>}<EOL><DEDENT>if h in info and info["<STR_LIT>"] != "<STR_LIT:->":<EOL><INDENT>if h != "<STR_LIT>" and info[h] != "<STR_LIT:->":<EOL><INDENT>self.process_tags[process][info["<STR_LIT>"]][h] =round(self._size_coverter(info[h]), <NUM_LIT:2>)<EOL><DEDENT>else:<EOL><INDENT>self.process_tags[process][info["<STR_LIT>"]][h] = info[h]<EOL><DEDENT><DEDENT><DEDENT>if "<STR_LIT>" in info and not self.processes[process]["<STR_LIT>"]:<EOL><INDENT>self.processes[process]["<STR_LIT>"] = info["<STR_LIT>"]<EOL><DEDENT>if "<STR_LIT>" in info and not self.processes[process]["<STR_LIT>"]:<EOL><INDENT>try:<EOL><INDENT>self.processes[process]["<STR_LIT>"] = self._size_coverter(<EOL>info["<STR_LIT>"])<EOL><DEDENT>except ValueError:<EOL><INDENT>self.processes[process]["<STR_LIT>"] = None<EOL><DEDENT><DEDENT>if info["<STR_LIT>"] in self.stored_ids:<EOL><INDENT>return<EOL><DEDENT>if "<STR_LIT>" in info:<EOL><INDENT>hs = info["<STR_LIT>"]<EOL>info["<STR_LIT>"] = self._expand_path(hs)<EOL><DEDENT>if "<STR_LIT>" in info:<EOL><INDENT>tag = info["<STR_LIT>"]<EOL>if tag != "<STR_LIT:->" and tag not in self.samples andtag.split()[<NUM_LIT:0>] not in self.samples:<EOL><INDENT>self.samples.append(tag)<EOL><DEDENT><DEDENT>self.trace_info[process].append(info)<EOL>self.stored_ids.append(info["<STR_LIT>"])<EOL>

Parses a trace line and updates the :attr:`status_info` attribute.

        Parameters
        ----------
        fields : list
            List of the tab-seperated elements of the trace line
        hm : dict
            Maps the column IDs to their position in the fields argument.
            This dictionary object is retrieve from :func:`_header_mapping`.


def _update_process_resources(self, process, vals):

resources = ["<STR_LIT>"]<EOL>for r in resources:<EOL><INDENT>if not self.processes[process][r]:<EOL><INDENT>try:<EOL><INDENT>self.processes[process][r] = vals[<NUM_LIT:0>]["<STR_LIT>"]<EOL><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>

Updates the resources info in :attr:`processes` dictionary.


def _cpu_load_parser(self, cpus, cpu_per, t):

try:<EOL><INDENT>_cpus = float(cpus)<EOL>_cpu_per = float(cpu_per.replace("<STR_LIT:U+002C>", "<STR_LIT:.>").replace("<STR_LIT:%>", "<STR_LIT>"))<EOL>hours = self._hms(t) / <NUM_LIT> / <NUM_LIT><EOL>return ((_cpu_per / (<NUM_LIT:100> * _cpus)) * _cpus) * hours<EOL><DEDENT>except ValueError:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>

Parses the cpu load from the number of cpus and its usage
        percentage and returnsde cpu/hour measure

        Parameters
        ----------
        cpus : str
            Number of cpus allocated.
        cpu_per : str
            Percentage of cpu load measured (e.g.: 200,5%).
        t : str
            The time string can be something like '20s', '1m30s' or '300ms'.


def _assess_resource_warnings(self, process, vals):

cpu_warnings = {}<EOL>mem_warnings = {}<EOL>for i in vals:<EOL><INDENT>try:<EOL><INDENT>expected_load = float(i["<STR_LIT>"]) * <NUM_LIT:100><EOL>cpu_load = float(i["<STR_LIT>"].replace("<STR_LIT:U+002C>", "<STR_LIT:.>").replace("<STR_LIT:%>", "<STR_LIT>"))<EOL>if expected_load * <NUM_LIT> > cpu_load > expected_load * <NUM_LIT>:<EOL><INDENT>cpu_warnings[i["<STR_LIT>"]] = {<EOL>"<STR_LIT>":  expected_load,<EOL>"<STR_LIT:value>": cpu_load<EOL>}<EOL><DEDENT><DEDENT>except (ValueError, KeyError):<EOL><INDENT>pass<EOL><DEDENT>try:<EOL><INDENT>rss = self._size_coverter(i["<STR_LIT>"])<EOL>mem_allocated = self._size_coverter(i["<STR_LIT>"])<EOL>if rss > mem_allocated * <NUM_LIT>:<EOL><INDENT>mem_warnings[i["<STR_LIT>"]] = {<EOL>"<STR_LIT>": mem_allocated,<EOL>"<STR_LIT:value>": rss<EOL>}<EOL><DEDENT><DEDENT>except (ValueError, KeyError):<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return cpu_warnings, mem_warnings<EOL>

Assess whether the cpu load or memory usage is above the allocation

        Parameters
        ----------
        process : str
            Process name
        vals : vals
            List of trace information for each tag of that process

        Returns
        -------
        cpu_warnings : dict
            Keys are tags and values are the excessive cpu load
        mem_warnings : dict
            Keys are tags and values are the excessive rss


good_status = ["<STR_LIT>", "<STR_LIT>"]<EOL>for process, vals in self.trace_info.items():<EOL><INDENT>vals = self._update_tag_status(process, vals)<EOL>self._update_process_resources(process, vals)<EOL>self.process_stats[process] = {}<EOL>inst = self.process_stats[process]<EOL>inst["<STR_LIT>"] = "<STR_LIT:{}>".format(<EOL>len([x for x in vals if x["<STR_LIT:status>"] in good_status]))<EOL>try:<EOL><INDENT>time_array = [self._hms(x["<STR_LIT>"]) for x in vals]<EOL>mean_time = round(sum(time_array) / len(time_array), <NUM_LIT:1>)<EOL>mean_time_str = strftime('<STR_LIT>', gmtime(mean_time))<EOL>inst["<STR_LIT>"] = mean_time_str<EOL><DEDENT>except KeyError:<EOL><INDENT>inst["<STR_LIT>"] = "<STR_LIT:->"<EOL><DEDENT>try:<EOL><INDENT>cpu_hours = [self._cpu_load_parser(<EOL>x["<STR_LIT>"], x["<STR_LIT>"], x["<STR_LIT>"]) for x in vals]<EOL>inst["<STR_LIT>"] = round(sum(cpu_hours), <NUM_LIT:2>)<EOL><DEDENT>except KeyError:<EOL><INDENT>inst["<STR_LIT>"] = "<STR_LIT:->"<EOL><DEDENT>inst["<STR_LIT>"], inst["<STR_LIT>"] =self._assess_resource_warnings(process, vals)<EOL>try:<EOL><INDENT>rss_values = [self._size_coverter(x["<STR_LIT>"]) for x in vals<EOL>if x["<STR_LIT>"] != "<STR_LIT:->"]<EOL>if rss_values:<EOL><INDENT>max_rss = round(max(rss_values))<EOL>rss_str = self._size_compress(max_rss)<EOL><DEDENT>else:<EOL><INDENT>rss_str = "<STR_LIT:->"<EOL><DEDENT>inst["<STR_LIT>"] = rss_str<EOL><DEDENT>except KeyError:<EOL><INDENT>inst["<STR_LIT>"] = "<STR_LIT:->"<EOL><DEDENT>try:<EOL><INDENT>rchar_values = [self._size_coverter(x["<STR_LIT>"]) for x in vals<EOL>if x["<STR_LIT>"] != "<STR_LIT:->"]<EOL>if rchar_values:<EOL><INDENT>avg_rchar = round(sum(rchar_values) / len(rchar_values))<EOL>rchar_str = self._size_compress(avg_rchar)<EOL><DEDENT>else:<EOL><INDENT>rchar_str = "<STR_LIT:->"<EOL><DEDENT><DEDENT>except KeyError:<EOL><INDENT>rchar_str = "<STR_LIT:->"<EOL><DEDENT>inst["<STR_LIT>"] = rchar_str<EOL>try:<EOL><INDENT>wchar_values = [self._size_coverter(x["<STR_LIT>"]) for x in vals<EOL>if x["<STR_LIT>"] != "<STR_LIT:->"]<EOL>if wchar_values:<EOL><INDENT>avg_wchar = round(sum(wchar_values) / len(wchar_values))<EOL>wchar_str = self._size_compress(avg_wchar)<EOL><DEDENT>else:<EOL><INDENT>wchar_str = "<STR_LIT:->"<EOL><DEDENT><DEDENT>except KeyError:<EOL><INDENT>wchar_str = "<STR_LIT:->"<EOL><DEDENT>inst["<STR_LIT>"] = wchar_str<EOL><DEDENT>

Updates the process stats with the information from the processes

        This method is called at the end of each static parsing of the nextflow
        trace file. It re-populates the :attr:`process_stats` dictionary
        with the new stat metrics.


<EOL>size_stamp = os.path.getsize(self.trace_file)<EOL>self.trace_retry = <NUM_LIT:0><EOL>if size_stamp and size_stamp == self.trace_sizestamp:<EOL><INDENT>return<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>".format(size_stamp))<EOL>self.trace_sizestamp = size_stamp<EOL><DEDENT>with open(self.trace_file) as fh:<EOL><INDENT>header = next(fh).strip()<EOL>while not header:<EOL><INDENT>header = next(fh).strip()<EOL><DEDENT>hm = self._header_mapping(header)<EOL>for line in fh:<EOL><INDENT>if line.strip() == "<STR_LIT>":<EOL><INDENT>continue<EOL><DEDENT>fields = line.strip().split("<STR_LIT:\t>")<EOL>if fields[hm["<STR_LIT>"]] in self.stored_ids:<EOL><INDENT>continue<EOL><DEDENT>self._update_trace_info(fields, hm)<EOL>self.send = True<EOL><DEDENT><DEDENT>self._update_process_stats()<EOL>self._update_barrier_status()<EOL>

Method that parses the trace file once and updates the
        :attr:`status_info` attribute with the new entries.


<EOL>size_stamp = os.path.getsize(self.log_file)<EOL>self.log_retry = <NUM_LIT:0><EOL>if size_stamp and size_stamp == self.log_sizestamp:<EOL><INDENT>return<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>".format(size_stamp))<EOL>self.log_sizestamp = size_stamp<EOL><DEDENT>r = "<STR_LIT>"<EOL>with open(self.log_file) as fh:<EOL><INDENT>for line in fh:<EOL><INDENT>if "<STR_LIT>" in line or"<STR_LIT>" in line or"<STR_LIT>" in line:<EOL><INDENT>m = re.match(r, line)<EOL>if not m:<EOL><INDENT>continue<EOL><DEDENT>time_start = m.group(<NUM_LIT:1>)<EOL>workdir = m.group(<NUM_LIT:2>)<EOL>process = m.group(<NUM_LIT:3>)<EOL>tag = m.group(<NUM_LIT:4>)<EOL>if time_start + tag not in self.stored_log_ids:<EOL><INDENT>self.stored_log_ids.append(time_start + tag)<EOL><DEDENT>else:<EOL><INDENT>continue<EOL><DEDENT>if process not in self.processes:<EOL><INDENT>continue<EOL><DEDENT>p = self.processes[process]<EOL>if tag in list(p["<STR_LIT>"]) + list(p["<STR_LIT>"]):<EOL><INDENT>continue<EOL><DEDENT>if tag in list(p["<STR_LIT>"]) and"<STR_LIT>" in line:<EOL><INDENT>p["<STR_LIT>"].add(tag)<EOL>self.send = True<EOL>continue<EOL><DEDENT>p["<STR_LIT>"] = "<STR_LIT:R>"<EOL>if tag not in p["<STR_LIT>"]:<EOL><INDENT>p["<STR_LIT>"].add(tag)<EOL>if tag not in self.process_tags[process]:<EOL><INDENT>self.process_tags[process][tag] = {<EOL>"<STR_LIT>": self._expand_path(workdir),<EOL>"<STR_LIT:start>": time_start<EOL>}<EOL>self.send = True<EOL><DEDENT>elif not self.process_tags[process][tag]["<STR_LIT:start>"]:<EOL><INDENT>self.process_tags[process][tag]["<STR_LIT:start>"] = time_start<EOL>self.send = True<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>self._update_pipeline_status()<EOL>

Method that parses the nextflow log file once and updates the
        submitted number of samples for each process


try:<EOL><INDENT>self.log_parser()<EOL><DEDENT>except (FileNotFoundError, StopIteration) as e:<EOL><INDENT>logger.debug("<STR_LIT>" + str(sys.exc_info()[<NUM_LIT:0>]))<EOL>self.log_retry += <NUM_LIT:1><EOL>if self.log_retry == self.MAX_RETRIES:<EOL><INDENT>raise e<EOL><DEDENT><DEDENT>try:<EOL><INDENT>self.trace_parser()<EOL><DEDENT>except (FileNotFoundError, StopIteration) as e:<EOL><INDENT>logger.debug("<STR_LIT>" + str(sys.exc_info()[<NUM_LIT:0>]))<EOL>self.trace_retry += <NUM_LIT:1><EOL>if self.trace_retry == self.MAX_RETRIES:<EOL><INDENT>raise e<EOL><DEDENT><DEDENT>

Wrapper method that calls the appropriate main updating methods of
        the inspection.

        It is meant to be used inside a loop (like while), so that it can
        continuously update the class attributes from the trace and log files.
        It already implements checks to parse these files only when they
        change, and they ignore entries that have been previously processes.


Displays the default pipeline inspection overview


if direction == "<STR_LIT>" and self.top_line != <NUM_LIT:0>:<EOL><INDENT>self.top_line -= <NUM_LIT:1><EOL><DEDENT>elif direction == "<STR_LIT>" andself.screen.getmaxyx()[<NUM_LIT:0>] + self.top_line<= self.content_lines + <NUM_LIT:3>:<EOL><INDENT>self.top_line += <NUM_LIT:1><EOL><DEDENT>


if direction == "<STR_LIT:left>" and self.padding != <NUM_LIT:0>:<EOL><INDENT>self.padding -= <NUM_LIT:1><EOL><DEDENT>if direction == "<STR_LIT:right>" andself.screen.getmaxyx()[<NUM_LIT:1>] + self.padding < self.max_width:<EOL><INDENT>self.padding += <NUM_LIT:1><EOL><DEDENT>


colors = {<EOL>"<STR_LIT>": <NUM_LIT:1>,<EOL>"<STR_LIT:R>": <NUM_LIT:2>,<EOL>"<STR_LIT:C>": <NUM_LIT:3><EOL>}<EOL>pc = {<EOL>"<STR_LIT>": <NUM_LIT:3>,<EOL>"<STR_LIT>": <NUM_LIT:3>,<EOL>"<STR_LIT>": <NUM_LIT:4>,<EOL>"<STR_LIT:error>": <NUM_LIT:4><EOL>}<EOL>curses.init_pair(<NUM_LIT:1>, curses.COLOR_WHITE, curses.COLOR_BLACK)<EOL>curses.init_pair(<NUM_LIT:2>, curses.COLOR_BLUE, curses.COLOR_BLACK)<EOL>curses.init_pair(<NUM_LIT:3>, curses.COLOR_GREEN, curses.COLOR_BLACK)<EOL>curses.init_pair(<NUM_LIT:4>, curses.COLOR_MAGENTA, curses.COLOR_BLACK)<EOL>height, width = self.screen.getmaxyx()<EOL>win = curses.newpad(height, <NUM_LIT>)<EOL>header = "<STR_LIT>".format(<EOL>self.pipeline_tag, strftime("<STR_LIT>", gmtime()))<EOL>win.addstr(<NUM_LIT:0>, <NUM_LIT:0>, header)<EOL>win.addstr(<NUM_LIT:0>, len(header), self.run_status,<EOL>curses.color_pair(pc[self.run_status]))<EOL>submission_str = "<STR_LIT>".format(<EOL>"<STR_LIT>".format(<EOL>sum([len(x["<STR_LIT>"]) for x in self.processes.values()])<EOL>),<EOL>"<STR_LIT>".format(<EOL>sum([len(x["<STR_LIT>"]) for x in self.processes.values()])<EOL>),<EOL>"<STR_LIT>".format(<EOL>sum([len(x["<STR_LIT>"]) for x in self.processes.values()])<EOL>),<EOL>"<STR_LIT>".format(<EOL>sum([len(x["<STR_LIT>"]) for x in self.processes.values()])<EOL>)<EOL>)<EOL>win.addstr(<EOL><NUM_LIT:1>, <NUM_LIT:0>, submission_str, curses.color_pair(<NUM_LIT:1>)<EOL>)<EOL>headers = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>header_str = "<STR_LIT>""<STR_LIT>""<STR_LIT>""<STR_LIT>""<STR_LIT>""<STR_LIT>""<STR_LIT>""<STR_LIT>""<STR_LIT>".format(*headers)<EOL>self.max_width = len(header_str)<EOL>win.addstr(<NUM_LIT:3>, <NUM_LIT:0>, header_str, curses.A_UNDERLINE | curses.A_REVERSE)<EOL>top = self.top_line<EOL>bottom = self.screen_lines - <NUM_LIT:4> + self.top_line<EOL>for p, process in enumerate(<EOL>list(self.processes.keys())[top:bottom]):<EOL><INDENT>if process not in self.process_stats:<EOL><INDENT>vals = ["<STR_LIT:->"] * <NUM_LIT:8><EOL>txt_fmt = curses.A_NORMAL<EOL><DEDENT>else:<EOL><INDENT>ref = self.process_stats[process]<EOL>vals = [ref["<STR_LIT>"],<EOL>len(self.processes[process]["<STR_LIT>"]),<EOL>ref["<STR_LIT>"],<EOL>ref["<STR_LIT>"], ref["<STR_LIT>"],<EOL>ref["<STR_LIT>"]]<EOL>txt_fmt = curses.A_BOLD<EOL><DEDENT>proc = self.processes[process]<EOL>if proc["<STR_LIT>"]:<EOL><INDENT>completed = "<STR_LIT>".format(len(proc["<STR_LIT>"]),<EOL>len(proc["<STR_LIT>"]))<EOL><DEDENT>else:<EOL><INDENT>completed = "<STR_LIT:{}>".format(len(proc["<STR_LIT>"]))<EOL><DEDENT>win.addstr(<EOL><NUM_LIT:4> + p, <NUM_LIT:0>, "<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(<EOL>proc["<STR_LIT>"],<EOL>process,<EOL>completed,<EOL>*vals),<EOL>curses.color_pair(colors[proc["<STR_LIT>"]]) | txt_fmt)<EOL><DEDENT>win.clrtoeol()<EOL>win.refresh(<NUM_LIT:0>, self.padding, <NUM_LIT:0>, <NUM_LIT:0>, height-<NUM_LIT:1>, width-<NUM_LIT:1>)<EOL>

Displays the default overview of the pipeline execution from the
        :attr:`status_info`, :attr:`processes` and :attr:`run_status`
        attributes into stdout.


with open(self.log_file) as fh:<EOL><INDENT>last_lines = fh.readlines()[-n:]<EOL><DEDENT>return last_lines<EOL>

Returns a list with the last ``n`` lines of the nextflow log file

        Parameters
        ----------
        n : int
            Number of last lines from the log file

        Returns
        -------
        list
            List of strings with the nextflow log


pipeline_files = {}<EOL>with open(join(self.workdir, self.pipeline_name)) as fh:<EOL><INDENT>pipeline_files["<STR_LIT>"] = fh.readlines()<EOL><DEDENT>nf_config = join(self.workdir, "<STR_LIT>")<EOL>if os.path.exists(nf_config):<EOL><INDENT>with open(nf_config) as fh:<EOL><INDENT>pipeline_files["<STR_LIT>"] = fh.readlines()<EOL><DEDENT><DEDENT>configs = {<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>}<EOL>for config, key in configs.items():<EOL><INDENT>cfile = join(self.workdir, config)<EOL>if os.path.exists(cfile):<EOL><INDENT>with open(cfile) as fh:<EOL><INDENT>pipeline_files[key] = fh.readlines()<EOL><DEDENT><DEDENT><DEDENT>return pipeline_files<EOL>

Prepares the first batch of information, containing static
        information such as the pipeline file, and configuration files

        Returns
        -------
        dict
            Dict with the static information for the first POST request


try:<EOL><INDENT>dag_file = open(os.path.join(self.workdir, "<STR_LIT>"))<EOL>dag_json = json.load(dag_file)<EOL><DEDENT>except (FileNotFoundError, json.decoder.JSONDecodeError):<EOL><INDENT>logger.warning(colored_print(<EOL>"<STR_LIT>",<EOL>"<STR_LIT>"))<EOL>dag_json = {}<EOL><DEDENT>return dag_json<EOL>

Function that opens the dotfile named .treeDag.json in the current
        working directory

        Returns
        -------
        Returns a dictionary with the dag object to be used in the post
        instance available through the method _establish_connection


<EOL>pipeline_path = get_nextflow_filepath(self.log_file)<EOL>pipeline_hash = hashlib.md5()<EOL>with open(pipeline_path, "<STR_LIT:rb>") as fh:<EOL><INDENT>for chunk in iter(lambda: fh.read(<NUM_LIT>), b"<STR_LIT>"):<EOL><INDENT>pipeline_hash.update(chunk)<EOL><DEDENT><DEDENT>workdir = self.workdir.encode("<STR_LIT:utf8>")<EOL>hostname = socket.gethostname().encode("<STR_LIT:utf8>")<EOL>hardware_addr = str(uuid.getnode()).encode("<STR_LIT:utf8>")<EOL>dir_hash = hashlib.md5(workdir + hostname + hardware_addr)<EOL>return pipeline_hash.hexdigest() + dir_hash.hexdigest()<EOL>


if sys.stdout.encoding != "<STR_LIT>":<EOL><INDENT>msg = "<STR_LIT>".join([i if ord(i) < <NUM_LIT> else "<STR_LIT>" for i in msg])<EOL><DEDENT>try:<EOL><INDENT>col = COLORS[color_label]<EOL><DEDENT>except KeyError:<EOL><INDENT>col = color_label<EOL><DEDENT>return "<STR_LIT>".format(col, msg)<EOL>

This function enables users to add a color to the print. It also enables
to pass end_char to print allowing to print several strings in the same line
in different prints.

Parameters
----------
color_string: str
    The color code to pass to the function, which enables color change as
    well as background color change.
msg: str
    The actual text to be printed
end_char: str
    The character in which each print should finish. By default it will be
    "\n".


logger.info(colored_print(<EOL>"<STR_LIT>", "<STR_LIT>"))<EOL>procs_dict_ordered = {k: procs_dict[k] for k in sorted(procs_dict)}<EOL>for template, dict_proc_info in procs_dict_ordered.items():<EOL><INDENT>template_str = "<STR_LIT>".format(template)<EOL>logger.info(colored_print(template_str, "<STR_LIT>"))<EOL>for info in dict_proc_info:<EOL><INDENT>info_str = "<STR_LIT>".format(info)<EOL>if isinstance(dict_proc_info[info], list):<EOL><INDENT>if not dict_proc_info[info]:<EOL><INDENT>arg_msg = "<STR_LIT:None>"<EOL><DEDENT>else:<EOL><INDENT>arg_msg = "<STR_LIT:U+002CU+0020>".join(dict_proc_info[info])<EOL><DEDENT><DEDENT>elif info == "<STR_LIT>":<EOL><INDENT>if not dict_proc_info[info]:<EOL><INDENT>arg_msg = "<STR_LIT:None>"<EOL><DEDENT>else:<EOL><INDENT>list_msg = ["<STR_LIT>".format(<EOL>templt,<EOL>"<STR_LIT>".join(["<STR_LIT>".format(dr, val)<EOL>for dr, val in drs.items()]))<EOL>for templt, drs in dict_proc_info[info].items()<EOL>]<EOL>arg_msg = "<STR_LIT>".join(list_msg)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>arg_msg = dict_proc_info[info]<EOL><DEDENT>logger.info("<STR_LIT>".format(<EOL>colored_print(info_str, "<STR_LIT>"), arg_msg<EOL>))<EOL><DEDENT><DEDENT>

This function handles the dictionary of attributes of each Process class
to print to stdout lists of all the components or the components which the
user specifies in the -t flag.

Parameters
----------
procs_dict: dict
    A dictionary with the class attributes for all the components (or
    components that are used by the -t flag), that allow to create
    both the short_list and detailed_list. Dictionary example:
    {"abyss": {'input_type': 'fastq', 'output_type': 'fasta',
    'dependencies': [], 'directives': {'abyss': {'cpus': 4,
    'memory': '{ 5.GB * task.attempt }', 'container': 'flowcraft/abyss',
    'version': '2.1.1', 'scratch': 'true'}}}


def proc_collector(process_map, args, pipeline_string):

arguments_list = []<EOL>if args.detailed_list:<EOL><INDENT>arguments_list += [<EOL>"<STR_LIT>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT:description>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT>",<EOL>"<STR_LIT>"<EOL>]<EOL><DEDENT>if args.short_list:<EOL><INDENT>arguments_list += [<EOL>"<STR_LIT:description>"<EOL>]<EOL><DEDENT>if arguments_list:<EOL><INDENT>procs_dict = {}<EOL>for name, cls in process_map.items():<EOL><INDENT>cls_inst = cls(template=name)<EOL>if pipeline_string:<EOL><INDENT>if name not in pipeline_string:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>d = {arg_key: vars(cls_inst)[arg_key] for arg_key in<EOL>vars(cls_inst) if arg_key in arguments_list}<EOL>procs_dict[name] = d<EOL><DEDENT>procs_dict_parser(procs_dict)<EOL>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>

Function that collects all processes available and stores a dictionary of
the required arguments of each process class to be passed to
procs_dict_parser

Parameters
----------
process_map: dict
    The dictionary with the Processes currently available in flowcraft
    and their corresponding classes as values
args: argparse.Namespace
    The arguments passed through argparser that will be access to check the
    type of list to be printed
pipeline_string: str
    the pipeline string


s1 = re.sub('<STR_LIT>', r'<STR_LIT>', name)<EOL>return re.sub('<STR_LIT>', r'<STR_LIT>', s1).lower()<EOL>

Convers a CamelCase string into a snake_case one

    Parameters
    ----------
    name : str
        An arbitrary string that may be CamelCase

    Returns
    -------
    str
        The input string converted into snake_case


process_map = {}<EOL>prefix = "<STR_LIT>".format(components.__name__)<EOL>for importer, modname, _ in pkgutil.iter_modules(components.__path__,<EOL>prefix):<EOL><INDENT>_module = importer.find_module(modname).load_module(modname)<EOL>_component_classes = [<EOL>cls for cls in _module.__dict__.values() if<EOL>isinstance(cls, type) and cls.__name__ != "<STR_LIT>"<EOL>]<EOL>for cls in _component_classes:<EOL><INDENT>process_map[convert_camel_case(cls.__name__)] = cls<EOL><DEDENT><DEDENT>return process_map<EOL>

Collects Process classes and return dict mapping templates to classes

    This function crawls through the components module and retrieves all
    classes that inherit from the Process class. Then, it converts the name
    of the classes (which should be CamelCase) to snake_case, which is used
    as the template name.

    Returns
    -------
    dict
        Dictionary mapping the template name (snake_case) to the corresponding
        process class.


save_list = []<EOL>for process in process_map:<EOL><INDENT>similarity = SequenceMatcher(None, process, query_str)<EOL>if similarity.ratio() > <NUM_LIT:0.5>:<EOL><INDENT>save_list.append(process)<EOL><DEDENT><DEDENT>if save_list:<EOL><INDENT>logger.info(colored_print(<EOL>"<STR_LIT>".format("<STR_LIT>".join(save_list)), "<STR_LIT>"))<EOL><DEDENT>logger.info(colored_print("<STR_LIT>"<EOL>"<STR_LIT>", "<STR_LIT>"))<EOL>

Function to guess processes based on strings that are not available in
process_map. If the string has typos and is somewhat similar (50%) to any
process available in flowcraft it will print info to the terminal,
suggesting the most similar processes available in flowcraft.

Parameters
----------
query_str: str
    The string of the process with potential typos
process_map:
    The dictionary that contains all the available processes


n = <NUM_LIT:1>  <EOL>while n:<EOL><INDENT>text, n = re.subn(r'<STR_LIT>', '<STR_LIT>', text)<EOL><DEDENT>return text<EOL>

Recursively removes nested brackets

    This function is used to remove nested brackets from fork strings using
    regular expressions

    Parameters
    ----------
    text: str
        The string that contains brackets with inner forks to be removed

    Returns
    -------
    text: str
        the string with only the processes that are not in inner forks, thus
        the processes that belong to a given fork.


if p_string.strip() == "<STR_LIT>":<EOL><INDENT>raise SanityError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>

Function to check if pipeline string is empty or has an empty string

Parameters
----------
p_string: str
     String with the definition of the pipeline, e.g.::
         'processA processB processC(ProcessD | ProcessE)'


if "<STR_LIT:|>" in p_string and "<STR_LIT:(>" not in p_string:<EOL><INDENT>raise SanityError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>

Function to check if a LANE_TOKEN is provided but no fork is initiated.
Parameters
----------
p_string: str
     String with the definition of the pipeline, e.g.::
         'processA processB processC(ProcessD | ProcessE)'


if p_string.count(FORK_TOKEN) != p_string.count(CLOSE_TOKEN):<EOL><INDENT>dict_values = {<EOL>FORK_TOKEN: p_string.count(FORK_TOKEN),<EOL>CLOSE_TOKEN: p_string.count(CLOSE_TOKEN)<EOL>}<EOL>max_bracket = max(dict_values, key=dict_values.get)<EOL>raise SanityError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(<EOL>str(abs(<EOL>p_string.count(FORK_TOKEN) - p_string.count(CLOSE_TOKEN))),<EOL>max_bracket))<EOL><DEDENT>

This function performs a check for different number of '(' and ')'
characters, which indicates that some forks are poorly constructed.

Parameters
----------
p_string: str
     String with the definition of the pipeline, e.g.::
         'processA processB processC(ProcessD | ProcessE)'


if LANE_TOKEN + LANE_TOKEN in p_string:<EOL><INDENT>raise SanityError("<STR_LIT>")<EOL><DEDENT>

This function performs a sanity check for multiple '|' character
between two processes.

Parameters
----------
p_string: str
     String with the definition of the pipeline, e.g.::
         'processA processB processC(ProcessD | ProcessE)'


<EOL>if p_string.endswith(LANE_TOKEN):<EOL><INDENT>raise SanityError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>

This function checks if lane token is the last element of the pipeline
string.

Parameters
----------
p_string: str
     String with the definition of the pipeline, e.g.::
         'processA processB processC(ProcessD | ProcessE)'


<EOL>if FORK_TOKEN + LANE_TOKEN in p_string orLANE_TOKEN + CLOSE_TOKEN in p_string orLANE_TOKEN + FORK_TOKEN in p_string:<EOL><INDENT>raise SanityError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>

This function checks if the pipeline string contains a process between
the fork start token or end token and the separator (lane) token. Checks for
the absence of processes in one of the branches of the fork ['|)' and '(|']
and for the existence of a process before starting a fork (in an inner fork)
['|('].

Parameters
----------
p_string: str
     String with the definition of the pipeline, e.g.::
         'processA processB processC(ProcessD | ProcessE)'


if FORK_TOKEN + FORK_TOKEN in p_string:<EOL><INDENT>raise SanityError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>

This function checks if there is a starting process after the beginning of
each fork. It checks for duplicated start tokens ['(('].

Parameters
----------
p_string: str
     String with the definition of the pipeline, e.g.::
         'processA processB processC(ProcessD | ProcessE)'


if re.search('<STR_LIT>'.format(CLOSE_TOKEN), p_string):<EOL><INDENT>raise SanityError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>

This function checks if there are processes after the close token. It
searches for everything that isn't "|" or ")" after a ")" token.

Parameters
----------
p_string: str
     String with the definition of the pipeline, e.g.::
         'processA processB processC(ProcessD | ProcessE)'


<EOL>list_of_forks = []  <EOL>left_indexes = []  <EOL>for pos, char in enumerate(pipeline_string):<EOL><INDENT>if char == FORK_TOKEN:<EOL><INDENT>left_indexes.append(pos)<EOL><DEDENT>elif char == CLOSE_TOKEN and len(left_indexes) > <NUM_LIT:0>:<EOL><INDENT>list_of_forks.append(pipeline_string[left_indexes[-<NUM_LIT:1>] + <NUM_LIT:1>: pos])<EOL>left_indexes = left_indexes[:-<NUM_LIT:1>]<EOL><DEDENT><DEDENT>list_of_forks.sort(key=lambda x: x.count(FORK_TOKEN), reverse=True)<EOL>for fork in list_of_forks:<EOL><INDENT>for subfork in list_of_forks:<EOL><INDENT>if subfork in list_of_forks and subfork != fork:<EOL><INDENT>fork_simplified = fork.replace("<STR_LIT>".format(subfork), "<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>fork_simplified = fork<EOL><DEDENT><DEDENT>if not len(fork_simplified.split(LANE_TOKEN)) > <NUM_LIT:1>:<EOL><INDENT>raise SanityError("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(fork))<EOL><DEDENT><DEDENT>

This function performs two sanity checks in the pipeline string. The first
check, assures that each fork contains a lane token '|', while the second
check looks for duplicated processes within the same fork.

Parameters
----------
pipeline_string: str
     String with the definition of the pipeline, e.g.::
         'processA processB processC(ProcessD | ProcessE)'


<EOL>p_string = pipeline_str.replace("<STR_LIT:U+0020>", "<STR_LIT>").strip()<EOL>checks = [<EOL>[p_string, [<EOL>empty_tasks,<EOL>brackets_but_no_lanes,<EOL>brackets_insanity_check,<EOL>lane_char_insanity_check,<EOL>final_char_insanity_check,<EOL>fork_procs_insanity_check,<EOL>start_proc_insanity_check,<EOL>late_proc_insanity_check<EOL>]],<EOL>[pipeline_str, [<EOL>inner_fork_insanity_checks<EOL>]]<EOL>]<EOL>for param, func_list in checks:<EOL><INDENT>for func in func_list:<EOL><INDENT>func(param)<EOL><DEDENT><DEDENT>

Wrapper that performs all sanity checks on the pipeline string

    Parameters
    ----------
    pipeline_str : str
        String with the pipeline definition