def __init__(self, config, base_path, output_file=None):

self.config = config<EOL>self.base_path = base_path<EOL>self._output_needs_closing = False<EOL>if output_file:<EOL><INDENT>self.output_file = output_file<EOL><DEDENT>elif self.config.get('<STR_LIT:file>'):<EOL><INDENT>self.output_file = click.open_file(<EOL>self.config['<STR_LIT:file>'],<EOL>mode='<STR_LIT:w>',<EOL>encoding='<STR_LIT:utf-8>',<EOL>)<EOL>self._output_needs_closing = True<EOL><DEDENT>else:<EOL><INDENT>self.output_file = click.get_text_stream('<STR_LIT>')<EOL><DEDENT>

:param config:
    the configuration used during the analysis of the project
:type config: dict
:param base_path: the path to the project base directory
:type base_path: str


Produces the contents of the report.

Must be implemented by concrete classes.

:param collector: the collection of issues to report on
:type collector: tidypy.Collector


return Path(filename).relative_to(self.base_path).as_posix()<EOL>

Generates a path for the specified filename that is relative to the
project path.

:param filename: the filename to generate the path for
:type filename: str
:rtype: str


click.echo(text_type(msg), nl=newline, file=self.output_file)<EOL>

Writes the specified string to the output target of the report.

:param msg: the message to output.
:type msg: str
:param newline:
    whether or not to append a newline to the end of the message
:type newline: str


self.base_path = Path(base_path).resolve()<EOL>self.excludes = compile_masks(config['<STR_LIT>'])<EOL>self._found = dict()<EOL>self._find(self.base_path)<EOL>self._found = dict([<EOL>(dirname, files)<EOL>for dirname, files in iteritems(self._found)<EOL>if files<EOL>])<EOL>

:param base_path: the path to the base of the project
:type base_path: str
:param config: the configuration to use when searching the project
:type config: dict


return text_type(self.base_path)<EOL>

The path to the project that this Finder is operating from.


return text_type(Path(filepath).relative_to(self.base_path))<EOL>

Reformats a file path to be relative to this Finder's project path.

:param filepath: the path to reformat
:type filepath: str or pathlib.Path
:rtype: str


relpath = path.relative_to(self.base_path).as_posix()<EOL>return matches_masks(relpath, self.excludes)<EOL>

Determines whether or not the specified file is excluded by the
project's configuration.

:param path: the path to check
:type path: pathlib.Path
:rtype: bool


if self.is_excluded(path):<EOL><INDENT>return True<EOL><DEDENT>return matches_masks(path.name, ALWAYS_EXCLUDED_DIRS)<EOL>

Determines whether or not the specified directory is excluded by the
project's configuration.

:param path: the path to check
:type path: pathlib.Path
:rtype: bool


filters = compile_masks(filters or [r'<STR_LIT>'])<EOL>for files in itervalues(self._found):<EOL><INDENT>for file_ in files:<EOL><INDENT>relpath = text_type(Path(file_).relative_to(self.base_path))<EOL>if matches_masks(relpath, filters):<EOL><INDENT>yield file_<EOL><DEDENT><DEDENT><DEDENT>

A generator that produces a sequence of paths to files in the project
that matches the specified filters.

:param filters:
    the regular expressions to use when finding files in the project.
    If not specified, all files are returned.
:type filters: list(str)


def directories(self, filters=None, containing=None):

filters = compile_masks(filters or [r'<STR_LIT>'])<EOL>contains = compile_masks(containing)<EOL>for dirname, files in iteritems(self._found):<EOL><INDENT>relpath = text_type(Path(dirname).relative_to(self.base_path))<EOL>if matches_masks(relpath, filters):<EOL><INDENT>if not contains or self._contains(files, contains):<EOL><INDENT>yield dirname<EOL><DEDENT><DEDENT><DEDENT>

A generator that produces a sequence of paths to directories in the
project that matches the specified filters.

:param filters:
    the regular expressions to use when finding directories in the
    project. If not specified, all directories are returned.
:type filters: list(str)
:param containing:
    if a directory passes through the specified filters, it is checked
    for the presence of a file that matches one of the regular
    expressions in this parameter.
:type containing: list(str)


return self.directories(<EOL>filters=filters,<EOL>containing=[r'<STR_LIT>'],<EOL>)<EOL>

A generator that produces a sequence of paths to directories that look
to be Python packages (e.g., they contain an ``__init__.py``).

:param filters:
    the regular expressions to use when finding directories in the
    project. If not specified, all directories are returned.
:type filters: list(str)


masks = compile_masks(r'<STR_LIT>')<EOL>for file_ in self.files(filters=filters):<EOL><INDENT>if matches_masks(file_, masks):<EOL><INDENT>yield file_<EOL><DEDENT><DEDENT>

A generator that produces a sequence of paths to files that look to be
Python modules (e.g., ``*.py``).

:param filters:
    the regular expressions to use when finding files in the project.
    If not specified, all files are returned.
:type filters: list(str)


paths = set()<EOL>packages = list(self.packages(filters=filters))<EOL>for module in self.modules(filters=filters):<EOL><INDENT>parent = text_type(Path(module).parent)<EOL>if parent not in packages:<EOL><INDENT>paths.add(parent)<EOL><DEDENT><DEDENT>paths.update(self.topmost_directories([<EOL>text_type(Path(package).parent)<EOL>for package in packages<EOL>]))<EOL>return list(paths)<EOL>

Produces a list of paths that would be suitable to use in ``sys.path``
in order to access the Python modules/packages found in this project.

:param filters:
    the regular expressions to use when finding files in the project.
    If not specified, all files are returned.
:type filters: list(str)


return read_file(filepath)<EOL>

Retrieves the contents of the specified file.

This function performs simple caching so that the same file isn't read
more than once per process.

:param filepath: the file to read.
:type filepath: str
:rtype: str


timedelta = datetime.timedelta(seconds=int(seconds))<EOL>mm, ss = divmod(timedelta.seconds, <NUM_LIT>)<EOL>if mm < <NUM_LIT>:<EOL><INDENT>return "<STR_LIT>" % (mm, ss)<EOL><DEDENT>hh, mm = divmod(mm, <NUM_LIT>)<EOL>if hh < <NUM_LIT>:<EOL><INDENT>return "<STR_LIT>" % (hh, mm, ss)<EOL><DEDENT>dd, hh = divmod(mm, <NUM_LIT>)<EOL>return "<STR_LIT>" % (dd, hh, mm, ss)<EOL>


if pbar.currval == <NUM_LIT:0>:<EOL><INDENT>return self.eta_format % '<STR_LIT>'<EOL><DEDENT>elif pbar.finished:<EOL><INDENT>return self.complete_format % self.format_time(pbar.seconds_elapsed)<EOL><DEDENT>else:<EOL><INDENT>elapsed = pbar.seconds_elapsed<EOL>eta = elapsed * pbar.maxval / pbar.currval - elapsed<EOL>return self.eta_format % self.format_time(eta)<EOL><DEDENT>

Updates the widget to show the ETA or total time when finished.


loop = GeventLoop(interval, callback)<EOL>loop.start()<EOL>try:<EOL><INDENT>yield loop<EOL><DEDENT>finally:<EOL><INDENT>if loop.has_started():<EOL><INDENT>loop.stop()<EOL><DEDENT><DEDENT>

When entering the context, spawns a greenlet that sleeps for `interval` seconds between `callback` executions.
When leaving the context stops the greenlet.
The yielded object is the `GeventLoop` object so the loop can be stopped from within the context.

For example:
```
with loop_in_background(60.0, purge_cache) as purge_cache_job:
    ...
    ...
    if should_stop_cache():
        purge_cache_job.stop()
```


raise NotImplementedError()<EOL>

Subclasses should implement this function - called every loop iteration.


assert not self.has_started(), "<STR_LIT>"<EOL>self._stop_event = Event()<EOL>self._greenlet = gevent.spawn(self._loop)<EOL>

Starts the loop. Calling a running loop is an error.


assert self.has_started(), "<STR_LIT>"<EOL>greenlet = self._greenlet if gevent.getcurrent() != self._greenlet else None<EOL>self._stop_event.set()<EOL>if greenlet is not None and timeout is not None:<EOL><INDENT>greenlet.join(timeout)<EOL>return greenlet.ready<EOL><DEDENT>else:<EOL><INDENT>return True<EOL><DEDENT>

Stops a running loop and waits for it to end if timeout is set. Calling a non-running loop is an error.
:param timeout: time (in seconds) to wait for the loop to end after signalling it. ``None`` is to block till it
ends.
:return: True if the loop stopped, False if still stopping.


assert self.has_started(), "<STR_LIT>"<EOL>self._stop_event.set()<EOL>self._greenlet.kill()<EOL>self._clear()<EOL>

Kills the running loop and waits till it gets killed.


return self._greenlet is not None<EOL>


def encode(python_object, indent=None, large_object=False):

<EOL>kwargs = dict(indent=indent)<EOL>if can_dumps_sort_keys():<EOL><INDENT>kwargs.update(sort_keys=True)<EOL><DEDENT>try:<EOL><INDENT>if large_object:<EOL><INDENT>out = GreenletFriendlyStringIO()<EOL>json.dump(python_object, out, **kwargs)<EOL>return out.getvalue()<EOL><DEDENT>else:<EOL><INDENT>return json.dumps(python_object, **kwargs)<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>logger.exception()<EOL>raise EncodeError('<STR_LIT>', python_object)<EOL><DEDENT>

:returns: a JSON-representation of the object


try:<EOL><INDENT>return json.loads(json_string)<EOL><DEDENT>except:<EOL><INDENT>raise DecodeError()<EOL><DEDENT>


def fopen(name, mode='<STR_LIT:r>', buffering=-<NUM_LIT:1>):

f = _fopen(name, mode, buffering)<EOL>return _FileObjectThreadWithContext(f, mode, buffering)<EOL>

Similar to Python's built-in `open()` function.


return gevent.spawn(wrap_uncaught_greenlet_exceptions(func), *args, **kwargs)<EOL>

spawns a greenlet that does not print exceptions to the screen.
    if you use this function you MUST use this module's join or joinall otherwise the exception will be lost


def wrapped_func(*args, **kwargs):<EOL><INDENT>try:<EOL><INDENT>result = original_func(*args, **kwargs)<EOL>return True, result<EOL><DEDENT>except:<EOL><INDENT>return False, sys.exc_info()<EOL><DEDENT><DEDENT>def new_func(*args, **kwargs):<EOL><INDENT>status, result = gevent.get_hub().threadpool.apply(wrapped_func, args, kwargs)<EOL>if status:<EOL><INDENT>return result<EOL><DEDENT>else:<EOL><INDENT>six.reraise(*result)<EOL><DEDENT><DEDENT>new_func.__name__ = original_func.__name__<EOL>new_func.__doc__ = "<STR_LIT>" + ("<STR_LIT:U+0020>" + original_func.__doc__ if original_func.__doc__ is not None else "<STR_LIT>")<EOL>return new_func<EOL>

Returns a function wrapper that defers function calls execute inside gevent's threadpool but keeps any exception
or backtrace in the caller's context.
:param original_func: function to wrap
:returns: wrapper function


global _uncaught_exception_handler<EOL>prev_handler, _uncaught_exception_handler = _uncaught_exception_handler, func<EOL>return prev_handler<EOL>

Sets a global greenlet uncaught exception handler that will get called if a greenlet spawned by one of this module's
wrappers raises an uncaught exception.
:param func: exception handler function that will receive the exc_info tuple as an argument
:returns: previous exception handler function or None


if not hasattr(instance_method, '<STR_LIT>'):<EOL><INDENT>return instance_method<EOL><DEDENT>def _func(*args, **kwargs):<EOL><INDENT>return instance_method(*args, **kwargs)<EOL><DEDENT>_func.__name__ = repr(instance_method)<EOL>return _func<EOL>

wraps(instancemethod) returns a function, not an instancemethod so its repr() is all messed up;
we want the original repr to show up in the logs, therefore we do this trick


def safe_joinall(greenlets, timeout=None, raise_error=False):

greenlets = list(greenlets)<EOL>try:<EOL><INDENT>gevent.joinall(greenlets, timeout=timeout, raise_error=raise_error)<EOL><DEDENT>except gevent.GreenletExit:<EOL><INDENT>[greenlet.kill() for greenlet in greenlets if not greenlet.ready()]<EOL>raise<EOL><DEDENT>return greenlets<EOL>

Wrapper for gevent.joinall if the greenlet that waits for the joins is killed, it kills all the greenlets it
joins for.


key = make_key(<EOL>event=self.object.event,<EOL>owner_name=self.object.owner.username,<EOL>identifier=self.object.identifier<EOL>)<EOL>return redis.lrange(key, <NUM_LIT:0>, <NUM_LIT:20>)<EOL>


@job<EOL>def worker(wrapped, dkwargs, hash_value=None, *args, **kwargs):

if "<STR_LIT>" not in dkwargs:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg)<EOL><DEDENT>event = dkwargs['<STR_LIT>']<EOL>if "<STR_LIT>" not in kwargs:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg)<EOL><DEDENT>owner = kwargs['<STR_LIT>']<EOL>if "<STR_LIT>" not in kwargs:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg)<EOL><DEDENT>identifier = kwargs['<STR_LIT>']<EOL>senderobj = DjangoRQSenderable(<EOL>wrapped, dkwargs, hash_value, WEBHOOK_ATTEMPTS, *args, **kwargs<EOL>)<EOL>senderobj.webhook_target = WebhookTarget.objects.get(<EOL>event=event,<EOL>owner=owner,<EOL>identifier=identifier<EOL>)<EOL>senderobj.url = senderobj.webhook_target.target_url<EOL>senderobj.payload = senderobj.get_payload()<EOL>senderobj.payload['<STR_LIT>'] = getattr(kwargs['<STR_LIT>'], WEBHOOK_OWNER_FIELD)<EOL>senderobj.payload['<STR_LIT>'] = dkwargs['<STR_LIT>']<EOL>return senderobj.send()<EOL>

This is an asynchronous sender callable that uses the Django ORM to store
    webhooks. Redis is used to handle the message queue.

dkwargs argument requires the following key/values:

    :event: A string representing an event.

kwargs argument requires the following key/values

    :owner: The user who created/owns the event


def redislog_callable(wrapped, dkwargs, hash_value=None, *args, **kwargs):

if "<STR_LIT>" not in dkwargs:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg)<EOL><DEDENT>event = dkwargs['<STR_LIT>']<EOL>if "<STR_LIT>" not in kwargs:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg)<EOL><DEDENT>owner = kwargs['<STR_LIT>']<EOL>if "<STR_LIT>" not in kwargs:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg)<EOL><DEDENT>identifier = kwargs['<STR_LIT>']<EOL>senderobj = RedisLogSenderable(<EOL>wrapped, dkwargs, hash_value, WEBHOOK_ATTEMPTS, *args, **kwargs<EOL>)<EOL>try:<EOL><INDENT>senderobj.webhook_target = WebhookTarget.objects.get(<EOL>event=event,<EOL>owner=owner,<EOL>identifier=identifier<EOL>)<EOL><DEDENT>except WebhookTarget.DoesNotExist:<EOL><INDENT>return {"<STR_LIT:error>": "<STR_LIT>"}<EOL><DEDENT>senderobj.url = senderobj.webhook_target.target_url<EOL>senderobj.payload = senderobj.get_payload()<EOL>senderobj.payload['<STR_LIT>'] = getattr(kwargs['<STR_LIT>'], WEBHOOK_OWNER_FIELD)<EOL>senderobj.payload['<STR_LIT>'] = dkwargs['<STR_LIT>']<EOL>return senderobj.send()<EOL>

This is a synchronous sender callable that uses the Django ORM to store
    webhooks and Redis for the delivery log.

dkwargs argument requires the following key/values:

    :event: A string representing an event.

kwargs argument requires the following key/values

    :owner: The user who created/owns the event


def __init__(self, wrapped, dkwargs, hash_value, attempts, *args, **kwargs):

self.event = dkwargs['<STR_LIT>']<EOL>self.owner = kwargs['<STR_LIT>']<EOL>self.identifier = kwargs['<STR_LIT>']<EOL>super(RedisLogSenderable, self).__init__(wrapped, dkwargs, hash_value, attempts, *args, **kwargs)<EOL>

:wrapped: Function that has been wrapped and whose payload is being sent


data = dict(<EOL>payload=self.payload,<EOL>attempt=self.attempt,<EOL>success=self.success,<EOL>response_message=self.response_content,<EOL>hash_value=self.hash_value,<EOL>response_status=self.response.status_code,<EOL>notification=message,<EOL>created=timezone.now()<EOL>)<EOL>value = json.dumps(data, cls=StandardJSONEncoder)<EOL>key = make_key(self.event, self.owner.username, self.identifier)<EOL>redis.lpush(key, value)<EOL>

TODO: Add code to lpush to redis stack
        rpop when stack hits size 'X'


def orm_callable(wrapped, dkwargs, hash_value=None, *args, **kwargs):

if "<STR_LIT>" not in dkwargs:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg)<EOL><DEDENT>event = dkwargs['<STR_LIT>']<EOL>if "<STR_LIT>" not in kwargs:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg)<EOL><DEDENT>owner = kwargs['<STR_LIT>']<EOL>if "<STR_LIT>" not in kwargs:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg)<EOL><DEDENT>identifier = kwargs['<STR_LIT>']<EOL>senderobj = DjangoSenderable(<EOL>wrapped, dkwargs, hash_value, WEBHOOK_ATTEMPTS, *args, **kwargs<EOL>)<EOL>try:<EOL><INDENT>senderobj.webhook_target = WebhookTarget.objects.get(<EOL>event=event,<EOL>owner=owner,<EOL>identifier=identifier<EOL>)<EOL><DEDENT>except WebhookTarget.DoesNotExist:<EOL><INDENT>return {"<STR_LIT:error>": "<STR_LIT>"}<EOL><DEDENT>senderobj.url = senderobj.webhook_target.target_url<EOL>senderobj.payload = senderobj.get_payload()<EOL>senderobj.payload['<STR_LIT>'] = getattr(kwargs['<STR_LIT>'], WEBHOOK_OWNER_FIELD)<EOL>senderobj.payload['<STR_LIT>'] = dkwargs['<STR_LIT>']<EOL>return senderobj.send()<EOL>

This is a synchronous sender callable that uses the Django ORM to store
    webhooks and the delivery log. Meant as proof of concept and not
    as something for heavy production sites.

dkwargs argument requires the following key/values:

    :event: A string representing an event.

kwargs argument requires the following key/values

    :owner: The user who created/owns the event


if sys.version > '<STR_LIT:3>':<EOL><INDENT>return value.decode('<STR_LIT:utf-8>')<EOL><DEDENT>return value<EOL>

Regardless of the Python version, this always returns a string


if events is None:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise ImproperlyConfigured(msg)<EOL><DEDENT>try:<EOL><INDENT>choices = [(x, x) for x in events]<EOL><DEDENT>except TypeError:<EOL><INDENT>"""<STR_LIT>"""<EOL>msg = "<STR_LIT>"<EOL>raise ImproperlyConfigured(msg)<EOL><DEDENT>return choices<EOL>


class Memodict(dict):<EOL><INDENT>"""<STR_LIT>"""<EOL>def __getitem__(self, *key):<EOL><INDENT>return dict.__getitem__(self, key)<EOL><DEDENT>def __missing__(self, key):<EOL><INDENT>"""<STR_LIT>"""<EOL>ret = self[key] = func(*key)<EOL>return ret<EOL><DEDENT><DEDENT>return Memodict().__getitem__<EOL>

Memoization decorator for a function taking one or more arguments.


resos = []<EOL>if any([i in list("<STR_LIT>") for i in seq]):<EOL><INDENT>for base in list("<STR_LIT>"):<EOL><INDENT>if base in seq:<EOL><INDENT>resos.append(seq.replace(base, AMBIGS[base][<NUM_LIT:0>]))<EOL>resos.append(seq.replace(base, AMBIGS[base][<NUM_LIT:1>]))<EOL><DEDENT><DEDENT>return resos<EOL><DEDENT>else:<EOL><INDENT>return [seq, "<STR_LIT>"]<EOL><DEDENT>

Returns both resolutions of a cut site that has an ambiguous base in
it, else the single cut site


<EOL>allele1 = list(consensus)<EOL>allele2 = list(consensus)<EOL>hidx = [i for (i, j) in enumerate(consensus) if j in "<STR_LIT>"]<EOL>for idx in hidx:<EOL><INDENT>hsite = consensus[idx]<EOL>if hsite.isupper():<EOL><INDENT>allele1[idx] = PRIORITY[hsite]<EOL>allele2[idx] = MINOR[hsite]<EOL><DEDENT>else:<EOL><INDENT>allele1[idx] = MINOR[hsite.upper()]<EOL>allele2[idx] = PRIORITY[hsite.upper()]<EOL><DEDENT><DEDENT>allele1 = "<STR_LIT>".join(allele1)<EOL>allele2 = "<STR_LIT>".join(allele2)<EOL>return allele1, allele2<EOL>

takes diploid consensus alleles with phase data stored as a mixture
    of upper and lower case characters and splits it into 2 alleles


<EOL>return seq.replace("<STR_LIT:A>", '<STR_LIT:t>').replace('<STR_LIT:T>', '<STR_LIT:a>').replace('<STR_LIT:C>', '<STR_LIT:g>').replace('<STR_LIT>', '<STR_LIT:c>').replace('<STR_LIT:n>', '<STR_LIT>').upper().replace("<STR_LIT>", "<STR_LIT:n>")<EOL>

returns a seq with complement. Preserves little n's for splitters.


<EOL>seq = seq.replace("<STR_LIT:A>", '<STR_LIT:u>').replace('<STR_LIT:T>', '<STR_LIT:v>').replace('<STR_LIT:C>', '<STR_LIT:p>').replace('<STR_LIT>', '<STR_LIT:z>').replace('<STR_LIT:u>', '<STR_LIT:T>').replace('<STR_LIT:v>', '<STR_LIT:A>').replace('<STR_LIT:p>', '<STR_LIT>').replace('<STR_LIT:z>', '<STR_LIT:C>')<EOL>seq = seq.replace('<STR_LIT:R>', '<STR_LIT:u>').replace('<STR_LIT>', '<STR_LIT:v>').replace('<STR_LIT:Y>', '<STR_LIT:b>').replace('<STR_LIT:M>', '<STR_LIT:o>').replace('<STR_LIT:u>', '<STR_LIT:Y>').replace('<STR_LIT:v>', '<STR_LIT:M>').replace('<STR_LIT:b>', '<STR_LIT:R>').replace('<STR_LIT:o>', '<STR_LIT>')<EOL>seq = seq.replace('<STR_LIT:r>', '<STR_LIT:u>').replace('<STR_LIT:k>', '<STR_LIT:v>').replace('<STR_LIT:y>', '<STR_LIT:b>').replace('<STR_LIT:m>', '<STR_LIT:o>').replace('<STR_LIT:u>', '<STR_LIT:y>').replace('<STR_LIT:v>', '<STR_LIT:m>').replace('<STR_LIT:b>', '<STR_LIT:r>').replace('<STR_LIT:o>', '<STR_LIT:k>')<EOL>return seq<EOL>

returns complement of sequence including ambiguity characters,
    and saves lower case info for multiple hetero sequences


def fastq_touchup_for_vsearch_merge(read, outfile, reverse=False):

counts = <NUM_LIT:0><EOL>with open(outfile, '<STR_LIT:w>') as out:<EOL><INDENT>if read.endswith("<STR_LIT>"):<EOL><INDENT>fr1 = gzip.open(read, '<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>fr1 = open(read, '<STR_LIT:rb>')<EOL><DEDENT>quarts = itertools.izip(*[iter(fr1)]*<NUM_LIT:4>)<EOL>writing = []<EOL>while <NUM_LIT:1>:<EOL><INDENT>try:<EOL><INDENT>lines = quarts.next()<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>if reverse:<EOL><INDENT>seq = lines[<NUM_LIT:1>].strip()[::-<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>seq = lines[<NUM_LIT:1>].strip()<EOL><DEDENT>writing.append("<STR_LIT>".join([<EOL>lines[<NUM_LIT:0>],<EOL>seq+"<STR_LIT:\n>",<EOL>lines[<NUM_LIT:2>],<EOL>"<STR_LIT:B>"*len(seq)<EOL>]))<EOL>counts += <NUM_LIT:1><EOL>if not counts % <NUM_LIT:1000>:<EOL><INDENT>out.write("<STR_LIT:\n>".join(writing)+"<STR_LIT:\n>")<EOL>writing = []<EOL><DEDENT><DEDENT>if writing:<EOL><INDENT>out.write("<STR_LIT:\n>".join(writing))<EOL><DEDENT><DEDENT>out.close()<EOL>fr1.close()<EOL>

option to change orientation of reads and sets Qscore to B


def merge_pairs_after_refmapping(data, two_files, merged_out):

<EOL>nonmerged1 = tempfile.NamedTemporaryFile(<EOL>mode='<STR_LIT:wb>',<EOL>dir=data.dirs.edits,<EOL>suffix="<STR_LIT>").name<EOL>nonmerged2 = tempfile.NamedTemporaryFile(<EOL>mode='<STR_LIT:wb>',<EOL>dir=data.dirs.edits,<EOL>suffix="<STR_LIT>").name <EOL>minlen = str(max(<NUM_LIT:32>, data.paramsdict["<STR_LIT>"]))<EOL>try:<EOL><INDENT>maxn = sum(data.paramsdict['<STR_LIT>'])<EOL><DEDENT>except TypeError:<EOL><INDENT>maxn = data.paramsdict['<STR_LIT>']<EOL><DEDENT>tmp1 = two_files[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>tmp2 = two_files[<NUM_LIT:0>][<NUM_LIT:1>]<EOL>fastq_touchup_for_vsearch_merge(tmp1, tmp1+"<STR_LIT>", False)<EOL>fastq_touchup_for_vsearch_merge(tmp2, tmp2+"<STR_LIT>", True)<EOL>cmd = [ipyrad.bins.vsearch,<EOL>"<STR_LIT>", tmp1+"<STR_LIT>",<EOL>"<STR_LIT>", tmp2+"<STR_LIT>",<EOL>"<STR_LIT>", merged_out,<EOL>"<STR_LIT>", nonmerged1,<EOL>"<STR_LIT>", nonmerged2,<EOL>"<STR_LIT>", "<STR_LIT:0>",<EOL>"<STR_LIT>", minlen,<EOL>"<STR_LIT>", str(maxn),<EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT:4>",<EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT:2>",<EOL>"<STR_LIT>"]<EOL>proc = sps.Popen(cmd, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>try:<EOL><INDENT>res = proc.communicate()[<NUM_LIT:0>]<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>proc.kill()<EOL><DEDENT>if proc.returncode:<EOL><INDENT>LOGGER.error("<STR_LIT>", cmd, res)<EOL>raise IPyradWarningExit("<STR_LIT>", cmd, res)<EOL><DEDENT>with open(merged_out, '<STR_LIT:r>') as tmpf:<EOL><INDENT>nmerged = sum(<NUM_LIT:1> for i in tmpf.readlines()) // <NUM_LIT:4><EOL><DEDENT>with open(merged_out, '<STR_LIT>') as combout:<EOL><INDENT>fr1 = open(nonmerged1, '<STR_LIT:rb>')<EOL>quart1 = itertools.izip(*[iter(fr1)]*<NUM_LIT:4>)<EOL>fr2 = open(nonmerged2, '<STR_LIT:rb>')<EOL>quart2 = itertools.izip(*[iter(fr2)]*<NUM_LIT:4>)<EOL>quarts = itertools.izip(quart1, quart2)<EOL>writing = []<EOL>counts = <NUM_LIT:0><EOL>while <NUM_LIT:1>:<EOL><INDENT>try:<EOL><INDENT>read1s, read2s = quarts.next()<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>writing.append("<STR_LIT>".join([<EOL>read1s[<NUM_LIT:0>],<EOL>read1s[<NUM_LIT:1>].strip() + "<STR_LIT>" +read2s[<NUM_LIT:1>], <EOL>read1s[<NUM_LIT:2>],<EOL>read1s[<NUM_LIT:3>].strip() + "<STR_LIT>" +read2s[<NUM_LIT:3>], <EOL>]))<EOL>counts += <NUM_LIT:1><EOL>if not counts % <NUM_LIT:10>:<EOL><INDENT>combout.write("<STR_LIT>".join(writing))<EOL>writing = []<EOL><DEDENT><DEDENT>if writing:<EOL><INDENT>combout.write("<STR_LIT>".join(writing))<EOL><DEDENT>fr1.close()<EOL>fr2.close()<EOL>combout.close()<EOL><DEDENT>rmfiles = [nonmerged1, nonmerged2, tmp1, tmp2, tmp1+"<STR_LIT>", tmp2+"<STR_LIT>"]<EOL>for rmfile in rmfiles:<EOL><INDENT>if os.path.exists(rmfile):<EOL><INDENT>os.remove(rmfile)<EOL><DEDENT><DEDENT>return nmerged<EOL>

A function to merge fastq files produced by bam2fq.


try:<EOL><INDENT>r1file = tempfile.NamedTemporaryFile(mode='<STR_LIT:wb>', delete=False,<EOL>dir=data.dirs.edits,<EOL>suffix="<STR_LIT>")<EOL>r2file = tempfile.NamedTemporaryFile(mode='<STR_LIT:wb>', delete=False,<EOL>dir=data.dirs.edits,<EOL>suffix="<STR_LIT>")<EOL>r1dat = []<EOL>r2dat = []<EOL>for locus in clust:<EOL><INDENT>sname, seq = locus.split("<STR_LIT:\n>")<EOL>sname = "<STR_LIT:@>" + sname[<NUM_LIT:1>:]<EOL>r1, r2 = seq.split("<STR_LIT>")<EOL>r1dat.append("<STR_LIT>".format(sname, r1, "<STR_LIT:+>", "<STR_LIT:B>"*(len(r1))))<EOL>r2dat.append("<STR_LIT>".format(sname, r2, "<STR_LIT:+>", "<STR_LIT:B>"*(len(r2))))<EOL><DEDENT>r1file.write("<STR_LIT:\n>".join(r1dat))<EOL>r2file.write("<STR_LIT:\n>".join(r2dat))<EOL>r1file.close()<EOL>r2file.close()<EOL>merged_file = tempfile.NamedTemporaryFile(mode='<STR_LIT:wb>',<EOL>dir=data.dirs.edits,<EOL>suffix="<STR_LIT>").name<EOL>clust = []<EOL>merge_pairs(data, [(r1file.name, r2file.name)], merged_file, <NUM_LIT:0>, <NUM_LIT:1>)<EOL>with open(merged_file) as infile:<EOL><INDENT>quarts = itertools.izip(*[iter(infile)]*<NUM_LIT:4>)<EOL>while <NUM_LIT:1>:<EOL><INDENT>try:<EOL><INDENT>sname, seq, _, _ = quarts.next()<EOL>if not "<STR_LIT>" in sname.rsplit("<STR_LIT:;>", <NUM_LIT:1>)[<NUM_LIT:1>]:<EOL><INDENT>try:<EOL><INDENT>R1, R2 = seq.split("<STR_LIT>")<EOL>seq = R1 + "<STR_LIT>" + revcomp(R2)<EOL><DEDENT>except ValueError as inst:<EOL><INDENT>LOGGER.error("<STR_LIT>".format(sname, seq))<EOL>raise<EOL><DEDENT><DEDENT><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>sname = "<STR_LIT:>>" + sname[<NUM_LIT:1>:]<EOL>clust.extend([sname.strip(), seq.strip()])<EOL><DEDENT><DEDENT><DEDENT>except:<EOL><INDENT>LOGGER.info("<STR_LIT>")<EOL>raise<EOL><DEDENT>finally:<EOL><INDENT>for i in [r1file.name, r2file.name, merged_file]:<EOL><INDENT>if os.path.exists(i):<EOL><INDENT>log_level = logging.getLevelName(LOGGER.getEffectiveLevel())<EOL>os.remove(i)<EOL><DEDENT><DEDENT><DEDENT>return clust<EOL>

This is for pysam post-flight merging. The input is a cluster
for an individual locus. We have to split the clusters, write
R1 and R2 to files then call merge_pairs(). This is not ideal,
it's slow, but it works. This is the absolute worst way to do this,
it bounces all the files for each locus off the disk. I/O _hog_.


def merge_pairs(data, two_files, merged_out, revcomp, merge):

LOGGER.debug("<STR_LIT>")<EOL>nmerged = -<NUM_LIT:1><EOL>for fhandle in two_files[<NUM_LIT:0>]:<EOL><INDENT>if not os.path.exists(fhandle):<EOL><INDENT>raise IPyradWarningExit("""<STR_LIT>""".format(fhandle))<EOL><DEDENT><DEDENT>if os.path.exists(merged_out):<EOL><INDENT>os.remove(merged_out)<EOL><DEDENT>if merge:<EOL><INDENT>nonmerged1 = tempfile.NamedTemporaryFile(mode='<STR_LIT:wb>',<EOL>dir=data.dirs.edits,<EOL>suffix="<STR_LIT>").name<EOL>nonmerged2 = tempfile.NamedTemporaryFile(mode='<STR_LIT:wb>',<EOL>dir=data.dirs.edits,<EOL>suffix="<STR_LIT>").name<EOL><DEDENT>else:<EOL><INDENT>nonmerged1 = two_files[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>nonmerged2 = two_files[<NUM_LIT:0>][<NUM_LIT:1>]<EOL><DEDENT>try:<EOL><INDENT>maxn = sum(data.paramsdict['<STR_LIT>'])<EOL><DEDENT>except TypeError:<EOL><INDENT>maxn = data.paramsdict['<STR_LIT>']<EOL><DEDENT>minlen = str(max(<NUM_LIT:32>, data.paramsdict["<STR_LIT>"]))<EOL>if merge and two_files[<NUM_LIT:0>][<NUM_LIT:0>].endswith("<STR_LIT>"):<EOL><INDENT>LOGGER.info("<STR_LIT>")<EOL>tmp1 = os.path.splitext(two_files[<NUM_LIT:0>][<NUM_LIT:0>])[<NUM_LIT:0>]+"<STR_LIT>"<EOL>tmp2 = os.path.splitext(two_files[<NUM_LIT:0>][<NUM_LIT:1>])[<NUM_LIT:0>]+"<STR_LIT>"<EOL>out1 = open(tmp1, '<STR_LIT:w>')<EOL>out2 = open(tmp2, '<STR_LIT:w>')<EOL>gun1 = sps.Popen(["<STR_LIT>", "<STR_LIT:-c>", two_files[<NUM_LIT:0>][<NUM_LIT:0>]],<EOL>stderr=sps.STDOUT, stdout=out1, close_fds=True)<EOL>gun2 = sps.Popen(["<STR_LIT>", "<STR_LIT:-c>", two_files[<NUM_LIT:0>][<NUM_LIT:1>]],<EOL>stderr=sps.STDOUT, stdout=out2, close_fds=True)<EOL>_ = gun1.communicate()<EOL>_ = gun2.communicate()<EOL>out1.close()<EOL>out2.close()<EOL><DEDENT>else:<EOL><INDENT>tmp1 = two_files[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>tmp2 = two_files[<NUM_LIT:0>][<NUM_LIT:1>]<EOL><DEDENT>try:<EOL><INDENT>if merge:<EOL><INDENT>cmd = [ipyrad.bins.vsearch,<EOL>"<STR_LIT>", tmp1,<EOL>"<STR_LIT>", tmp2,<EOL>"<STR_LIT>", merged_out,<EOL>"<STR_LIT>", nonmerged1,<EOL>"<STR_LIT>", nonmerged2,<EOL>"<STR_LIT>", "<STR_LIT:0>",<EOL>"<STR_LIT>", minlen,<EOL>"<STR_LIT>", str(maxn),<EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT:4>",<EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT:2>",<EOL>"<STR_LIT>"]<EOL>LOGGER.debug("<STR_LIT>", "<STR_LIT:U+0020>".join(cmd))<EOL>proc = sps.Popen(cmd, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>try:<EOL><INDENT>res = proc.communicate()[<NUM_LIT:0>]<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>proc.kill()<EOL><DEDENT>if proc.returncode:<EOL><INDENT>LOGGER.error("<STR_LIT>", cmd, res)<EOL>rmfiles = [os.path.splitext(two_files[<NUM_LIT:0>][<NUM_LIT:0>])[<NUM_LIT:0>]+"<STR_LIT>",<EOL>os.path.splitext(two_files[<NUM_LIT:0>][<NUM_LIT:1>])[<NUM_LIT:0>]+"<STR_LIT>",<EOL>nonmerged1, nonmerged2]<EOL>for rmfile in rmfiles:<EOL><INDENT>if os.path.exists(rmfile):<EOL><INDENT>os.remove(rmfile)<EOL><DEDENT><DEDENT>raise IPyradWarningExit("<STR_LIT>", cmd, res)<EOL><DEDENT>with open(merged_out, '<STR_LIT:r>') as tmpf:<EOL><INDENT>nmerged = sum(<NUM_LIT:1> for i in tmpf.readlines()) // <NUM_LIT:4><EOL><DEDENT><DEDENT>with open(merged_out, '<STR_LIT>') as combout:<EOL><INDENT>if nonmerged1.endswith("<STR_LIT>"):<EOL><INDENT>fr1 = gzip.open(nonmerged1, '<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>fr1 = open(nonmerged1, '<STR_LIT:rb>')<EOL><DEDENT>quart1 = itertools.izip(*[iter(fr1)]*<NUM_LIT:4>)<EOL>if nonmerged2.endswith("<STR_LIT>"):<EOL><INDENT>fr2 = gzip.open(nonmerged2, '<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>fr2 = open(nonmerged2, '<STR_LIT:rb>')<EOL><DEDENT>quart2 = itertools.izip(*[iter(fr2)]*<NUM_LIT:4>)<EOL>quarts = itertools.izip(quart1, quart2)<EOL>writing = []<EOL>counts = <NUM_LIT:0><EOL>while <NUM_LIT:1>:<EOL><INDENT>try:<EOL><INDENT>read1s, read2s = quarts.next()<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>if revcomp:<EOL><INDENT>writing.append("<STR_LIT>".join([<EOL>read1s[<NUM_LIT:0>],<EOL>read1s[<NUM_LIT:1>].strip() + "<STR_LIT>" +comp(read2s[<NUM_LIT:1>].strip()[::-<NUM_LIT:1>]) + "<STR_LIT:\n>",<EOL>read1s[<NUM_LIT:2>],<EOL>read1s[<NUM_LIT:3>].strip() + "<STR_LIT>" +read2s[<NUM_LIT:3>].strip()[::-<NUM_LIT:1>] + "<STR_LIT:\n>",<EOL>]))<EOL><DEDENT>else:<EOL><INDENT>writing.append("<STR_LIT>".join([<EOL>read1s[<NUM_LIT:0>],<EOL>read1s[<NUM_LIT:1>].strip() + "<STR_LIT>" +read2s[<NUM_LIT:1>],<EOL>read1s[<NUM_LIT:2>],<EOL>read1s[<NUM_LIT:3>].strip() + "<STR_LIT>" +read2s[<NUM_LIT:3>],<EOL>]))<EOL><DEDENT>counts += <NUM_LIT:1><EOL>if not counts % <NUM_LIT:10>:<EOL><INDENT>combout.write("<STR_LIT>".join(writing)) <EOL>writing = []<EOL><DEDENT><DEDENT>if writing:<EOL><INDENT>combout.write("<STR_LIT>".join(writing))<EOL><DEDENT><DEDENT>fr1.close()<EOL>fr2.close()<EOL>combout.close()<EOL><DEDENT>except Exception as inst:<EOL><INDENT>LOGGER.error("<STR_LIT>".format(inst))<EOL>raise<EOL><DEDENT>finally:<EOL><INDENT>if merge:<EOL><INDENT>rmfiles = [nonmerged1, nonmerged2,<EOL>os.path.splitext(two_files[<NUM_LIT:0>][<NUM_LIT:0>])[<NUM_LIT:0>]+"<STR_LIT>",<EOL>os.path.splitext(two_files[<NUM_LIT:0>][<NUM_LIT:1>])[<NUM_LIT:0>]+"<STR_LIT>"]<EOL>for rmfile in rmfiles:<EOL><INDENT>if os.path.exists(rmfile):<EOL><INDENT>os.remove(rmfile)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return nmerged<EOL>

Merge PE reads. Takes in a list of unmerged files [r1, r2] and the
filehandle to write merged data to, and it returns the number of reads
that were merged (overlapping). If merge==0 then only concat pairs (nnnn),
no merging in vsearch.

Parameters
-----------
two_files (tuple):
    A list or tuple of the [r1, r2] files to be merged.

merged_out (str):
    A string file handle for the merged data to be written to.

revcomp (bool):
    Whether or not to revcomp the R2s.

merge (bool):
    Whether or not to perform vsearch merging. If not then reads are simply
    concatenated with a 'nnnn' separator.

Returns
--------
If merge is on then the func will return the number of pairs
successfully merged, else it returns -1.


sequence = sequence[::-<NUM_LIT:1>].strip().replace("<STR_LIT:A>", "<STR_LIT:t>").replace("<STR_LIT:T>", "<STR_LIT:a>").replace("<STR_LIT:C>", "<STR_LIT:g>").replace("<STR_LIT>", "<STR_LIT:c>").upper()<EOL>return sequence<EOL>


amb = amb.upper()<EOL>return AMBIGS.get(amb)<EOL>


amb = amb.upper()<EOL>return DUCT.get(amb)<EOL>

This is copied from pyrad.alignable, and is referenced in
    several of the loci2*.py conversion modules. It duplicates some
    of the effort of unhetero(), but i guess it's fine for now. Probably
    could merge these two functions if you wanted to.


ccnt = <NUM_LIT:0><EOL>chunk = []<EOL>while ccnt < optim:<EOL><INDENT>try:<EOL><INDENT>taker = itertools.takewhile(lambda x: x[<NUM_LIT:0>] != "<STR_LIT>", pairdealer)<EOL>oneclust = ["<STR_LIT>".join(taker.next())]<EOL><DEDENT>except StopIteration:<EOL><INDENT>return <NUM_LIT:1>, chunk<EOL><DEDENT>while <NUM_LIT:1>:<EOL><INDENT>try:<EOL><INDENT>oneclust.append("<STR_LIT>".join(taker.next()))<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT><DEDENT>chunk.append("<STR_LIT>".join(oneclust))<EOL>ccnt += <NUM_LIT:1><EOL><DEDENT>return <NUM_LIT:0>, chunk<EOL>

return optim clusters given iterators, and whether it got all or not


def progressbar(njobs, finished, msg="<STR_LIT>", spacer="<STR_LIT:U+0020>"):

if njobs:<EOL><INDENT>progress = <NUM_LIT:100>*(finished / float(njobs))<EOL><DEDENT>else:<EOL><INDENT>progress = <NUM_LIT:100><EOL><DEDENT>hashes = '<STR_LIT:#>'*int(progress/<NUM_LIT>)<EOL>nohash = '<STR_LIT:U+0020>'*int(<NUM_LIT:20>-len(hashes))<EOL>if not ipyrad.__interactive__:<EOL><INDENT>msg = msg.rsplit("<STR_LIT:|>", <NUM_LIT:2>)[<NUM_LIT:0>]<EOL><DEDENT>args = [spacer, hashes+nohash, int(progress), msg]<EOL>print("<STR_LIT>".format(*args), end="<STR_LIT>")<EOL>sys.stdout.flush()<EOL>


<EOL>eids = ipyclient.ids<EOL>dview = ipyclient.direct_view()<EOL>hosts = dview.apply_sync(socket.gethostname)<EOL>hostdict = defaultdict(list)<EOL>for host, eid in zip(hosts, eids):<EOL><INDENT>hostdict[host].append(eid)<EOL><DEDENT>hostdictkeys = hostdict.keys()<EOL>for key in hostdictkeys:<EOL><INDENT>gids = hostdict[key]<EOL>maxt = len(gids)<EOL>if len(gids) >= <NUM_LIT:4>:<EOL><INDENT>maxt = <NUM_LIT:2><EOL><DEDENT>if (len(gids) == <NUM_LIT:4>) and (len(hosts) >= <NUM_LIT:4>):<EOL><INDENT>maxt = <NUM_LIT:4><EOL><DEDENT>if len(gids) >= <NUM_LIT:6>:<EOL><INDENT>maxt = <NUM_LIT:3><EOL><DEDENT>if len(gids) >= <NUM_LIT:8>:<EOL><INDENT>maxt = <NUM_LIT:4><EOL><DEDENT>if len(gids) >= <NUM_LIT:16>:<EOL><INDENT>maxt = <NUM_LIT:4><EOL><DEDENT>threaded = [gids[i:i+maxt] for i in xrange(<NUM_LIT:0>, len(gids), maxt)]<EOL>lth = len(threaded)<EOL>if lth > <NUM_LIT:1>:<EOL><INDENT>hostdict.pop(key)<EOL>for hostid in range(lth):<EOL><INDENT>hostdict[str(key)+"<STR_LIT:_>"+str(hostid)] = threaded[hostid]<EOL><DEDENT><DEDENT><DEDENT>LOGGER.info("<STR_LIT>", dict(hostdict))<EOL>return hostdict<EOL>

gets optimum threaded view of ids given the host setup


<EOL>if hasattr(os, "<STR_LIT>"):<EOL><INDENT>if os.sysconf_names.has_key("<STR_LIT>"):<EOL><INDENT>ncpus = os.sysconf("<STR_LIT>")<EOL>if isinstance(ncpus, int) and ncpus > <NUM_LIT:0>:<EOL><INDENT>return ncpus<EOL><DEDENT><DEDENT>else: <EOL><INDENT>return int(os.popen2("<STR_LIT>")[<NUM_LIT:1>].read())<EOL><DEDENT><DEDENT>if os.environ.has_key("<STR_LIT>"):<EOL><INDENT>ncpus = int(os.environ["<STR_LIT>"])<EOL>if ncpus > <NUM_LIT:0>:<EOL><INDENT>return ncpus<EOL><DEDENT><DEDENT>return <NUM_LIT:1><EOL>

Detects the number of CPUs on a system. This is better than asking
ipyparallel since ipp has to wait for Engines to spin up.


def likelihood1(errors, bfreqs, ustacks):

<EOL>totals = np.array([ustacks.sum(axis=<NUM_LIT:1>)]*<NUM_LIT:4>).T<EOL>prob = scipy.stats.binom.pmf(totals-ustacks, totals, errors)<EOL>lik1 = np.sum(bfreqs*prob, axis=<NUM_LIT:1>)<EOL>return lik1<EOL>


<EOL>tots = np.empty((ustacks.shape[<NUM_LIT:0>], <NUM_LIT:1>))<EOL>twos = np.empty((ustacks.shape[<NUM_LIT:0>], <NUM_LIT:6>))<EOL>thrs = np.empty((ustacks.shape[<NUM_LIT:0>], <NUM_LIT:6>, <NUM_LIT:2>))<EOL>for idx in xrange(ustacks.shape[<NUM_LIT:0>]):<EOL><INDENT>ust = ustacks[idx]<EOL>tot = ust.sum()<EOL>tots[idx] = tot<EOL>i = <NUM_LIT:0><EOL>for jdx in xrange(<NUM_LIT:4>):<EOL><INDENT>for kdx in xrange(<NUM_LIT:4>):<EOL><INDENT>if jdx < kdx:<EOL><INDENT>twos[idx][i] = tot - ust[jdx] - ust[kdx]<EOL>thrs[idx][i] = ust[jdx], ust[jdx] + ust[kdx]<EOL>i += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>return tots, twos, thrs<EOL>

JIT'd function builds large array that can be used to calc binom pmf


def lik2_calc(err, one, tots, twos, thrs, four):

<EOL>_twos = scipy.stats.binom.pmf(twos, tots, <NUM_LIT:0.5>)<EOL>_thrs = thrs.reshape(thrs.shape[<NUM_LIT:0>] * thrs.shape[<NUM_LIT:1>], thrs.shape[<NUM_LIT:2>])<EOL>_thrs = scipy.stats.binom.pmf(_thrs[:, <NUM_LIT:0>], _thrs[:, <NUM_LIT:1>], (<NUM_LIT>*err) / <NUM_LIT>)<EOL>_thrs = _thrs.reshape(thrs.shape[<NUM_LIT:0>], <NUM_LIT:6>)<EOL>return np.sum(one * _twos * (_thrs / four), axis=<NUM_LIT:1>)<EOL>

vectorized calc of binom pmf on large arrays


def nlikelihood2(errors, bfreqs, ustacks):

one = [<NUM_LIT> * bfreqs[i] * bfreqs[j] for i, j in itertools.combinations(range(<NUM_LIT:4>), <NUM_LIT:2>)]<EOL>four = <NUM_LIT:1.> - np.sum(bfreqs**<NUM_LIT:2>) <EOL>tots, twos, thrs = nblik2_build(ustacks)<EOL>res2 = lik2_calc(errors, one, tots, twos, thrs, four)<EOL>return res2<EOL>

calls nblik2_build and lik2_calc for a given err


def nget_diploid_lik(pstart, bfreqs, ustacks, counts):

hetero, errors = pstart<EOL>if (hetero <= <NUM_LIT:0.>) or (errors <= <NUM_LIT:0.>):<EOL><INDENT>score = np.exp(<NUM_LIT:100>)<EOL><DEDENT>else:<EOL><INDENT>lik1 = (<NUM_LIT:1.>-hetero) * likelihood1(errors, bfreqs, ustacks)<EOL>lik2 = (hetero) * nlikelihood2(errors, bfreqs, ustacks)<EOL>liks = lik1 + lik2<EOL>logliks = np.log(liks[liks > <NUM_LIT:0>]) * counts[liks > <NUM_LIT:0>]<EOL>score = -logliks.sum()<EOL><DEDENT>return score<EOL>


def get_haploid_lik(errors, bfreqs, ustacks, counts):

hetero = <NUM_LIT:0.><EOL>if errors <= <NUM_LIT:0.>:<EOL><INDENT>score = np.exp(<NUM_LIT:100>)<EOL><DEDENT>else:<EOL><INDENT>lik1 = ((<NUM_LIT:1.>-hetero)*likelihood1(errors, bfreqs, ustacks)) <EOL>liks = lik1<EOL>logliks = np.log(liks[liks > <NUM_LIT:0>])*counts[liks > <NUM_LIT:0>]<EOL>score = -logliks.sum()<EOL><DEDENT>return score<EOL>


<EOL>majrdepth = data.paramsdict["<STR_LIT>"]<EOL>statdepth = data.paramsdict["<STR_LIT>"]    <EOL>maxlen = data._hackersonly["<STR_LIT>"]<EOL>if not hasattr(sample.stats_dfs.s3, "<STR_LIT>"):<EOL><INDENT>sample.stats_dfs.s3["<STR_LIT>"] = data.paramsdict["<STR_LIT>"]<EOL><DEDENT>if <NUM_LIT:1>: <EOL><INDENT>LOGGER.info("<STR_LIT>")<EOL>maxlens, depths = get_quick_depths(data, sample)<EOL>hidepths = depths >= majrdepth<EOL>stathidepths = depths >= statdepth<EOL>keepmj = depths[hidepths]<EOL>keepst = depths[stathidepths]<EOL>try:<EOL><INDENT>statlens = maxlens[stathidepths]<EOL>statlen = int(statlens.mean() + (<NUM_LIT>*statlens.std()))<EOL><DEDENT>except:<EOL><INDENT>raise IPyradError("<STR_LIT>")<EOL><DEDENT>LOGGER.info("<STR_LIT>", maxlens.shape, maxlens.mean(), maxlens.std())<EOL>maxlens = maxlens[hidepths]<EOL>maxlen = int(maxlens.mean() + (<NUM_LIT>*maxlens.std()))<EOL>sample.stats["<STR_LIT>"] = keepmj.shape[<NUM_LIT:0>]<EOL>sample.stats_dfs.s3["<STR_LIT>"] = keepmj.shape[<NUM_LIT:0>]        <EOL><DEDENT>return sample, keepmj.shape[<NUM_LIT:0>], maxlen, keepst.shape[<NUM_LIT:0>], statlen<EOL>

if mindepth setting were changed then 'clusters_hidepth' needs to be
recalculated. Check and recalculate if necessary.


<EOL>sample, _, _, nhidepth, maxlen = recal_hidepth(data, sample)<EOL>clusters = gzip.open(sample.files.clusters)<EOL>pairdealer = itertools.izip(*[iter(clusters)]*<NUM_LIT:2>)<EOL>dims = (nhidepth, maxlen, <NUM_LIT:4>)<EOL>stacked = np.zeros(dims, dtype=np.uint64)<EOL>cutlens = [None, None]<EOL>try:<EOL><INDENT>cutlens[<NUM_LIT:0>] = len(data.paramsdict["<STR_LIT>"][<NUM_LIT:0>])<EOL>cutlens[<NUM_LIT:1>] = maxlen - len(data.paramsdict["<STR_LIT>"][<NUM_LIT:1>])<EOL><DEDENT>except TypeError:<EOL><INDENT>pass<EOL><DEDENT>nclust = <NUM_LIT:0><EOL>done = <NUM_LIT:0><EOL>while not done:<EOL><INDENT>try:<EOL><INDENT>done, chunk = clustdealer(pairdealer, <NUM_LIT:1>)<EOL><DEDENT>except IndexError:<EOL><INDENT>raise IPyradError("<STR_LIT>", chunk)<EOL><DEDENT>if chunk:<EOL><INDENT>piece = chunk[<NUM_LIT:0>].strip().split("<STR_LIT:\n>")<EOL>names = piece[<NUM_LIT:0>::<NUM_LIT:2>]<EOL>seqs = piece[<NUM_LIT:1>::<NUM_LIT:2>]<EOL>reps = [int(sname.split("<STR_LIT:;>")[-<NUM_LIT:2>][<NUM_LIT:5>:]) for sname in names]<EOL>sseqs = [list(seq) for seq in seqs]<EOL>arrayed = np.concatenate(<EOL>[[seq]*rep for seq, rep in zip(sseqs, reps)])<EOL>if arrayed.shape[<NUM_LIT:0>] >= data.paramsdict["<STR_LIT>"]:<EOL><INDENT>arrayed = arrayed[:<NUM_LIT>, cutlens[<NUM_LIT:0>]:cutlens[<NUM_LIT:1>]]<EOL>arrayed = arrayed[:, ~np.any(arrayed == "<STR_LIT:n>", axis=<NUM_LIT:0>)]<EOL>arrayed[arrayed == "<STR_LIT:->"] = "<STR_LIT:N>"<EOL>arrayed = arrayed[:, ~np.all(arrayed == "<STR_LIT:N>", axis=<NUM_LIT:0>)]<EOL>catg = np.array([np.sum(arrayed == i, axis=<NUM_LIT:0>) for i in list("<STR_LIT>")], <EOL>dtype=np.uint64).T<EOL>stacked[nclust, :catg.shape[<NUM_LIT:0>], :] = catg<EOL>nclust += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>newstack = stacked[stacked.sum(axis=<NUM_LIT:2>) > <NUM_LIT:0>]<EOL>assert not np.any(newstack.sum(axis=<NUM_LIT:1>) == <NUM_LIT:0>), "<STR_LIT>"<EOL>clusters.close()<EOL>return newstack<EOL>


hetero = <NUM_LIT><EOL>errors = <NUM_LIT><EOL>success = False<EOL>try:<EOL><INDENT>stacked = stackarray(data, sample)<EOL>bfreqs = stacked.sum(axis=<NUM_LIT:0>) / float(stacked.sum())<EOL>if np.isnan(bfreqs).any():<EOL><INDENT>raise IPyradWarningExit("<STR_LIT>".format(sample.name, bfreqs))<EOL><DEDENT>tstack = Counter([j.tostring() for j in stacked])<EOL>ustacks = np.array([np.fromstring(i, dtype=np.uint64)for i in tstack.iterkeys()])<EOL>counts = np.array(tstack.values())<EOL>del tstack<EOL>if int(data.paramsdict["<STR_LIT>"]) == <NUM_LIT:1>:<EOL><INDENT>pstart = np.array([<NUM_LIT>], dtype=np.float64)<EOL>hetero = <NUM_LIT:0.><EOL>errors = scipy.optimize.fmin(get_haploid_lik, pstart,<EOL>(bfreqs, ustacks, counts),<EOL>maxfun=<NUM_LIT:50>,<EOL>maxiter=<NUM_LIT:50>,<EOL>disp=False,<EOL>full_output=False)<EOL><DEDENT>else:<EOL><INDENT>pstart = np.array([<NUM_LIT>, <NUM_LIT>], dtype=np.float64)<EOL>hetero, errors = scipy.optimize.fmin(nget_diploid_lik, pstart,<EOL>(bfreqs, ustacks, counts),<EOL>maxfun=<NUM_LIT:50>,<EOL>maxiter=<NUM_LIT:50>,<EOL>disp=False,<EOL>full_output=False)<EOL><DEDENT>success = True<EOL><DEDENT>except IPyradError as inst:<EOL><INDENT>LOGGER.debug("<STR_LIT>".format(sample.name))<EOL>pass<EOL><DEDENT>return hetero, errors, success<EOL>

func scipy optimize to find best parameters


def run(data, samples, force, ipyclient):

<EOL>if data.paramsdict["<STR_LIT>"] == <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>".format(data._spacer))<EOL><DEDENT>subsamples = []<EOL>for sample in samples:<EOL><INDENT>if not force:<EOL><INDENT>if sample.stats.state >= <NUM_LIT:4>:<EOL><INDENT>print("<STR_LIT>".format(sample.name)+"<STR_LIT>")<EOL><DEDENT>elif sample.stats.state < <NUM_LIT:3>:<EOL><INDENT>print("<STR_LIT>".format(sample.name)+"<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>subsamples.append(sample)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if sample.stats.state < <NUM_LIT:3>:<EOL><INDENT>print("<STR_LIT:U+0020>"+sample.name+"<STR_LIT>")<EOL><DEDENT>elif sample.stats.clusters_hidepth < <NUM_LIT:2>:<EOL><INDENT>print("<STR_LIT>".format(sample.name, sample.stats.clusters_hidepth))<EOL><DEDENT>else:<EOL><INDENT>subsamples.append(sample)<EOL><DEDENT><DEDENT><DEDENT>if subsamples:<EOL><INDENT>submit(data, subsamples, ipyclient)<EOL><DEDENT>


def submit(data, subsamples, ipyclient):

<EOL>subsamples.sort(key=lambda x: x.stats.clusters_hidepth, reverse=True)<EOL>lbview = ipyclient.load_balanced_view()<EOL>jobs = {}<EOL>for sample in subsamples:<EOL><INDENT>jobs[sample.name] = lbview.apply(optim, *(data, sample))<EOL><DEDENT>start = time.time() <EOL>printstr = "<STR_LIT>"<EOL>try:<EOL><INDENT>kbd = <NUM_LIT:0><EOL>while <NUM_LIT:1>:<EOL><INDENT>fin = [i.ready() for i in jobs.values()]<EOL>elapsed = datetime.timedelta(seconds=int(time.time() - start))<EOL>progressbar(len(fin), sum(fin), printstr.format(elapsed), spacer=data._spacer)<EOL>time.sleep(<NUM_LIT:0.1>)<EOL>if len(fin) == sum(fin):<EOL><INDENT>print("<STR_LIT>")<EOL>break<EOL><DEDENT><DEDENT>for job in jobs:<EOL><INDENT>if jobs[job].successful():<EOL><INDENT>hest, eest, success = jobs[job].result()<EOL>sample_cleanup(data.samples[job], hest, eest, success)<EOL><DEDENT>else:<EOL><INDENT>LOGGER.error("<STR_LIT>", <EOL>job, jobs[job].exception())<EOL>raise IPyradWarningExit(jobs[job].result())<EOL><DEDENT><DEDENT><DEDENT>except KeyboardInterrupt as kbd:<EOL><INDENT>pass<EOL><DEDENT>finally:<EOL><INDENT>assembly_cleanup(data)<EOL>if kbd:<EOL><INDENT>raise KeyboardInterrupt<EOL><DEDENT><DEDENT>

Sends jobs to engines and cleans up failures. Print progress.


def sample_cleanup(sample, hest, eest, success):

<EOL>sample.stats.state = <NUM_LIT:4><EOL>sample.stats.hetero_est = float(hest)<EOL>sample.stats.error_est = float(eest)<EOL>sample.stats_dfs.s4.hetero_est = float(hest)<EOL>sample.stats_dfs.s4.error_est = float(eest)<EOL>if not success:<EOL><INDENT>msg = """<STR_LIT>""".format(sample.name)<EOL>print(msg)<EOL><DEDENT>

Stores results to the Assembly object, writes to stats file,
and cleans up temp files


<EOL>data.stats_dfs.s4 = data._build_stat("<STR_LIT>")<EOL>data.stats_files.s4 = os.path.join(data.dirs.clusts, <EOL>"<STR_LIT>")<EOL>with io.open(data.stats_files.s4, '<STR_LIT:w>') as outfile:<EOL><INDENT>data.stats_dfs.s4.to_string(outfile)<EOL><DEDENT>fails = data.stats[data.stats["<STR_LIT:state>"] == <NUM_LIT:3>].index.values<EOL>if fails:<EOL><INDENT>msg = """<STR_LIT>""".format(fails)<EOL>print(msg)<EOL><DEDENT>


umap1file = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>umap2file = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>unmapped = os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>")<EOL>samplesam = os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>")<EOL>split1 = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>split2 = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>refmap_derep = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>for f in [umap1file, umap2file, unmapped, samplesam, split1, split2, refmap_derep]:<EOL><INDENT>try:<EOL><INDENT>os.remove(f)<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>


<EOL>refseq_file = data.paramsdict['<STR_LIT>']<EOL>index_files = []<EOL>if "<STR_LIT>" in data._hackersonly["<STR_LIT>"]:<EOL><INDENT>index_files.extend(["<STR_LIT>", "<STR_LIT>"])<EOL><DEDENT>else:<EOL><INDENT>index_files.extend(["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"])<EOL><DEDENT>index_files.extend(["<STR_LIT>"])<EOL>if not force:<EOL><INDENT>if all([os.path.isfile(refseq_file+i) for i in index_files]):<EOL><INDENT>return<EOL><DEDENT><DEDENT>if "<STR_LIT>" in data._hackersonly["<STR_LIT>"]:<EOL><INDENT>cmd1 = [ipyrad.bins.smalt, "<STR_LIT:index>", <EOL>"<STR_LIT>", str(data._hackersonly["<STR_LIT>"]), <EOL>refseq_file, <EOL>refseq_file]<EOL><DEDENT>else:<EOL><INDENT>cmd1 = [ipyrad.bins.bwa, "<STR_LIT:index>", refseq_file]<EOL><DEDENT>LOGGER.info("<STR_LIT:U+0020>".join(cmd1))<EOL>proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>error1 = proc1.communicate()[<NUM_LIT:0>]<EOL>cmd2 = [ipyrad.bins.samtools, "<STR_LIT>", refseq_file]<EOL>LOGGER.info("<STR_LIT:U+0020>".join(cmd2))<EOL>proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>error2 = proc2.communicate()[<NUM_LIT:0>]<EOL>if proc1.returncode:<EOL><INDENT>raise IPyradWarningExit(error1)<EOL><DEDENT>if error2:<EOL><INDENT>if "<STR_LIT>" in error2:<EOL><INDENT>raise IPyradWarningExit(NO_ZIP_BINS.format(refseq_file))<EOL><DEDENT>else:<EOL><INDENT>raise IPyradWarningExit(error2)<EOL><DEDENT><DEDENT>

Index the reference sequence, unless it already exists. Also make a mapping
of scaffolds to index numbers for later user in steps 5-6.


def mapreads(data, sample, nthreads, force):

LOGGER.info("<STR_LIT>", sample.name, nthreads)<EOL>derepfile = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>sample.files.dereps = [derepfile]<EOL>mumapfile = sample.files.unmapped_reads<EOL>umap1file = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>umap2file = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")        <EOL>if "<STR_LIT>" in data.paramsdict["<STR_LIT>"]:<EOL><INDENT>sample.files.split1 = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>sample.files.split2 = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>sample.files.dereps = [sample.files.split1, sample.files.split2]<EOL>split_merged_reads(sample.files.dereps, derepfile)<EOL><DEDENT>if "<STR_LIT>" in data._hackersonly["<STR_LIT>"]:<EOL><INDENT>cmd1 = [ipyrad.bins.smalt, "<STR_LIT>", <EOL>"<STR_LIT>", "<STR_LIT>", <EOL>"<STR_LIT>", str(max(<NUM_LIT:1>, nthreads)),<EOL>"<STR_LIT>", str(data.paramsdict['<STR_LIT>']), <EOL>"<STR_LIT>", os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>"),<EOL>"<STR_LIT>",<EOL>data.paramsdict['<STR_LIT>']<EOL>] + sample.files.dereps<EOL>cmd1_stdout = sps.PIPE<EOL>cmd1_stderr = sps.STDOUT<EOL><DEDENT>else:<EOL><INDENT>cmd1 = [ipyrad.bins.bwa, "<STR_LIT>",<EOL>"<STR_LIT>", str(max(<NUM_LIT:1>, nthreads)),<EOL>"<STR_LIT>",<EOL>data.paramsdict['<STR_LIT>']<EOL>] + sample.files.dereps<EOL>try:<EOL><INDENT>bwa_args = data._hackersonly["<STR_LIT>"].split()<EOL>bwa_args.reverse()<EOL>for arg in bwa_args:<EOL><INDENT>cmd1.insert(<NUM_LIT:2>, arg)<EOL><DEDENT><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT>cmd1_stdout = open(os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>"), '<STR_LIT:w>')<EOL>cmd1_stderr = None<EOL><DEDENT>cmd2 = [ipyrad.bins.samtools, "<STR_LIT>", <EOL>"<STR_LIT>", <EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>"), <EOL>os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>")]<EOL>cmd3 = [ipyrad.bins.samtools, "<STR_LIT>", <EOL>"<STR_LIT>", os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>"),<EOL>"<STR_LIT>", "<STR_LIT>", <EOL>"<STR_LIT>", sample.files.mapped_reads]<EOL>cmd4 = [ipyrad.bins.samtools, "<STR_LIT:index>", sample.files.mapped_reads]<EOL>cmd5 = [ipyrad.bins.samtools, "<STR_LIT>", "<STR_LIT>",<EOL>os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>")]<EOL>if '<STR_LIT>' in data.paramsdict["<STR_LIT>"]:<EOL><INDENT>if "<STR_LIT>" in data._hackersonly["<STR_LIT>"]:<EOL><INDENT>cmd1.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL>cmd1.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT>cmd2.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL>cmd2.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL>cmd5.insert(<NUM_LIT:2>, umap1file)<EOL>cmd5.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL>cmd5.insert(<NUM_LIT:2>, umap2file)<EOL>cmd5.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>cmd5.insert(<NUM_LIT:2>, mumapfile)<EOL>cmd5.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL><DEDENT>LOGGER.debug("<STR_LIT:U+0020>".join(cmd1))<EOL>proc1 = sps.Popen(cmd1, stderr=cmd1_stderr, stdout=cmd1_stdout)<EOL>try:<EOL><INDENT>error1 = proc1.communicate()[<NUM_LIT:0>]<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>proc1.kill()<EOL><DEDENT>if proc1.returncode:<EOL><INDENT>raise IPyradWarningExit(error1)<EOL><DEDENT>LOGGER.debug("<STR_LIT:U+0020>".join(cmd2))<EOL>proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>LOGGER.debug("<STR_LIT:U+0020>".join(cmd3))<EOL>proc3 = sps.Popen(cmd3, stderr=sps.STDOUT, stdout=sps.PIPE, stdin=proc2.stdout)<EOL>error3 = proc3.communicate()[<NUM_LIT:0>]<EOL>if proc3.returncode:<EOL><INDENT>raise IPyradWarningExit(error3)<EOL><DEDENT>proc2.stdout.close()<EOL>LOGGER.debug("<STR_LIT:U+0020>".join(cmd4))<EOL>proc4 = sps.Popen(cmd4, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>error4 = proc4.communicate()[<NUM_LIT:0>]<EOL>if proc4.returncode:<EOL><INDENT>raise IPyradWarningExit(error4)<EOL><DEDENT>LOGGER.debug("<STR_LIT:U+0020>".join(cmd5))<EOL>proc5 = sps.Popen(cmd5, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>error5 = proc5.communicate()[<NUM_LIT:0>]<EOL>if proc5.returncode:<EOL><INDENT>raise IPyradWarningExit(error5)<EOL><DEDENT>if '<STR_LIT>' in data.paramsdict["<STR_LIT>"]:<EOL><INDENT>LOGGER.info("<STR_LIT>".format(umap1file, umap2file))<EOL>merge_pairs_after_refmapping(data, [(umap1file, umap2file)], mumapfile)<EOL><DEDENT>

Attempt to map reads to reference sequence. This reads in the fasta files
(samples.files.edits), and maps each read to the reference. Unmapped reads
are dropped right back in the de novo pipeline. Reads that map successfully
are processed and pushed downstream and joined with the rest of the data
post muscle_align.

Mapped reads end up in a sam file.


def fetch_cluster_se(data, samfile, chrom, rstart, rend):

<EOL>overlap_buffer = data._hackersonly["<STR_LIT>"]<EOL>rstart_buff = rstart + overlap_buffer<EOL>rend_buff = rend - overlap_buffer<EOL>if rstart_buff > rend_buff:<EOL><INDENT>tmp = rstart_buff<EOL>rstart_buff = rend_buff<EOL>rend_buff = tmp<EOL><DEDENT>if rstart_buff == rend_buff:<EOL><INDENT>rend_buff += <NUM_LIT:1><EOL><DEDENT>rdict = {}<EOL>clust = []<EOL>iterreg = []<EOL>iterreg = samfile.fetch(chrom, rstart_buff, rend_buff)<EOL>for read in iterreg:<EOL><INDENT>if read.qname not in rdict:<EOL><INDENT>rdict[read.qname] = read<EOL><DEDENT><DEDENT>sfunc = lambda x: int(x.split("<STR_LIT>")[<NUM_LIT:1>].split("<STR_LIT:;>")[<NUM_LIT:0>])<EOL>rkeys = sorted(rdict.keys(), key=sfunc, reverse=True)<EOL>try:<EOL><INDENT>read1 = rdict[rkeys[<NUM_LIT:0>]]<EOL><DEDENT>except ValueError:<EOL><INDENT>LOGGER.error("<STR_LIT>".format(rkeys[<NUM_LIT:0>], rdict))<EOL>return "<STR_LIT>"<EOL><DEDENT>poss = read1.get_reference_positions(full_length=True)<EOL>seed_r1start = min(poss)<EOL>seed_r1end = max(poss)<EOL>if read1.is_reverse:<EOL><INDENT>seq = revcomp(read1.seq)<EOL><DEDENT>else:<EOL><INDENT>seq = read1.seq<EOL><DEDENT>size = sfunc(rkeys[<NUM_LIT:0>])<EOL>clust.append("<STR_LIT>".format(chrom, seed_r1start, seed_r1end, size, seq))<EOL>if len(rkeys) > <NUM_LIT:1>:<EOL><INDENT>for key in rkeys[<NUM_LIT:1>:]:<EOL><INDENT>skip = False<EOL>try:<EOL><INDENT>read1 = rdict[key]<EOL><DEDENT>except ValueError:<EOL><INDENT>read1 = rdict[key][<NUM_LIT:0>]<EOL>skip = True<EOL><DEDENT>if not skip:<EOL><INDENT>poss = read1.get_reference_positions(full_length=True)<EOL>minpos = min(poss)<EOL>maxpos = max(poss)<EOL>if read1.is_reverse:<EOL><INDENT>seq = revcomp(read1.seq)<EOL><DEDENT>else:<EOL><INDENT>seq = read1.seq<EOL><DEDENT>size = sfunc(key)<EOL>clust.append("<STR_LIT>".format(chrom, minpos, maxpos, size, seq))<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>return clust<EOL>

Builds a single end cluster from the refmapped data.


def fetch_cluster_pairs(data, samfile, chrom, rstart, rend):

<EOL>rdict = {}<EOL>clust = []<EOL>iterreg = samfile.fetch(chrom, rstart, rend)<EOL>for read in iterreg:<EOL><INDENT>if read.qname not in rdict:<EOL><INDENT>rdict[read.qname] = [read]<EOL><DEDENT>else:<EOL><INDENT>rdict[read.qname].append(read)<EOL><DEDENT><DEDENT>sfunc = lambda x: int(x.split("<STR_LIT>")[<NUM_LIT:1>].split("<STR_LIT:;>")[<NUM_LIT:0>])<EOL>rkeys = sorted(rdict.keys(), key=sfunc, reverse=True)<EOL>try:<EOL><INDENT>read1, read2 = rdict[rkeys[<NUM_LIT:0>]]<EOL><DEDENT>except ValueError:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>poss = read1.get_reference_positions() + read2.get_reference_positions()<EOL>seed_r1start = min(poss)<EOL>seed_r2end = max(poss)<EOL>reads_overlap = False<EOL>if read1.is_reverse:<EOL><INDENT>if read2.aend > read1.get_blocks()[<NUM_LIT:0>][<NUM_LIT:0>]:<EOL><INDENT>reads_overlap = True<EOL>seq = read2.seq + "<STR_LIT>" + revcomp(read1.seq)<EOL><DEDENT>else:<EOL><INDENT>seq = read2.seq + "<STR_LIT>" + read1.seq<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if read1.aend > read2.get_blocks()[<NUM_LIT:0>][<NUM_LIT:0>]:<EOL><INDENT>reads_overlap = True<EOL>seq = read1.seq + "<STR_LIT>" + revcomp(read2.seq)<EOL><DEDENT>else:<EOL><INDENT>seq = read1.seq + "<STR_LIT>" + read2.seq<EOL><DEDENT><DEDENT>size = sfunc(rkeys[<NUM_LIT:0>])<EOL>clust.append("<STR_LIT>".format(chrom, seed_r1start, seed_r2end, size, seq))<EOL>if len(rkeys) > <NUM_LIT:1>:<EOL><INDENT>for key in rkeys[<NUM_LIT:1>:]:<EOL><INDENT>skip = False<EOL>try:<EOL><INDENT>read1, read2 = rdict[key]<EOL><DEDENT>except ValueError:<EOL><INDENT>read1 = rdict[key][<NUM_LIT:0>]<EOL>read2 = read1<EOL>skip = True<EOL><DEDENT>poss = read1.get_reference_positions() + read2.get_reference_positions()<EOL>minpos = min(poss)<EOL>maxpos = max(poss)<EOL>if read1.has_tag("<STR_LIT>") or read2.has_tag("<STR_LIT>"):<EOL><INDENT>skip = True<EOL><DEDENT>if (abs(minpos - seed_r1start) < <NUM_LIT:50>) and(abs(maxpos - seed_r2end) < <NUM_LIT:50>) and(not skip):<EOL><INDENT>if read1.is_reverse:<EOL><INDENT>if read2.aend > read1.get_blocks()[<NUM_LIT:0>][<NUM_LIT:0>]:<EOL><INDENT>reads_overlap = True<EOL>seq = read2.seq + "<STR_LIT>" + revcomp(read1.seq)<EOL><DEDENT>else:<EOL><INDENT>seq = read2.seq + "<STR_LIT>" + read1.seq<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if read1.aend > read2.get_blocks()[<NUM_LIT:0>][<NUM_LIT:0>]:<EOL><INDENT>reads_overlap = True<EOL>seq = read1.seq + "<STR_LIT>" + revcomp(read2.seq)<EOL><DEDENT>else:<EOL><INDENT>seq = read1.seq + "<STR_LIT>" + read2.seq<EOL><DEDENT><DEDENT>size = sfunc(key)<EOL>clust.append("<STR_LIT>".format(chrom, minpos, maxpos, size, seq))<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>if reads_overlap:<EOL><INDENT>if data._hackersonly["<STR_LIT>"]:<EOL><INDENT>clust = merge_after_pysam(data, clust)<EOL><DEDENT><DEDENT>return clust<EOL>

Builds a paired cluster from the refmapped data.


<EOL>regions = bedtools_merge(data, sample).strip().split("<STR_LIT:\n>")<EOL>nregions = len(regions)<EOL>chunksize = (nregions / <NUM_LIT:10>) + (nregions % <NUM_LIT:10>)<EOL>LOGGER.debug("<STR_LIT>".format(nregions, chunksize))<EOL>idx = <NUM_LIT:0><EOL>tmpfile = os.path.join(data.tmpdir, sample.name+"<STR_LIT>")<EOL>for i in range(<NUM_LIT:11>):<EOL><INDENT>if os.path.exists(tmpfile.format(i)):<EOL><INDENT>os.remove(tmpfile.format(i))<EOL><DEDENT><DEDENT>fopen = open<EOL>if data.paramsdict["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>tmpfile = os.path.join(data.dirs.clusts, sample.name+"<STR_LIT>")<EOL>fopen = gzip.open<EOL><DEDENT>samfile = pysam.AlignmentFile(sample.files.mapped_reads, '<STR_LIT:rb>')<EOL>clusts = []<EOL>nclusts = <NUM_LIT:0><EOL>for region in regions:<EOL><INDENT>chrom, pos1, pos2 = region.split()<EOL>try:<EOL><INDENT>if "<STR_LIT>" in data.paramsdict["<STR_LIT>"]:<EOL><INDENT>clust = fetch_cluster_pairs(data, samfile, chrom, int(pos1), int(pos2))<EOL><DEDENT>else:<EOL><INDENT>clust = fetch_cluster_se(data, samfile, chrom, int(pos1), int(pos2))<EOL><DEDENT><DEDENT>except IndexError as inst:<EOL><INDENT>LOGGER.error("<STR_LIT>".format(chrom, pos1, pos2))<EOL>continue<EOL><DEDENT>if clust:<EOL><INDENT>clusts.append("<STR_LIT:\n>".join(clust))<EOL>nclusts += <NUM_LIT:1><EOL>if nclusts == chunksize:<EOL><INDENT>tmphandle = tmpfile.format(idx)<EOL>with fopen(tmphandle, '<STR_LIT:a>') as tmp:<EOL><INDENT>tmp.write("<STR_LIT>".join(clusts)+"<STR_LIT>")<EOL><DEDENT>idx += <NUM_LIT:1><EOL>nclusts = <NUM_LIT:0><EOL>clusts = []<EOL><DEDENT><DEDENT><DEDENT>if clusts:<EOL><INDENT>with fopen(tmpfile.format(idx), '<STR_LIT:a>') as tmp:<EOL><INDENT>tmp.write("<STR_LIT>".join(clusts)+"<STR_LIT>")<EOL><DEDENT>clusts = []<EOL><DEDENT>if not data.paramsdict["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>chunkfiles = glob.glob(os.path.join(data.tmpdir, sample.name+"<STR_LIT>"))<EOL>LOGGER.info("<STR_LIT>", chunkfiles)<EOL><DEDENT>samfile.close()<EOL>

1. Run bedtools to get all overlapping regions
2. Parse out reads from regions using pysam and dump into chunk files.
   We measure it out to create 10 chunk files per sample.
3. If we really wanted to speed this up, though it is pretty fast already,
   we could parallelize it since we can easily break the regions into
   a list of chunks.


LOGGER.info('<STR_LIT>')<EOL>regions = bedtools_merge(data, sample)<EOL>if len(regions) > <NUM_LIT:0>:<EOL><INDENT>get_overlapping_reads(data, sample, regions)<EOL><DEDENT>else:<EOL><INDENT>msg = "<STR_LIT>".format(sample.name)<EOL>LOGGER.warn(msg)<EOL><DEDENT>

Run bedtools to get all overlapping regions. Pass this list into the func
'get_overlapping_reads' which will write fastq chunks to the clust.gz file.

1) Run bedtools merge to get a list of all contiguous blocks of bases
in the reference seqeunce where one or more of our reads overlap.
The output will look like this:
        1       45230754        45230783
        1       74956568        74956596
        ...
        1       116202035       116202060


def get_overlapping_reads(data, sample, regions):

<EOL>locus_list = []<EOL>reads_merged = <NUM_LIT:0><EOL>write_flag = '<STR_LIT:wb>'<EOL>if data.paramsdict["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>write_flag = '<STR_LIT>'<EOL><DEDENT>sample.files.clusters = os.path.join(data.dirs.clusts, sample.name+"<STR_LIT>")<EOL>outfile = gzip.open(sample.files.clusters, write_flag)<EOL>if data.paramsdict["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>outfile.write("<STR_LIT>")<EOL><DEDENT>proc1 = sps.Popen("<STR_LIT>", stdin=sps.PIPE, stdout=sps.PIPE, universal_newlines=True)<EOL>try:<EOL><INDENT>for line in regions.strip().split("<STR_LIT:\n>"):<EOL><INDENT>if line == "<STR_LIT>":<EOL><INDENT>continue<EOL><DEDENT>chrom, region_start, region_end = line.strip().split()[<NUM_LIT:0>:<NUM_LIT:3>]<EOL>args = [data, sample, proc1, chrom, region_start, region_end]<EOL>clust = bam_region_to_fasta(*args)<EOL>if not clust:<EOL><INDENT>continue<EOL><DEDENT>locus_list.append(clust)<EOL>if not len(locus_list) % <NUM_LIT:1000>:<EOL><INDENT>outfile.write("<STR_LIT>".join(locus_list)+"<STR_LIT>")<EOL>locus_list = []<EOL><DEDENT><DEDENT>if any(locus_list):<EOL><INDENT>outfile.write("<STR_LIT>".join(locus_list))<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT>outfile.close()<EOL><DEDENT>except Exception as inst:<EOL><INDENT>LOGGER.error("<STR_LIT>".format(inst))<EOL>raise<EOL><DEDENT>finally:<EOL><INDENT>if "<STR_LIT>" in data.paramsdict["<STR_LIT>"]:<EOL><INDENT>LOGGER.info("<STR_LIT>".format(sample.name, reads_merged))<EOL>sample.stats.reads_merged = reads_merged<EOL><DEDENT><DEDENT>

For SE data, this pulls mapped reads out of sorted mapped bam files and
appends them to the clust.gz file so they fall into downstream
(muscle alignment) analysis.

For PE data, this pulls mapped reads out of sorted mapped bam files, splits
R1s from R2s and writes them to separate files. Once all reads are written,
it calls merge_reads (vsearch) to find merged and non-merged reads. These
are then put into clust.gz with either an nnnn separator or as merged.

The main func being called here is 'bam_region_to_fasta', which calls
samtools to pull out the mapped reads.

1) Coming into this function we have sample.files.mapped_reads
    as a sorted bam file, and a passed in list of regions to evaluate.
2) Get all reads overlapping with each individual region.
3) Pipe to vsearch for clustering.
4) Append to the clust.gz file.


handle1, handle2 = outhandles<EOL>splitderep1 = open(handle1, '<STR_LIT:w>')<EOL>splitderep2 = open(handle2, '<STR_LIT:w>')<EOL>with open(input_derep, '<STR_LIT:r>') as infile: <EOL><INDENT>duo = itertools.izip(*[iter(infile)]*<NUM_LIT:2>)<EOL>split1s = []<EOL>split2s = []<EOL>idx = <NUM_LIT:0><EOL>while <NUM_LIT:1>:<EOL><INDENT>try:<EOL><INDENT>itera = duo.next()<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>part1, part2 = itera[<NUM_LIT:1>].split("<STR_LIT>")<EOL>idx += <NUM_LIT:1><EOL>split1s.append("<STR_LIT>".format(itera[<NUM_LIT:0>], part1))<EOL>split2s.append("<STR_LIT>".format(itera[<NUM_LIT:0>], part2))<EOL>if not idx % <NUM_LIT>:<EOL><INDENT>splitderep1.write("<STR_LIT>".join(split1s))<EOL>splitderep2.write("<STR_LIT>".join(split2s))<EOL>split1s = []<EOL>split2s = [] <EOL><DEDENT><DEDENT><DEDENT>if any(split1s):<EOL><INDENT>splitderep1.write("<STR_LIT>".join(split1s))<EOL>splitderep2.write("<STR_LIT>".join(split2s))<EOL><DEDENT>splitderep1.close()<EOL>splitderep2.close()<EOL>

Takes merged/concat derep file from vsearch derep and split it back into
separate R1 and R2 parts.
- sample_fastq: a list of the two file paths to write out to.
- input_reads: the path to the input merged reads


<EOL>cmd1 = [ipyrad.bins.samtools, "<STR_LIT>", sample.files.mapped_reads]<EOL>cmd2 = ["<STR_LIT>", "<STR_LIT>"]<EOL>proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE, stdin=proc1.stdout)<EOL>res = proc2.communicate()[<NUM_LIT:0>]<EOL>if proc2.returncode:<EOL><INDENT>raise IPyradWarningExit("<STR_LIT>", cmd2, res)<EOL><DEDENT>avg_insert = <NUM_LIT:0><EOL>stdv_insert = <NUM_LIT:0><EOL>avg_len = <NUM_LIT:0><EOL>for line in res.split("<STR_LIT:\n>"):<EOL><INDENT>if "<STR_LIT>" in line:<EOL><INDENT>avg_insert = float(line.split("<STR_LIT::>")[-<NUM_LIT:1>].strip())<EOL><DEDENT>elif "<STR_LIT>" in line:<EOL><INDENT>stdv_insert = float(line.split("<STR_LIT::>")[-<NUM_LIT:1>].strip()) + <NUM_LIT:0.1><EOL><DEDENT>elif "<STR_LIT>" in line:<EOL><INDENT>avg_len = float(line.split("<STR_LIT::>")[-<NUM_LIT:1>].strip())<EOL><DEDENT><DEDENT>LOGGER.debug("<STR_LIT>".format(avg_insert, stdv_insert, avg_len))<EOL>if all([avg_insert, stdv_insert, avg_len]):<EOL><INDENT>if stdv_insert < <NUM_LIT:5>:<EOL><INDENT>stdv_insert = <NUM_LIT><EOL><DEDENT>if (<NUM_LIT:2> * avg_len) < avg_insert:<EOL><INDENT>hack = avg_insert + (<NUM_LIT:3> * np.math.ceil(stdv_insert)) - (<NUM_LIT:2> * avg_len)<EOL><DEDENT>else:<EOL><INDENT>hack = (avg_insert - avg_len) + (<NUM_LIT:3> * np.math.ceil(stdv_insert))<EOL><DEDENT>LOGGER.info("<STR_LIT>", hack)<EOL>data._hackersonly["<STR_LIT>"] = int(np.math.ceil(hack))<EOL><DEDENT>else:<EOL><INDENT>data._hackersonly["<STR_LIT>"] = <NUM_LIT><EOL>LOGGER.debug("<STR_LIT>".format(sample.name,data._hackersonly["<STR_LIT>"]))<EOL><DEDENT>

check mean insert size for this sample and update
hackersonly.max_inner_mate_distance if need be. This value controls how
far apart mate pairs can be to still be considered for bedtools merging
downstream.


LOGGER.info("<STR_LIT>", sample.name)<EOL>mappedreads = os.path.join(data.dirs.refmapping, <EOL>sample.name+"<STR_LIT>")<EOL>cmd1 = [ipyrad.bins.bedtools, "<STR_LIT>", "<STR_LIT>", mappedreads]<EOL>cmd2 = [ipyrad.bins.bedtools, "<STR_LIT>", "<STR_LIT>", "<STR_LIT:->"]<EOL>if '<STR_LIT>' in data.paramsdict["<STR_LIT>"]:<EOL><INDENT>check_insert_size(data, sample)<EOL>cmd2.insert(<NUM_LIT:2>, str(data._hackersonly["<STR_LIT>"]))<EOL>cmd2.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>cmd2.insert(<NUM_LIT:2>, str(-<NUM_LIT:1> * data._hackersonly["<STR_LIT>"]))<EOL>cmd2.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL><DEDENT>LOGGER.info("<STR_LIT>", cmd1, cmd2)<EOL>proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE, stdin=proc1.stdout)<EOL>result = proc2.communicate()[<NUM_LIT:0>]<EOL>proc1.stdout.close()<EOL>if proc2.returncode:<EOL><INDENT>raise IPyradWarningExit("<STR_LIT>", cmd2, result)<EOL><DEDENT>if os.path.exists(ipyrad.__debugflag__):<EOL><INDENT>with open(os.path.join(data.dirs.refmapping, sample.name + "<STR_LIT>"), '<STR_LIT:w>') as outfile:<EOL><INDENT>outfile.write(result)<EOL><DEDENT><DEDENT>nregions = len(result.strip().split("<STR_LIT:\n>"))<EOL>LOGGER.info("<STR_LIT>", nregions)<EOL>return result<EOL>

Get all contiguous genomic regions with one or more overlapping
reads. This is the shell command we'll eventually run

bedtools bamtobed -i 1A_0.sorted.bam | bedtools merge [-d 100]
    -i <input_bam>  :   specifies the input file to bed'ize
    -d <int>        :   For PE set max distance between reads


LOGGER.debug("<STR_LIT>".format(fasta[<NUM_LIT:0>]))<EOL>if "<STR_LIT>" in fasta[<NUM_LIT:1>]:<EOL><INDENT>r1_len = len(fasta[<NUM_LIT:1>].split("<STR_LIT:\n>")[<NUM_LIT:1>].split("<STR_LIT>")[<NUM_LIT:0>])<EOL>r2_len = len(fasta[<NUM_LIT:1>].split("<STR_LIT:\n>")[<NUM_LIT:1>].split("<STR_LIT>")[<NUM_LIT:1>])<EOL>new_seq = fasta[<NUM_LIT:0>].split("<STR_LIT:\n>")[<NUM_LIT:1>][:r1_len]+("<STR_LIT>")+ revcomp(fasta[<NUM_LIT:0>].split("<STR_LIT:\n>")[<NUM_LIT:1>][-r2_len:])<EOL>fasta[<NUM_LIT:0>] = fasta[<NUM_LIT:0>].split("<STR_LIT:\n>")[<NUM_LIT:0>]+"<STR_LIT:\n>"+new_seq<EOL><DEDENT>LOGGER.debug("<STR_LIT>".format(fasta[<NUM_LIT:0>]))<EOL>return fasta<EOL>

If doing PE and R1/R2 don't overlap then the reference sequence
will be quite long and will cause indel hell during the
alignment stage. Here trim the reference sequence to the length
of the merged reads. Input is a list of alternating locus labels
and sequence data. The first locus label is the reference
sequence label and the first seq is the reference seq. Returns
the same list except with the reference sequence trimmed to
the length of the rad tags


def bam_region_to_fasta(data, sample, proc1, chrom, region_start, region_end):

<EOL>bamf = sample.files.mapped_reads<EOL>if not os.path.exists(bamf):<EOL><INDENT>raise IPyradWarningExit("<STR_LIT>", bamf)<EOL><DEDENT>chrom.replace("<STR_LIT:|>", r"<STR_LIT>")<EOL>chrom.replace("<STR_LIT:(>", r"<STR_LIT>")<EOL>chrom.replace("<STR_LIT:)>", r"<STR_LIT>")<EOL>rstring_id1 = "<STR_LIT>".format(chrom, str(int(region_start)+<NUM_LIT:1>), region_end)<EOL>rstring_id0 = "<STR_LIT>".format(chrom, region_start, region_end)<EOL>overlap_buffer = <NUM_LIT:0><EOL>if not "<STR_LIT>" in data.paramsdict["<STR_LIT>"]:<EOL><INDENT>overlap_buffer = data._hackersonly["<STR_LIT>"]<EOL><DEDENT>rstring_id0_buffered = "<STR_LIT>".format(chrom, int(region_start) + overlap_buffer,int(region_end) - overlap_buffer)<EOL>cmd1 = [ipyrad.bins.samtools, "<STR_LIT>", <EOL>data.paramsdict["<STR_LIT>"], <EOL>rstring_id1, "<STR_LIT>"]<EOL>LOGGER.info("<STR_LIT>".format(cmd1))<EOL>print("<STR_LIT:U+0020>".join(cmd1), file=proc1.stdin)<EOL>ref = "<STR_LIT>"<EOL>for line in iter(proc1.stdout.readline, "<STR_LIT>"):<EOL><INDENT>if "<STR_LIT>" in line:<EOL><INDENT>break<EOL><DEDENT>ref += line<EOL><DEDENT>fasta = []<EOL>try:<EOL><INDENT>name, seq = ref.strip().split("<STR_LIT:\n>", <NUM_LIT:1>)<EOL>seq = "<STR_LIT>".join(seq.split("<STR_LIT:\n>"))<EOL>fasta = ["<STR_LIT>".format(name, <NUM_LIT>, seq)]<EOL><DEDENT>except ValueError as inst:<EOL><INDENT>LOGGER.error("<STR_LIT>".format(ref))<EOL>LOGGER.error("<STR_LIT:U+0020>".join(cmd1))<EOL><DEDENT>if "<STR_LIT>" in data.paramsdict["<STR_LIT>"]:<EOL><INDENT>fasta = []<EOL>if os.path.exists("<STR_LIT>"):<EOL><INDENT>prefix = os.path.join("<STR_LIT>",<EOL>"<STR_LIT>".format(sample.name, rstring_id0))<EOL><DEDENT>else:<EOL><INDENT>prefix = os.path.join(data.dirs.refmapping, <EOL>"<STR_LIT>".format(sample.name, rstring_id0))<EOL><DEDENT>read1 = "<STR_LIT>".format(prefix)<EOL>read2 = "<STR_LIT>".format(prefix)<EOL>merged = "<STR_LIT>".format(prefix)<EOL>cmd1 = "<STR_LIT:U+0020>".join([ipyrad.bins.samtools, "<STR_LIT>", "<STR_LIT>", bamf, rstring_id0])<EOL>cmd2 = "<STR_LIT:U+0020>".join([ipyrad.bins.samtools, "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", read1, "<STR_LIT>", read2, "<STR_LIT:->", "<STR_LIT>"])<EOL>cmd = "<STR_LIT>".join([cmd1, cmd2])<EOL>print(cmd, file=proc1.stdin)<EOL>for line in iter(proc1.stdout.readline, "<STR_LIT>"):<EOL><INDENT>if "<STR_LIT>" in line:<EOL><INDENT>break<EOL><DEDENT><DEDENT>try:<EOL><INDENT>_ = merge_pairs(data, [(read1, read2)], merged, <NUM_LIT:0>, <NUM_LIT:1>)<EOL>with open(merged, '<STR_LIT:r>') as infile:<EOL><INDENT>quatro = itertools.izip(*[iter(infile)]*<NUM_LIT:4>)<EOL>while <NUM_LIT:1>:<EOL><INDENT>try:<EOL><INDENT>bits = quatro.next()<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>orient = "<STR_LIT:+>"<EOL>fullfast = "<STR_LIT>".format(<EOL>a=bits[<NUM_LIT:0>].split("<STR_LIT:;>")[<NUM_LIT:0>],<EOL>b=rstring_id1,<EOL>c=bits[<NUM_LIT:0>].split("<STR_LIT:;>")[<NUM_LIT:1>], <EOL>d=orient,<EOL>e=bits[<NUM_LIT:1>].strip())<EOL>fasta.append(fullfast)<EOL><DEDENT><DEDENT><DEDENT>except (OSError, ValueError, IPyradError) as inst:<EOL><INDENT>LOGGER.debug("<STR_LIT>", inst)<EOL>LOGGER.error("<STR_LIT>".format(cmd))<EOL>return "<STR_LIT>"<EOL><DEDENT>finally:<EOL><INDENT>if os.path.exists(merged):<EOL><INDENT>os.remove(merged)<EOL><DEDENT>if os.path.exists(read1):<EOL><INDENT>os.remove(read1)<EOL><DEDENT>if os.path.exists(read2):<EOL><INDENT>os.remove(read2)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>cmd2 = [ipyrad.bins.samtools, "<STR_LIT>", bamf, rstring_id0_buffered]<EOL>proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>res = proc2.communicate()[<NUM_LIT:0>]<EOL>if proc2.returncode:<EOL><INDENT>raise IPyradWarningExit("<STR_LIT>".format(cmd2, res))<EOL><DEDENT>if "<STR_LIT>" in res:<EOL><INDENT>raise IPyradError("<STR_LIT>".format(rstring_id0_buffered))<EOL><DEDENT>for line in res.strip().split("<STR_LIT:\n>"):<EOL><INDENT>bits = line.split("<STR_LIT:\t>")<EOL>orient = "<STR_LIT:+>"<EOL>if int('<STR_LIT>'.format(int(bits[<NUM_LIT:1>]))[<NUM_LIT:7>]):<EOL><INDENT>orient = "<STR_LIT:->"<EOL><DEDENT>fullfast = "<STR_LIT>".format(<EOL>a=bits[<NUM_LIT:0>].split("<STR_LIT:;>")[<NUM_LIT:0>],<EOL>b=rstring_id0,<EOL>c=bits[<NUM_LIT:0>].split("<STR_LIT:;>")[<NUM_LIT:1>],<EOL>d=orient,<EOL>e=bits[<NUM_LIT:9>])<EOL>fasta.append(fullfast)<EOL><DEDENT><DEDENT>except IPyradError as inst:<EOL><INDENT>LOGGER.debug("<STR_LIT>".format(inst))<EOL>return "<STR_LIT>"<EOL><DEDENT>except (OSError, ValueError, Exception) as inst:<EOL><INDENT>LOGGER.error("<STR_LIT>", inst)<EOL>LOGGER.error("<STR_LIT>".format(cmd2))<EOL>return "<STR_LIT>"<EOL><DEDENT><DEDENT>return "<STR_LIT:\n>".join(fasta)<EOL>

Take the chromosome position, and start and end bases and return sequences
of all reads that overlap these sites. This is the command we're building:

samtools view -b 1A_sorted.bam 1:116202035-116202060 | \
         samtools bam2fq <options> -

        -b      : output bam format
        -0      : For SE, output all reads to this file
        -1/-2   : For PE, output first and second reads to different files
        -       : Tell samtools to read in from the pipe

Write out the sam output and parse it to return as fasta for clust.gz file.
We also grab the reference sequence with a @REF header to aid in alignment
for single-end data. This will be removed post-alignment.


<EOL>mapf = os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>")<EOL>umapf = os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>")<EOL>cmd1 = [ipyrad.bins.samtools, "<STR_LIT>", umapf]<EOL>proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>result1 = proc1.communicate()[<NUM_LIT:0>]<EOL>cmd2 = [ipyrad.bins.samtools, "<STR_LIT>", mapf]<EOL>proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>result2 = proc2.communicate()[<NUM_LIT:0>]<EOL>if "<STR_LIT>" in data.paramsdict["<STR_LIT>"]:<EOL><INDENT>sample.stats["<STR_LIT>"] = int(result1.split()[<NUM_LIT:0>]) / <NUM_LIT:2><EOL>sample.stats["<STR_LIT>"] = int(result2.split()[<NUM_LIT:0>]) / <NUM_LIT:2><EOL><DEDENT>else:<EOL><INDENT>sample.stats["<STR_LIT>"] = int(result1.split()[<NUM_LIT:0>])<EOL>sample.stats["<STR_LIT>"] = int(result2.split()[<NUM_LIT:0>])<EOL><DEDENT>sample_cleanup(data, sample)<EOL>

Get the number of mapped and unmapped reads for a sample
and update sample.stats


<EOL>sample.files.unmapped_reads = os.path.join(data.dirs.edits, <EOL>"<STR_LIT>".format(sample.name))<EOL>sample.files.mapped_reads = os.path.join(data.dirs.refmapping,<EOL>"<STR_LIT>".format(sample.name))<EOL>


<EOL>panel = Panel(ttree)          <EOL>if not kwargs.get("<STR_LIT:width>"):<EOL><INDENT>panel.kwargs["<STR_LIT:width>"] = min(<NUM_LIT:1000>, <NUM_LIT>*len(panel.tree))<EOL><DEDENT>if not kwargs.get("<STR_LIT>"):<EOL><INDENT>panel.kwargs["<STR_LIT>"] = panel.kwargs["<STR_LIT:width>"]<EOL><DEDENT>panel.kwargs.update(kwargs)<EOL>if panel.kwargs["<STR_LIT>"]:<EOL><INDENT>nnodes = sum(<NUM_LIT:1> for i in panel.tree.traverse()) - len(panel.tree)<EOL>supps = [int(panel.tree.search_nodes(idx=j)[<NUM_LIT:0>].support)for j in range(nnodes)]<EOL>if not panel.kwargs["<STR_LIT>"]:<EOL><INDENT>panel.kwargs["<STR_LIT>"] = <NUM_LIT:20><EOL><DEDENT>sizes = [panel.kwargs["<STR_LIT>"] for j in range(nnodes)]<EOL>supps += ["<STR_LIT>"] * len(panel.tree)<EOL>sizes += [<NUM_LIT:0>] * len(panel.tree)<EOL>panel.kwargs["<STR_LIT>"] = supps<EOL>panel.kwargs["<STR_LIT>"] = sizes<EOL>panel.kwargs["<STR_LIT>"] = True<EOL>if len(panel.tree.children) > <NUM_LIT:2>:<EOL><INDENT>supps[<NUM_LIT:0>] = "<STR_LIT>"<EOL>sizes[<NUM_LIT:0>] = <NUM_LIT:0><EOL><DEDENT><DEDENT>elif panel.kwargs.get("<STR_LIT>"):<EOL><INDENT>panel.kwargs["<STR_LIT>"] = panel.kwargs["<STR_LIT>"]<EOL>panel.kwargs["<STR_LIT>"] = True<EOL><DEDENT>else:<EOL><INDENT>panel.kwargs["<STR_LIT>"] = panel.node_labels.keys() <EOL><DEDENT>if print_args:<EOL><INDENT>print(panel.kwargs)<EOL><DEDENT>canvas = toyplot.Canvas(height=panel.kwargs['<STR_LIT>'], width=panel.kwargs['<STR_LIT:width>'])<EOL>axes = canvas.cartesian(bounds=("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"))    <EOL>axes.show = panel.kwargs["<STR_LIT>"]<EOL>panel._panel_tree(axes)<EOL>if panel.kwargs["<STR_LIT>"]:<EOL><INDENT>panel._panel_tip_labels(axes)<EOL><DEDENT>return canvas, axes, panel<EOL>


<EOL>colormap = toyplot.color.LinearMap(toyplot.color.brewer.palette("<STR_LIT>"), <EOL>domain_min=share.min(), domain_max=share.max())<EOL>if not kwargs.get('<STR_LIT:width>'):<EOL><INDENT>width=<NUM_LIT><EOL><DEDENT>else:<EOL><INDENT>width = kwargs['<STR_LIT:width>']<EOL><DEDENT>canvas = toyplot.Canvas(width=width, height=width*<NUM_LIT>)<EOL>table = canvas.matrix((share, colormap), <EOL>bounds=(<NUM_LIT:50>, canvas.height-<NUM_LIT:100>,<EOL><NUM_LIT:50>, canvas.height-<NUM_LIT:100>), <EOL>step=<NUM_LIT:5>, tshow=False, lshow=False)<EOL>table.body.grid.vlines[..., [<NUM_LIT:0>, -<NUM_LIT:1>]] = '<STR_LIT>'<EOL>table.body.grid.hlines[[<NUM_LIT:0>, -<NUM_LIT:1>], ...] = '<STR_LIT>'<EOL>for i, j in itertools.product(range(len(share)), repeat=<NUM_LIT:2>):<EOL><INDENT>table.body.cell(i,j).title = "<STR_LIT>" % (names[i], names[j], int(share[i,j]))<EOL><DEDENT>axes = canvas.cartesian(bounds=(<NUM_LIT>, <NUM_LIT>, <NUM_LIT>, <NUM_LIT>))<EOL>zf = zip(names[::-<NUM_LIT:1>], share.diagonal()[::-<NUM_LIT:1>])<EOL>barfloater = ["<STR_LIT>" % (i, int(j)) for i, j in zf]<EOL>axes.bars(share.diagonal()[::-<NUM_LIT:1>], along='<STR_LIT:y>', title=barfloater)<EOL>axes.y.spine.show = False<EOL>axes.y.ticks.labels.offset = <NUM_LIT:0><EOL>axes.y.ticks.locator = toyplot.locator.Explicit(range(len(names)), <EOL>labels=names[::-<NUM_LIT:1>])<EOL>axes.y.ticks.labels.angle = -<NUM_LIT><EOL>axes.y.ticks.labels.style = {"<STR_LIT>":<NUM_LIT:0>,<EOL>"<STR_LIT>":"<STR_LIT:end>", <EOL>"<STR_LIT>":"<STR_LIT>"}<EOL>axes.x.ticks.labels.angle = <NUM_LIT><EOL>axes.x.ticks.labels.offset = <NUM_LIT:20><EOL>axes.x.ticks.locator = toyplot.locator.Explicit(<EOL>range(<NUM_LIT:0>, int(share.max()), <EOL>int(share.max() / <NUM_LIT:10>)), <EOL>["<STR_LIT:{}>".format(i) for i in range(<NUM_LIT:0>, int(share.max()),<EOL>int(share.max() / <NUM_LIT:10>))])<EOL>axes.x.ticks.labels.style = {"<STR_LIT>":<NUM_LIT:0>,<EOL>"<STR_LIT>":"<STR_LIT:end>", <EOL>"<STR_LIT>":"<STR_LIT>"}<EOL>axes.x.ticks.show = True<EOL>label_style = {"<STR_LIT>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>"}<EOL>canvas.text(<NUM_LIT>, <NUM_LIT>, "<STR_LIT>", style=label_style)<EOL>canvas.text(<NUM_LIT>, <NUM_LIT>, "<STR_LIT>", style=label_style)<EOL>return canvas, axes<EOL>


<EOL>tree.ladderize()<EOL>snames = tree.get_leaf_names()<EOL>lxs = np.zeros((len(snames), len(loci)), dtype=np.int)<EOL>for loc in xrange(len(loci)):<EOL><INDENT>for seq in loci[loc].split("<STR_LIT:\n>")[:-<NUM_LIT:1>]:<EOL><INDENT>lxs[snames.index(seq.split()[<NUM_LIT:0>]), loc] += <NUM_LIT:1><EOL><DEDENT><DEDENT>return lxs, snames<EOL>

parse the loci file list and return presence/absence matrix
ordered by the tips on the tree


<EOL>share = np.zeros((lxs.shape[<NUM_LIT:0>], lxs.shape[<NUM_LIT:0>]))<EOL>names = range(lxs.shape[<NUM_LIT:0>])<EOL>for row in lxs:<EOL><INDENT>for samp1, samp2 in itertools.combinations(names, <NUM_LIT:2>):<EOL><INDENT>shared = lxs[samp1, lxs[samp2] > <NUM_LIT:0>].sum()<EOL>share[samp1, samp2] = shared<EOL><DEDENT><DEDENT>for row in xrange(len(names)):<EOL><INDENT>share[row, row] = lxs[row].sum()<EOL><DEDENT>return share<EOL>

fill a matrix with pairwise data sharing


def depthplot(data, samples=None, dims=(None,None), canvas=(None,None), <EOL>xmax=<NUM_LIT:50>, log=False, outprefix=None, use_maxdepth=False):

<EOL>if not samples:<EOL><INDENT>samples = data.samples.keys()<EOL>samples.sort()<EOL><DEDENT>subsamples = OrderedDict([(i, data.samples[i]) for i in samples])<EOL>if any(dims):<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>if len(subsamples) <= <NUM_LIT:4>:<EOL><INDENT>dims = (<NUM_LIT:1>, len(subsamples))<EOL><DEDENT>else:<EOL><INDENT>dims = (len(subsamples)/<NUM_LIT:4>, <NUM_LIT:4>)<EOL><DEDENT><DEDENT>if any(canvas):<EOL><INDENT>print("<STR_LIT>")<EOL>canvas = toyplot.Canvas(width=canvas[<NUM_LIT:0>], height=canvas[<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>canvas = toyplot.Canvas(width=<NUM_LIT:200>*dims[<NUM_LIT:1>], height=<NUM_LIT>*dims[<NUM_LIT:0>])<EOL><DEDENT>for panel, sample in enumerate(subsamples):<EOL><INDENT>statdat = subsamples[sample].depths<EOL>statdat = statdat[statdat >= data.paramsdict["<STR_LIT>"]]<EOL>if use_maxdepth:<EOL><INDENT>statdat = {i:j for (i, j) in statdat ifi < data.paramsdict["<STR_LIT>"]}<EOL><DEDENT>sdat = np.histogram(statdat, range(<NUM_LIT:50>))<EOL>statdat = subsamples[sample].depths<EOL>statdat = statdat[statdat < data.paramsdict["<STR_LIT>"]]<EOL>statdat = statdat[statdat >= data.paramsdict["<STR_LIT>"]]<EOL>if use_maxdepth:<EOL><INDENT>statdat = statdat[statdat < data.paramsdict["<STR_LIT>"]]<EOL><DEDENT>mdat = np.histogram(statdat, range(<NUM_LIT:50>))<EOL>tots = data.samples[sample].depths<EOL>tots = tots[tots < data.paramsdict["<STR_LIT>"]]<EOL>if use_maxdepth:<EOL><INDENT>tots = tots[tots < data.paramsdict["<STR_LIT>"]]<EOL><DEDENT>edat = np.histogram(tots, range(<NUM_LIT:50>))<EOL>axes = canvas.cartesian(grid=(dims[<NUM_LIT:0>], dims[<NUM_LIT:1>], panel), gutter=<NUM_LIT>)<EOL>axes.x.domain.xmax = xmax<EOL>axes.label.text = sample<EOL>if log:<EOL><INDENT>axes.y.scale = "<STR_LIT>"<EOL><DEDENT>axes.bars(sdat)<EOL>axes.bars(edat)<EOL>axes.bars(mdat)<EOL><DEDENT>if outprefix:<EOL><INDENT>toyplot.html.render(canvas, fobj=outprefix+"<STR_LIT>")<EOL>toyplot.svg.render(canvas, fobj=outprefix+"<STR_LIT>")<EOL><DEDENT>

plots histogram of coverages across clusters


def share_matrix(locifile, tree=None, nameorder=None):

<EOL>with open(locifile, '<STR_LIT:r>') as locidata:<EOL><INDENT>loci = locidata.read().split("<STR_LIT>")[:-<NUM_LIT:1>]<EOL><DEDENT>if tree:<EOL><INDENT>tree = ete.Tree(tree)<EOL>tree.ladderize()<EOL>snames = tree.get_leaf_names()<EOL>lxs, names = _getarray(loci, snames)<EOL><DEDENT>elif nameorder:<EOL><INDENT>lxs, names = _getarray(loci, nameorder)<EOL><DEDENT>else:<EOL><INDENT>raise IOError("<STR_LIT>")<EOL><DEDENT>share = _countmatrix(lxs)<EOL>return share<EOL>

returns a matrix of shared RAD-seq data

Parameters:
-----------
locifile (str):
    Path to a ipyrad .loci file.
tree (str):
    Path to Newick file or a Newick string representation of
    a tree. If used, names will be ordered by the ladderized
    tip order.
nameorder (list):
    If a tree is not provided you can alternatively enter
    the sample order as a list here. The tree argument will
    override this argument.

Returns:
--------
matrix (numpy.array):
    A uint64 numpy array of the number of shared loci between
    all pairs of samples.


<EOL>colormap = toyplot.color.LinearMap(<EOL>toyplot.color.brewer.palette("<STR_LIT>"), <EOL>domain_min=share.min(), <EOL>domain_max=share.max())<EOL>if not kwargs.get('<STR_LIT:width>'):<EOL><INDENT>width=<NUM_LIT><EOL><DEDENT>else:<EOL><INDENT>width = kwargs['<STR_LIT:width>']<EOL><DEDENT>canvas = toyplot.Canvas(width=width, height=width*<NUM_LIT>)<EOL>table = canvas.matrix((share, colormap), <EOL>bounds=(<NUM_LIT:50>, canvas.height-<NUM_LIT:100>,<EOL><NUM_LIT:50>, canvas.height-<NUM_LIT:100>), <EOL>step=<NUM_LIT:5>, tshow=False, lshow=False)<EOL>table.body.grid.vlines[..., [<NUM_LIT:0>, -<NUM_LIT:1>]] = '<STR_LIT>'<EOL>table.body.grid.hlines[[<NUM_LIT:0>, -<NUM_LIT:1>], ...] = '<STR_LIT>'<EOL>for i, j in itertools.product(range(len(share)), repeat=<NUM_LIT:2>):<EOL><INDENT>table.body.cell(i,j).title = "<STR_LIT>" % (names[i], names[j], int(share[i,j]))<EOL><DEDENT>axes = canvas.cartesian(bounds=(<NUM_LIT>, <NUM_LIT>, <NUM_LIT>, <NUM_LIT>))<EOL>zf = zip(names[::-<NUM_LIT:1>], share.diagonal()[::-<NUM_LIT:1>])<EOL>barfloater = ["<STR_LIT>" % (i, int(j)) for i, j in zf]<EOL>axes.bars(share.diagonal()[::-<NUM_LIT:1>], along='<STR_LIT:y>', title=barfloater)<EOL>axes.y.spine.show = False<EOL>axes.y.ticks.labels.offset = <NUM_LIT:0><EOL>axes.y.ticks.locator = toyplot.locator.Explicit(range(len(names)), <EOL>labels=names[::-<NUM_LIT:1>])<EOL>axes.y.ticks.labels.angle = -<NUM_LIT><EOL>axes.y.ticks.labels.style = {"<STR_LIT>":<NUM_LIT:0>,<EOL>"<STR_LIT>":"<STR_LIT:end>", <EOL>"<STR_LIT>":"<STR_LIT>"}<EOL>axes.x.ticks.labels.angle = <NUM_LIT><EOL>axes.x.ticks.labels.offset = <NUM_LIT:20><EOL>axes.x.ticks.locator = toyplot.locator.Explicit(<EOL>range(<NUM_LIT:0>, int(share.max()), <EOL>int(share.max() / <NUM_LIT:10>)), <EOL>["<STR_LIT:{}>".format(i) for i in range(<NUM_LIT:0>, int(share.max()),<EOL>int(share.max() / <NUM_LIT:10>))])<EOL>axes.x.ticks.labels.style = {"<STR_LIT>":<NUM_LIT:0>,<EOL>"<STR_LIT>":"<STR_LIT:end>", <EOL>"<STR_LIT>":"<STR_LIT>"}<EOL>axes.x.ticks.show = True<EOL>label_style = {"<STR_LIT>": "<STR_LIT>", "<STR_LIT>": "<STR_LIT>"}<EOL>canvas.text(<NUM_LIT>, <NUM_LIT>, "<STR_LIT>", style=label_style)<EOL>canvas.text(<NUM_LIT>, <NUM_LIT>, "<STR_LIT>", style=label_style)<EOL>return canvas, axes<EOL>


<EOL>lxs = np.zeros((len(snames), len(loci)), dtype=np.uint64)<EOL>for loc in xrange(len(loci)):<EOL><INDENT>for seq in loci[loc].split("<STR_LIT:\n>"):<EOL><INDENT>if "<STR_LIT>" not in seq:<EOL><INDENT>lxs[snames.index(seq.split()[<NUM_LIT:0>][:]), loc] += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>return lxs, snames<EOL>

parse loci list and return presence/absence matrix
ordered by the tips on the tree or list of names.


<EOL>share = np.zeros((lxs.shape[<NUM_LIT:0>], lxs.shape[<NUM_LIT:0>]), dtype=np.uint64)<EOL>names = range(lxs.shape[<NUM_LIT:0>])<EOL>for row in lxs:<EOL><INDENT>for samp1, samp2 in itertools.combinations(names, <NUM_LIT:2>):<EOL><INDENT>shared = lxs[samp1, lxs[samp2] > <NUM_LIT:0>].sum()<EOL>share[samp1, samp2] = shared<EOL><DEDENT><DEDENT>share += share.T<EOL>for row in xrange(len(names)):<EOL><INDENT>share[row, row] = lxs[row].sum()<EOL><DEDENT>return share<EOL>


<EOL>iparg = "<STR_LIT>"<EOL>if "<STR_LIT>" in data._ipcluster["<STR_LIT>"]:<EOL><INDENT>iparg = "<STR_LIT>"<EOL><DEDENT>standard = """<STR_LIT>""".format(data._ipcluster["<STR_LIT>"], <EOL>data._ipcluster["<STR_LIT>"], <EOL>data._ipcluster["<STR_LIT>"],<EOL>data._ipcluster["<STR_LIT>"],<EOL>iparg)<EOL>try: <EOL><INDENT>LOGGER.info(shlex.split(standard))<EOL>subprocess.check_call(shlex.split(standard), <EOL>stderr=subprocess.STDOUT,<EOL>stdout=subprocess.PIPE)<EOL><DEDENT>except subprocess.CalledProcessError as inst:<EOL><INDENT>LOGGER.debug("<STR_LIT>")<EOL>raise<EOL><DEDENT>except Exception as inst:<EOL><INDENT>sys.exit("<STR_LIT>".format(inst))<EOL><DEDENT>


<EOL>data._ipcluster["<STR_LIT>"] = "<STR_LIT>"+str(os.getpid())<EOL>start_ipcluster(data)<EOL>return data<EOL>

The name is a unique id that keeps this __init__ of ipyrad distinct
from interfering with other ipcontrollers. Run statements are wrapped
so that ipcluster will be killed on exit.


def get_client(cluster_id, profile, engines, timeout, cores, quiet, spacer, **kwargs):

<EOL>save_stdout = sys.stdout <EOL>save_stderr = sys.stderr<EOL>sys.stdout = cStringIO.StringIO()<EOL>sys.stderr = cStringIO.StringIO()<EOL>connection_string = "<STR_LIT>".format(spacer)<EOL>try: <EOL><INDENT>if profile not in [None, "<STR_LIT:default>"]:<EOL><INDENT>args = {'<STR_LIT>': profile, "<STR_LIT>": timeout}<EOL><DEDENT>else:<EOL><INDENT>clusterargs = [cluster_id, profile, timeout]<EOL>argnames = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>args = {key:value for key, value in zip(argnames, clusterargs)}<EOL><DEDENT>ipyclient = ipp.Client(**args)<EOL>sys.stdout = save_stdout<EOL>sys.stderr = save_stderr<EOL>if (engines == "<STR_LIT>") or ("<STR_LIT>" in cluster_id):<EOL><INDENT>if not quiet:<EOL><INDENT>print(connection_string)<EOL><DEDENT><DEDENT>for _ in range(<NUM_LIT>):<EOL><INDENT>initid = len(ipyclient)<EOL>time.sleep(<NUM_LIT>)<EOL>if (engines == "<STR_LIT>") or ("<STR_LIT>" in cluster_id):<EOL><INDENT>if cores:<EOL><INDENT>time.sleep(<NUM_LIT:0.1>)<EOL>if initid == cores:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if initid:<EOL><INDENT>time.sleep(<NUM_LIT:3>)<EOL>if len(ipyclient) == initid:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if cores:<EOL><INDENT>if initid == cores:<EOL><INDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if initid:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>except KeyboardInterrupt as inst:<EOL><INDENT>sys.stdout = save_stdout<EOL>sys.stderr = save_stderr<EOL>raise inst<EOL><DEDENT>except IOError as inst:<EOL><INDENT>sys.stdout = save_stdout<EOL>sys.stderr = save_stderr<EOL>if "<STR_LIT>" in cluster_id:<EOL><INDENT>raise IPyradWarningExit(NO_IPCLUSTER_CLI)<EOL><DEDENT>else:<EOL><INDENT>raise IPyradWarningExit(NO_IPCLUSTER_API)<EOL><DEDENT><DEDENT>except (ipp.TimeoutError, ipp.NoEnginesRegistered) as inst:<EOL><INDENT>sys.stdout = save_stdout<EOL>sys.stderr = save_stderr<EOL>raise inst<EOL><DEDENT>except Exception as inst:<EOL><INDENT>sys.stdout = save_stdout<EOL>sys.stderr = save_stderr<EOL>raise inst<EOL><DEDENT>finally:<EOL><INDENT>sys.stdout = save_stdout<EOL>sys.stderr = save_stderr<EOL><DEDENT>return ipyclient<EOL>

Creates a client to view ipcluster engines for a given profile and
returns it with at least one engine spun up and ready to go. If no
engines are found after nwait amount of time then an error is raised.
If engines==MPI it waits a bit longer to find engines. If the number
of engines is set then it waits even longer to try to find that number
of engines.


try: <EOL><INDENT>name = pinfo[str(param)][<NUM_LIT:0>].strip().split("<STR_LIT:U+0020>")[<NUM_LIT:1>]<EOL><DEDENT>except (KeyError, ValueError) as err:<EOL><INDENT>print("<STR_LIT>".format(param), err)<EOL>raise<EOL><DEDENT>return name<EOL>

Get the param name from the dict index value.


<EOL>if short:<EOL><INDENT>desc = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>desc = <NUM_LIT:0><EOL><DEDENT>try: <EOL><INDENT>description = pinfo[str(param)][desc]<EOL><DEDENT>except (KeyError, ValueError) as err:<EOL><INDENT>print("<STR_LIT>".format(param), err)<EOL>raise<EOL><DEDENT>return description<EOL>

Returns detailed information for the numbered parameter.
        Further information is available in the tutorial.
        Unlike params() this function doesn't deal well with *
        It only takes one parameter at a time and returns the desc


if short:<EOL><INDENT>desc = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>desc = <NUM_LIT:0><EOL><DEDENT>if param == "<STR_LIT:*>":<EOL><INDENT>for key in pinfo:<EOL><INDENT>print(pinfo[str(key)][desc])<EOL><DEDENT><DEDENT>elif param:<EOL><INDENT>try:<EOL><INDENT>print(pinfo[str(param)][desc])<EOL><DEDENT>except (KeyError, ValueError) as err:<EOL><INDENT>print("<STR_LIT>", err)<EOL>raise<EOL><DEDENT><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL>for key in pinfo:<EOL><INDENT>print(pinfo[str(key)][desc].split("<STR_LIT:\n>")[<NUM_LIT:1>][<NUM_LIT:2>:-<NUM_LIT:10>])<EOL><DEDENT><DEDENT>

This is the human readable version of the paramsinfo() function.
        You give it a param and it prints to stdout.


<EOL>returndict = OrderedDict([<EOL>("<STR_LIT:name>", self.name),<EOL>("<STR_LIT>", self.barcode),<EOL>("<STR_LIT>", self.files),<EOL>("<STR_LIT>", {<EOL>"<STR_LIT>": self.stats_dfs.s1.to_dict(),<EOL>"<STR_LIT>": self.stats_dfs.s2.to_dict(),                <EOL>"<STR_LIT>": self.stats_dfs.s3.to_dict(),<EOL>"<STR_LIT>": self.stats_dfs.s4.to_dict(),<EOL>"<STR_LIT>": self.stats_dfs.s5.to_dict(),<EOL>}),<EOL>("<STR_LIT>", self.stats.to_dict()),<EOL>("<STR_LIT>", self.depths)<EOL>])<EOL>return returndict<EOL>

Write to dict including data frames. All sample dicts
are combined in save() to dump JSON output


<EOL>try:<EOL><INDENT>with open(args.params) as paramsin:<EOL><INDENT>plines = paramsin.readlines()<EOL><DEDENT><DEDENT>except IOError as _:<EOL><INDENT>sys.exit("<STR_LIT>")<EOL><DEDENT>legacy_version = <NUM_LIT:0><EOL>try:<EOL><INDENT>legacy_version = <NUM_LIT:1><EOL>if not len(plines[<NUM_LIT:0>].split()[<NUM_LIT:0>]) == <NUM_LIT:7>:<EOL><INDENT>raise IPyradWarningExit("""<STR_LIT>""".format(args.params, ip.__version__))<EOL><DEDENT><DEDENT>except IndexError:<EOL><INDENT>raise IPyradWarningExit("""<STR_LIT>""".format(args.params))<EOL><DEDENT>if legacy_version:<EOL><INDENT>pass<EOL><DEDENT>items = [i.split("<STR_LIT>")[<NUM_LIT:0>].strip() for i in plines[<NUM_LIT:1>:] if not i.strip() == "<STR_LIT>"]<EOL>keys = ip.Assembly('<STR_LIT:null>', quiet=True).paramsdict.keys()<EOL>parsedict = {str(i):j for i, j in zip(keys, items)}<EOL>return parsedict<EOL>

Parse the params file args, create and return Assembly object.


<EOL>project_dir = parsedict["<STR_LIT>"]<EOL>if not project_dir:<EOL><INDENT>project_dir = "<STR_LIT>"<EOL><DEDENT>assembly_name = parsedict["<STR_LIT>"]<EOL>my_assembly = os.path.join(project_dir, assembly_name)<EOL>if not os.path.isdir(project_dir):<EOL><INDENT>msg = """<STR_LIT>""".format(project_dir)<EOL>sys.exit(msg)<EOL><DEDENT>if not assembly_name:<EOL><INDENT>msg = """<STR_LIT>""".format(project_dir)<EOL>raise IPyradError(msg)<EOL><DEDENT>data = ip.load_json(my_assembly, quiet=True, cli=True)<EOL>print("<STR_LIT>".format(data.name)+"<STR_LIT>")<EOL>if not data.stats.empty:<EOL><INDENT>print(data.stats)<EOL>print("<STR_LIT>"+"<STR_LIT>")<EOL>fullcurdir = os.path.realpath(os.path.curdir)<EOL>for i in range(<NUM_LIT:1>, <NUM_LIT:8>):<EOL><INDENT>key = "<STR_LIT:s>"+str(i)<EOL>try:<EOL><INDENT>val = data.stats_files[key]<EOL>val = val.replace(fullcurdir, "<STR_LIT:.>")                <EOL>print("<STR_LIT>".format(i, val))<EOL><DEDENT>except (KeyError, AttributeError):<EOL><INDENT>print("<STR_LIT>".format(i))<EOL><DEDENT><DEDENT>print("<STR_LIT:\n>")<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT>

loads assembly or dies, and print stats to screen


<EOL>data = getassembly(args, parsedict)<EOL>bargs = args.branch<EOL>newname = bargs[<NUM_LIT:0>]<EOL>if newname.endswith("<STR_LIT>"):<EOL><INDENT>newname = newname[:-<NUM_LIT:4>]<EOL><DEDENT>if len(bargs) > <NUM_LIT:1>:<EOL><INDENT>if any([x.stats.state == <NUM_LIT:6> for x in data.samples.values()]):<EOL><INDENT>pass<EOL><DEDENT>subsamples = bargs[<NUM_LIT:1>:]<EOL>if bargs[<NUM_LIT:1>] == "<STR_LIT:->":<EOL><INDENT>fails = [i for i in subsamples[<NUM_LIT:1>:] if i not in data.samples.keys()]<EOL>if any(fails):<EOL><INDENT>raise IPyradWarningExit("<STR_LIT>".format("<STR_LIT>".join([i for i in fails])))<EOL><DEDENT>print("<STR_LIT>".format(len(subsamples)-<NUM_LIT:1>))<EOL>subsamples = list(set(data.samples.keys()) - set(subsamples))<EOL><DEDENT>if os.path.exists(bargs[<NUM_LIT:1>]):<EOL><INDENT>new_data = data.branch(newname, infile=bargs[<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>new_data = data.branch(newname, subsamples)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>new_data = data.branch(newname, None)<EOL><DEDENT>print("<STR_LIT>".format(new_data.name, len(new_data.samples)))<EOL>print("<STR_LIT>".format("<STR_LIT>"+new_data.name+"<STR_LIT>"))<EOL>new_data.write_params("<STR_LIT>"+new_data.name+"<STR_LIT>", force=args.force)<EOL>

Load the passed in assembly and create a branch. Copy it
to a new assembly, and also write out the appropriate params.txt


print("<STR_LIT>".format(args.merge[<NUM_LIT:1>:]))<EOL>if len(args.merge) < <NUM_LIT:3>:<EOL><INDENT>sys.exit(_WRONG_NUM_CLI_MERGE)<EOL><DEDENT>newname = args.merge[<NUM_LIT:0>]<EOL>if os.path.exists(newname) and "<STR_LIT>" in newname:<EOL><INDENT>sys.exit(_WRONG_ORDER_CLI_MERGE) <EOL><DEDENT>if os.path.exists("<STR_LIT>" + newname + "<STR_LIT>") and not args.force:<EOL><INDENT>sys.exit(_NAME_EXISTS_MERGE.format("<STR_LIT>" + newname + "<STR_LIT>"))<EOL><DEDENT>assemblies_to_merge = args.merge[<NUM_LIT:1>:]<EOL>for assembly in assemblies_to_merge:<EOL><INDENT>if not os.path.exists(assembly):<EOL><INDENT>sys.exit(_DOES_NOT_EXIST_MERGE.format(assembly))<EOL><DEDENT><DEDENT>assemblies = []<EOL>for params_file in args.merge[<NUM_LIT:1>:]:<EOL><INDENT>args.params = params_file<EOL>parsedict = parse_params(args)<EOL>assemblies.append(getassembly(args, parsedict))<EOL><DEDENT>merged_assembly = ip.merge(newname, assemblies)<EOL>merged_assembly.write_params("<STR_LIT>".format(newname), force=args.force)<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>".format(newname))<EOL>

merge all given assemblies into a new assembly. Copies the params
from the first passed in extant assembly. this function is called
with the ipyrad -m flag. You must pass it at least 3 values, the first
is a new assembly name (a new `param-newname.txt` will be created).
The second and third args must be params files for currently existing
assemblies. Any args beyond the third must also be params file for
extant assemblies.