def make_network_graph(compact, expression_names, lookup_names):

nodes = set()<EOL>edges = defaultdict(set) <EOL>var_name_dict = dict()<EOL>if expression_names:<EOL><INDENT>for e in graphviz_items: <EOL><INDENT>if e in expression_names:<EOL><INDENT>var_name_dict[e.vindex] = expression_names[e]<EOL><DEDENT><DEDENT><DEDENT>rnn_bldr_name = defaultdict(lambda: chr(len(rnn_bldr_name)+ord('<STR_LIT:A>')))<EOL>def vidx2str(vidx): return '<STR_LIT>' % ('<STR_LIT:N>', vidx)<EOL>for e in graphviz_items: <EOL><INDENT>vidx = e.vindex<EOL>f_name = e.name<EOL>args = e.args<EOL>output_dim = e.dim<EOL>input_dim = None <EOL>children = set()<EOL>node_type = '<STR_LIT>'<EOL>if f_name == '<STR_LIT>':<EOL><INDENT>[_dim] = args<EOL>arg_strs = []<EOL><DEDENT>elif f_name == '<STR_LIT>':<EOL><INDENT>[_v] = args<EOL>arg_strs = []<EOL><DEDENT>elif f_name == '<STR_LIT>':<EOL><INDENT>[_d1, _d2] = args<EOL>arg_strs = []<EOL><DEDENT>elif f_name == '<STR_LIT>':<EOL><INDENT>[_v, _d] = args<EOL>arg_strs = []<EOL><DEDENT>elif f_name == '<STR_LIT>':<EOL><INDENT>[_dim] = args<EOL>arg_strs = []<EOL>if compact:<EOL><INDENT>if vidx in var_name_dict:<EOL><INDENT>f_name = var_name_dict[vidx]<EOL><DEDENT><DEDENT>node_type = '<STR_LIT>'<EOL><DEDENT>elif f_name == '<STR_LIT>':<EOL><INDENT>[_dim] = args<EOL>arg_strs = []<EOL>if compact:<EOL><INDENT>if vidx in var_name_dict:<EOL><INDENT>f_name = var_name_dict[vidx]<EOL><DEDENT><DEDENT>node_type = '<STR_LIT>'<EOL><DEDENT>elif f_name == '<STR_LIT>':<EOL><INDENT>[p, idx, update] = args<EOL>[_dim] = p.args<EOL>if vidx in var_name_dict:<EOL><INDENT>name = var_name_dict[vidx]<EOL><DEDENT>else:<EOL><INDENT>name = None<EOL><DEDENT>item_name = None<EOL>if lookup_names and p in expression_names:<EOL><INDENT>param_name = expression_names[p]<EOL>if param_name in lookup_names:<EOL><INDENT>item_name = '<STR_LIT>' % (lookup_names[param_name][idx],)<EOL><DEDENT><DEDENT>if compact:<EOL><INDENT>if item_name is not None:<EOL><INDENT>f_name = item_name<EOL><DEDENT>elif name is not None:<EOL><INDENT>f_name = '<STR_LIT>' % (name, idx)<EOL><DEDENT>else:<EOL><INDENT>f_name = '<STR_LIT>' % (idx)<EOL><DEDENT>arg_strs = []<EOL><DEDENT>else:<EOL><INDENT>arg_strs = [var_name_dict.get(p.vindex, '<STR_LIT>' % (p.vindex))]<EOL>if item_name is not None:<EOL><INDENT>arg_strs.append(item_name)<EOL><DEDENT>vocab_size = _dim[<NUM_LIT:0>]<EOL>arg_strs.extend(['<STR_LIT:%s>' % (idx), '<STR_LIT:%s>' % (vocab_size), '<STR_LIT>' if update else '<STR_LIT>'])<EOL><DEDENT><DEDENT>elif f_name == '<STR_LIT>':<EOL><INDENT>[arg, input_dim, bldr_type, bldr_num, state_idx] = args <EOL>rnn_name = rnn_bldr_name[bldr_num]<EOL>if bldr_type.endswith('<STR_LIT>'):<EOL><INDENT>bldr_type[:-len('<STR_LIT>')]<EOL><DEDENT>f_name = '<STR_LIT>' % (bldr_type, rnn_name, state_idx)<EOL>if not compact:<EOL><INDENT>i = arg.vindex<EOL>s = var_name_dict.get(i, '<STR_LIT>' % (i))<EOL>arg_strs = [s]<EOL><DEDENT>else:<EOL><INDENT>arg_strs = []<EOL><DEDENT>children.add(vidx2str(arg.vindex))<EOL>node_type = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>arg_strs = []<EOL>for arg in args:<EOL><INDENT>if isinstance(arg, Expression):<EOL><INDENT>if not compact:<EOL><INDENT>i = arg.vindex<EOL>s = var_name_dict.get(i, '<STR_LIT>' % (i))<EOL>arg_strs.append(s)<EOL><DEDENT>children.add(vidx2str(arg.vindex))<EOL><DEDENT>elif isinstance(arg, float) and compact:<EOL><INDENT>s = re.sub('<STR_LIT>', '<STR_LIT>', '<STR_LIT>' % (arg))<EOL>if s == '<STR_LIT>':<EOL><INDENT>s = str(arg)<EOL><DEDENT>arg_strs.append(s)<EOL><DEDENT>else:<EOL><INDENT>arg_strs.append(str(arg))<EOL>

Make a network graph, represented as of nodes and a set of edges.
  The nodes are represented as tuples: (name: string, input_dim: Dim, label: string, output_dim: Dim, children: set[name], features: string)
#   The edges are represented as dict of children to sets of parents: (child: string) -> [(parent: string, features: string)]


def __init__(self, num_layers, input_dim, hidden_dim, model, rnn_builder_factory):

assert num_layers > <NUM_LIT:0><EOL>assert hidden_dim % <NUM_LIT:2> == <NUM_LIT:0><EOL>self.builder_layers = []<EOL>f = rnn_builder_factory(<NUM_LIT:1>, input_dim, hidden_dim/<NUM_LIT:2>, model)<EOL>b = rnn_builder_factory(<NUM_LIT:1>, input_dim, hidden_dim/<NUM_LIT:2>, model)<EOL>self.builder_layers.append((f,b))<EOL>for _ in xrange(num_layers-<NUM_LIT:1>):<EOL><INDENT>f = rnn_builder_factory(<NUM_LIT:1>, hidden_dim, hidden_dim/<NUM_LIT:2>, model)<EOL>b = rnn_builder_factory(<NUM_LIT:1>, hidden_dim, hidden_dim/<NUM_LIT:2>, model)<EOL>self.builder_layers.append((f,b))<EOL><DEDENT>

@param num_layers: depth of the BiRNN
@param input_dim: size of the inputs
@param hidden_dim: size of the outputs (and intermediate layer representations)
@param model
@param rnn_builder_factory: RNNBuilder subclass, e.g. LSTMBuilder


for e in es:<EOL><INDENT>ensure_freshness(e)<EOL><DEDENT>for (fb,bb) in self.builder_layers[:-<NUM_LIT:1>]:<EOL><INDENT>fs = fb.initial_state().transduce(es)<EOL>bs = bb.initial_state().transduce(reversed(es))<EOL>es = [concatenate([f,b]) for f,b in zip(fs, reversed(bs))]<EOL><DEDENT>(fb,bb) = self.builder_layers[-<NUM_LIT:1>]<EOL>fs = fb.initial_state().add_inputs(es)<EOL>bs = bb.initial_state().add_inputs(reversed(es))<EOL>return [(f,b) for f,b in zip(fs, reversed(bs))]<EOL>

returns the list of state pairs (stateF, stateB) obtained by adding
inputs to both forward (stateF) and backward (stateB) RNNs.

@param es: a list of Expression

see also transduce(xs)

.transduce(xs) is different from .add_inputs(xs) in the following way:

    .add_inputs(xs) returns a list of RNNState pairs. RNNState objects can be
     queried in various ways. In particular, they allow access to the previous
     state, as well as to the state-vectors (h() and s() )

    .transduce(xs) returns a list of Expression. These are just the output
     expressions. For many cases, this suffices.
     transduce is much more memory efficient than add_inputs.


if indent_str is None:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>return indent_str<EOL><DEDENT>


out_args = []<EOL>if arg_str is None:<EOL><INDENT>return out_str<EOL><DEDENT>args = arg_str.split('<STR_LIT:U+002C>')<EOL>for arg in args:<EOL><INDENT>components = arg.split('<STR_LIT:=>')<EOL>name_and_type=components[<NUM_LIT:0>].split('<STR_LIT:U+0020>')<EOL>if name_and_type[-<NUM_LIT:1>]=='<STR_LIT>' and len(name_and_type)><NUM_LIT:1>:<EOL><INDENT>name=name_and_type[-<NUM_LIT:2>]<EOL><DEDENT>else:<EOL><INDENT>name=name_and_type[-<NUM_LIT:1>]<EOL><DEDENT>if len(components)><NUM_LIT:1>:<EOL><INDENT>name+='<STR_LIT:=>'+components[<NUM_LIT:1>]<EOL><DEDENT>out_args.append(name)<EOL><DEDENT>return '<STR_LIT:U+002C>'.join(out_args)<EOL>

Remove types from function arguments in cython


if dataset is "<STR_LIT>":<EOL><INDENT>fname_img = os.path.join(path, "<STR_LIT>")<EOL>fname_lbl = os.path.join(path, "<STR_LIT>")<EOL><DEDENT>elif dataset is "<STR_LIT>":<EOL><INDENT>fname_img = os.path.join(path, "<STR_LIT>")<EOL>fname_lbl = os.path.join(path, "<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>with open(fname_lbl, "<STR_LIT:rb>") as flbl:<EOL><INDENT>magic, num = struct.unpack("<STR_LIT>", flbl.read(<NUM_LIT:8>))<EOL>labels = np.fromfile(flbl, dtype=np.int8)<EOL><DEDENT>with open(fname_img, "<STR_LIT:rb>") as fimg:<EOL><INDENT>magic, num, rows, cols = struct.unpack("<STR_LIT>", fimg.read(<NUM_LIT:16>))<EOL>images = np.multiply(<EOL>np.fromfile(fimg, dtype=np.uint8).reshape(len(labels), rows*cols),<EOL><NUM_LIT:1.0> / <NUM_LIT>)<EOL><DEDENT>get_instance = lambda idx: (labels[idx], images[idx].reshape(<NUM_LIT:1>, <NUM_LIT>, <NUM_LIT>))<EOL>size_reset = lambda x: x.reshape(<NUM_LIT:1>, <NUM_LIT>, <NUM_LIT>)<EOL>return list(map(size_reset, images))<EOL>

wget -O - http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz | gunzip > train-images-idx3-ubyte
wget -O - http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz | gunzip > train-labels-idx1-ubyte
wget -O - http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz | gunzip > t10k-images-idx3-ubyte
wget -O - http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz | gunzip > t10k-labels-idx1-ubyte


def make_grid(tensor, nrow=<NUM_LIT:8>, padding=<NUM_LIT:2>, pad_value=<NUM_LIT:0>):

if not (isinstance(tensor, np.ndarray) or<EOL>(isinstance(tensor, list) and all(isinstance(t, np.ndarray) for t in tensor))):<EOL><INDENT>raise TypeError('<STR_LIT>'.format(type(tensor)))<EOL><DEDENT>if isinstance(tensor, list):<EOL><INDENT>tensor = np.stack(tensor, <NUM_LIT:0>)<EOL><DEDENT>if tensor.ndim == <NUM_LIT:2>:  <EOL><INDENT>tensor = tensor.reshape((<NUM_LIT:1>, tensor.shape[<NUM_LIT:0>], tensor.shape[<NUM_LIT:1>]))<EOL><DEDENT>if tensor.ndim == <NUM_LIT:3>:<EOL><INDENT>if tensor.shape[<NUM_LIT:0>] == <NUM_LIT:1>:  <EOL><INDENT>tensor = np.concatenate((tensor, tensor, tensor), <NUM_LIT:0>)<EOL><DEDENT>tensor = tensor.reshape((<NUM_LIT:1>, tensor.shape[<NUM_LIT:0>], tensor.shape[<NUM_LIT:1>], tensor.shape[<NUM_LIT:2>]))<EOL><DEDENT>if tensor.ndim == <NUM_LIT:4> and tensor.shape[<NUM_LIT:1>] == <NUM_LIT:1>:  <EOL><INDENT>tensor = np.concatenate((tensor, tensor, tensor), <NUM_LIT:1>)<EOL><DEDENT>if tensor.shape[<NUM_LIT:0>] == <NUM_LIT:1>:<EOL><INDENT>return np.squeeze(tensor)<EOL><DEDENT>nmaps = tensor.shape[<NUM_LIT:0>]<EOL>xmaps = min(nrow, nmaps)<EOL>ymaps = int(math.ceil(float(nmaps) / xmaps))<EOL>height, width = int(tensor.shape[<NUM_LIT:2>] + padding), int(tensor.shape[<NUM_LIT:3>] + padding)<EOL>grid = np.ones((<NUM_LIT:3>, height * ymaps + padding, width * xmaps + padding)) * pad_value<EOL>k = <NUM_LIT:0><EOL>for y in range(ymaps):<EOL><INDENT>for x in range(xmaps):<EOL><INDENT>if k >= nmaps:<EOL><INDENT>break<EOL><DEDENT>grid[:, y * height + padding:(y+<NUM_LIT:1>) * height,x * width + padding:(x+<NUM_LIT:1>) * width] = tensor[k]<EOL>k = k + <NUM_LIT:1><EOL><DEDENT><DEDENT>return grid<EOL>

Make a grid of images, via numpy.

    Args:
        tensor (Tensor or list): 4D mini-batch Tensor of shape (B x C x H x W)
            or a list of images all of the same size.
        nrow (int, optional): Number of images displayed in each row of the grid.
            The Final grid size is (B / nrow, nrow). Default is 8.
        padding (int, optional): amount of padding. Default is 2.
        pad_value (float, optional): Value for the padded pixels.


def save_image(tensor, filename, nrow=<NUM_LIT:8>, padding=<NUM_LIT:2>, pad_value=<NUM_LIT:0>):

from PIL import Image<EOL>grid = make_grid(tensor, nrow=nrow, padding=padding, pad_value=pad_value)<EOL>im = Image.fromarray(pre_pillow_float_img_process(grid))<EOL>im.save(filename)<EOL>

Save a given Tensor into an image file.

    Args:
        tensor (Tensor or list): Image to be saved. If given a mini-batch tensor,
            saves the tensor as a grid of images by calling ``make_grid``.
        **kwargs: Other arguments are documented in ``make_grid``.


if sys.platform == '<STR_LIT:win32>':<EOL><INDENT>user_config = os.path.expanduser(r'<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>user_config = os.path.join(<EOL>os.getenv('<STR_LIT>') or os.path.expanduser('<STR_LIT>'),<EOL>'<STR_LIT>'<EOL>)<EOL><DEDENT>if os.path.exists(user_config):<EOL><INDENT>with open(user_config, '<STR_LIT:r>') as config_file:<EOL><INDENT>config = pytoml.load(config_file).get('<STR_LIT>', {})<EOL><DEDENT>config = merge_dict(get_default_config(), config)<EOL>config = process_extensions(config, project_path, use_cache=use_cache)<EOL>return config<EOL><DEDENT>return None<EOL>

Produces a TidyPy configuration that incorporates the configuration files
stored in the current user's home directory.

:param project_path: the path to the project that is going to be analyzed
:type project_path: str
:param use_cache:
    whether or not to use cached versions of any remote/referenced TidyPy
    configurations. If not specified, defaults to ``True``.
:type use_cache: bool
:rtype: dict


<EOL>if not hasattr(get_reports, '<STR_LIT>'):<EOL><INDENT>get_reports._CACHE = dict()<EOL>for entry in pkg_resources.iter_entry_points('<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>get_reports._CACHE[entry.name] = entry.load()<EOL><DEDENT>except ImportError as exc:  <EOL><INDENT>output_error(<EOL>'<STR_LIT>' % (<EOL>entry,<EOL>entry.dist,<EOL>exc,<EOL>),<EOL>)<EOL><DEDENT><DEDENT><DEDENT>return get_reports._CACHE<EOL>

Retrieves the TidyPy issue reports that are available in the current Python
environment.

The returned dictionary has keys are the report names and values are the
report classes.

:rtype: dict


cache_path = get_cache_path(location)<EOL>if location:<EOL><INDENT>os.remove(cache_path)<EOL><DEDENT>else:<EOL><INDENT>shutil.rmtree(cache_path)<EOL><DEDENT>

Clears out the cache of TidyPy configurations that were retrieved from
outside the normal locations.


with _AST_CACHE_LOCK:<EOL><INDENT>if filepath not in _AST_CACHE:<EOL><INDENT>source = read_file(filepath)<EOL>_AST_CACHE[filepath] = ast.parse(source, filename=filepath)<EOL><DEDENT><DEDENT>return _AST_CACHE[filepath]<EOL>

Retrieves the AST of the specified file.

This function performs simple caching so that the same file isn't read or
parsed more than once per process.

:param filepath: the file to parse
:type filepath: str
:returns: ast.AST


return self._stdout.getvalue()<EOL>

Retrieves the content that was written to ``stdout``.

:returns: str


for mask in masks:<EOL><INDENT>if mask.search(target):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

Determines whether or not the target string matches any of the regular
expressions specified.

:param target: the string to check
:type target: str
:param masks: the regular expressions to check against
:type masks: list(regular expression object)
:returns: bool


click.echo(click.style(msg, fg='<STR_LIT>'), err=True)<EOL>

Prints the specified string to ``stderr``.

:param msg: the message to print
:type msg: str


return _REQUESTS<EOL>

Retrieves a ``requests`` object to use within TidyPy.


if not masks:<EOL><INDENT>masks = []<EOL><DEDENT>elif not isinstance(masks, (list, tuple)):<EOL><INDENT>masks = [masks]<EOL><DEDENT>return [<EOL>re.compile(mask)<EOL>for mask in masks<EOL>]<EOL>

Compiles a list of regular expressions.

:param masks: the regular expressions to compile
:type masks: list(str) or str
:returns: list(regular expression object)


return {<EOL>'<STR_LIT>': True,<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': {},<EOL>}<EOL>

Produces a tool configuration stanza that acts as the base/default for
this tool.

rtype: dict


return True<EOL>

Indicates whether or not this tool can be executed now. Useful when you
need to check for certain environmental conditions (e.g., Python
version, dependency availability, etc).

Unless overridden, always returns ``True``.

:rtype: bool


with self._lock:<EOL><INDENT>if tool in self.current_tools:<EOL><INDENT>self.current_tools.remove(tool)<EOL>self.completed_tools.append(tool)<EOL><DEDENT><DEDENT>

Called when an individual tool completes execution.

:param tool: the name of the tool that completed
:type tool: str


Called after all tools in the suite have completed.


@classmethod<EOL><INDENT>def retrieve(cls, location, project_path):<DEDENT>

raise NotImplementedError()<EOL>

Retrieves a TidyPy configuration from the specified location.

:param location:
    a URI indicating where to retrieve the TidyPy configuration from
:type location: str
:param project_path: the full path to the project's base
:type project_path: str
:rtype: dict


@classmethod<EOL><INDENT>def parse(cls, content, is_pyproject=False):<DEDENT>

parsed = pytoml.loads(content)<EOL>if is_pyproject:<EOL><INDENT>parsed = parsed.get('<STR_LIT>', {})<EOL><DEDENT>parsed = parsed.get('<STR_LIT>', {})<EOL>return parsed<EOL>

A convenience method for parsing a TOML-serialized configuration.

:param content: a TOML string containing a TidyPy configuration
:type content: str
:param is_pyproject:
    whether or not the content is (or resembles) a ``pyproject.toml``
    file, where the TidyPy configuration is located within a key named
    ``tool``.
:type is_pyproject: bool
:rtype: dict


click.echo(text_type(msg), nl=newline, file=self.output_file)<EOL>

Writes the specified string to the output target of the report.

:param msg: the message to output.
:type msg: str
:param newline:
    whether or not to append a newline to the end of the message
:type newline: str


return Path(filename).relative_to(self.base_path).as_posix()<EOL>

Generates a path for the specified filename that is relative to the
project path.

:param filename: the filename to generate the path for
:type filename: str
:rtype: str


return read_file(filepath)<EOL>

Retrieves the contents of the specified file.

This function performs simple caching so that the same file isn't read
more than once per process.

:param filepath: the file to read.
:type filepath: str
:rtype: str


return text_type(self.base_path)<EOL>

The path to the project that this Finder is operating from.


if self.is_excluded(path):<EOL><INDENT>return True<EOL><DEDENT>return matches_masks(path.name, ALWAYS_EXCLUDED_DIRS)<EOL>

Determines whether or not the specified directory is excluded by the
project's configuration.

:param path: the path to check
:type path: pathlib.Path
:rtype: bool


self.base_path = Path(base_path).resolve()<EOL>self.excludes = compile_masks(config['<STR_LIT>'])<EOL>self._found = dict()<EOL>self._find(self.base_path)<EOL>self._found = dict([<EOL>(dirname, files)<EOL>for dirname, files in iteritems(self._found)<EOL>if files<EOL>])<EOL>

:param base_path: the path to the base of the project
:type base_path: str
:param config: the configuration to use when searching the project
:type config: dict


def directories(self, filters=None, containing=None):

filters = compile_masks(filters or [r'<STR_LIT>'])<EOL>contains = compile_masks(containing)<EOL>for dirname, files in iteritems(self._found):<EOL><INDENT>relpath = text_type(Path(dirname).relative_to(self.base_path))<EOL>if matches_masks(relpath, filters):<EOL><INDENT>if not contains or self._contains(files, contains):<EOL><INDENT>yield dirname<EOL><DEDENT><DEDENT><DEDENT>

A generator that produces a sequence of paths to directories in the
project that matches the specified filters.

:param filters:
    the regular expressions to use when finding directories in the
    project. If not specified, all directories are returned.
:type filters: list(str)
:param containing:
    if a directory passes through the specified filters, it is checked
    for the presence of a file that matches one of the regular
    expressions in this parameter.
:type containing: list(str)


timedelta = datetime.timedelta(seconds=int(seconds))<EOL>mm, ss = divmod(timedelta.seconds, <NUM_LIT>)<EOL>if mm < <NUM_LIT>:<EOL><INDENT>return "<STR_LIT>" % (mm, ss)<EOL><DEDENT>hh, mm = divmod(mm, <NUM_LIT>)<EOL>if hh < <NUM_LIT>:<EOL><INDENT>return "<STR_LIT>" % (hh, mm, ss)<EOL><DEDENT>dd, hh = divmod(mm, <NUM_LIT>)<EOL>return "<STR_LIT>" % (dd, hh, mm, ss)<EOL>


if pbar.currval == <NUM_LIT:0>:<EOL><INDENT>return self.eta_format % '<STR_LIT>'<EOL><DEDENT>elif pbar.finished:<EOL><INDENT>return self.complete_format % self.format_time(pbar.seconds_elapsed)<EOL><DEDENT>else:<EOL><INDENT>elapsed = pbar.seconds_elapsed<EOL>eta = elapsed * pbar.maxval / pbar.currval - elapsed<EOL>return self.eta_format % self.format_time(eta)<EOL><DEDENT>

Updates the widget to show the ETA or total time when finished.


raise NotImplementedError()<EOL>

Subclasses should implement this function - called every loop iteration.


assert not self.has_started(), "<STR_LIT>"<EOL>self._stop_event = Event()<EOL>self._greenlet = gevent.spawn(self._loop)<EOL>

Starts the loop. Calling a running loop is an error.


loop = GeventLoop(interval, callback)<EOL>loop.start()<EOL>try:<EOL><INDENT>yield loop<EOL><DEDENT>finally:<EOL><INDENT>if loop.has_started():<EOL><INDENT>loop.stop()<EOL><DEDENT><DEDENT>

When entering the context, spawns a greenlet that sleeps for `interval` seconds between `callback` executions.
When leaving the context stops the greenlet.
The yielded object is the `GeventLoop` object so the loop can be stopped from within the context.

For example:
```
with loop_in_background(60.0, purge_cache) as purge_cache_job:
    ...
    ...
    if should_stop_cache():
        purge_cache_job.stop()
```


assert self.has_started(), "<STR_LIT>"<EOL>greenlet = self._greenlet if gevent.getcurrent() != self._greenlet else None<EOL>self._stop_event.set()<EOL>if greenlet is not None and timeout is not None:<EOL><INDENT>greenlet.join(timeout)<EOL>return greenlet.ready<EOL><DEDENT>else:<EOL><INDENT>return True<EOL><DEDENT>

Stops a running loop and waits for it to end if timeout is set. Calling a non-running loop is an error.
:param timeout: time (in seconds) to wait for the loop to end after signalling it. ``None`` is to block till it
ends.
:return: True if the loop stopped, False if still stopping.


try:<EOL><INDENT>return json.loads(json_string)<EOL><DEDENT>except:<EOL><INDENT>raise DecodeError()<EOL><DEDENT>


def wrapped_func(*args, **kwargs):<EOL><INDENT>try:<EOL><INDENT>result = original_func(*args, **kwargs)<EOL>return True, result<EOL><DEDENT>except:<EOL><INDENT>return False, sys.exc_info()<EOL><DEDENT><DEDENT>def new_func(*args, **kwargs):<EOL><INDENT>status, result = gevent.get_hub().threadpool.apply(wrapped_func, args, kwargs)<EOL>if status:<EOL><INDENT>return result<EOL><DEDENT>else:<EOL><INDENT>six.reraise(*result)<EOL><DEDENT><DEDENT>new_func.__name__ = original_func.__name__<EOL>new_func.__doc__ = "<STR_LIT>" + ("<STR_LIT:U+0020>" + original_func.__doc__ if original_func.__doc__ is not None else "<STR_LIT>")<EOL>return new_func<EOL>

Returns a function wrapper that defers function calls execute inside gevent's threadpool but keeps any exception
or backtrace in the caller's context.
:param original_func: function to wrap
:returns: wrapper function


def safe_joinall(greenlets, timeout=None, raise_error=False):

greenlets = list(greenlets)<EOL>try:<EOL><INDENT>gevent.joinall(greenlets, timeout=timeout, raise_error=raise_error)<EOL><DEDENT>except gevent.GreenletExit:<EOL><INDENT>[greenlet.kill() for greenlet in greenlets if not greenlet.ready()]<EOL>raise<EOL><DEDENT>return greenlets<EOL>

Wrapper for gevent.joinall if the greenlet that waits for the joins is killed, it kills all the greenlets it
joins for.


if not hasattr(instance_method, '<STR_LIT>'):<EOL><INDENT>return instance_method<EOL><DEDENT>def _func(*args, **kwargs):<EOL><INDENT>return instance_method(*args, **kwargs)<EOL><DEDENT>_func.__name__ = repr(instance_method)<EOL>return _func<EOL>

wraps(instancemethod) returns a function, not an instancemethod so its repr() is all messed up;
we want the original repr to show up in the logs, therefore we do this trick


global _uncaught_exception_handler<EOL>prev_handler, _uncaught_exception_handler = _uncaught_exception_handler, func<EOL>return prev_handler<EOL>

Sets a global greenlet uncaught exception handler that will get called if a greenlet spawned by one of this module's
wrappers raises an uncaught exception.
:param func: exception handler function that will receive the exc_info tuple as an argument
:returns: previous exception handler function or None


@job<EOL>def worker(wrapped, dkwargs, hash_value=None, *args, **kwargs):

if "<STR_LIT>" not in dkwargs:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg)<EOL><DEDENT>event = dkwargs['<STR_LIT>']<EOL>if "<STR_LIT>" not in kwargs:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg)<EOL><DEDENT>owner = kwargs['<STR_LIT>']<EOL>if "<STR_LIT>" not in kwargs:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg)<EOL><DEDENT>identifier = kwargs['<STR_LIT>']<EOL>senderobj = DjangoRQSenderable(<EOL>wrapped, dkwargs, hash_value, WEBHOOK_ATTEMPTS, *args, **kwargs<EOL>)<EOL>senderobj.webhook_target = WebhookTarget.objects.get(<EOL>event=event,<EOL>owner=owner,<EOL>identifier=identifier<EOL>)<EOL>senderobj.url = senderobj.webhook_target.target_url<EOL>senderobj.payload = senderobj.get_payload()<EOL>senderobj.payload['<STR_LIT>'] = getattr(kwargs['<STR_LIT>'], WEBHOOK_OWNER_FIELD)<EOL>senderobj.payload['<STR_LIT>'] = dkwargs['<STR_LIT>']<EOL>return senderobj.send()<EOL>

This is an asynchronous sender callable that uses the Django ORM to store
    webhooks. Redis is used to handle the message queue.

dkwargs argument requires the following key/values:

    :event: A string representing an event.

kwargs argument requires the following key/values

    :owner: The user who created/owns the event


data = dict(<EOL>payload=self.payload,<EOL>attempt=self.attempt,<EOL>success=self.success,<EOL>response_message=self.response_content,<EOL>hash_value=self.hash_value,<EOL>response_status=self.response.status_code,<EOL>notification=message,<EOL>created=timezone.now()<EOL>)<EOL>value = json.dumps(data, cls=StandardJSONEncoder)<EOL>key = make_key(self.event, self.owner.username, self.identifier)<EOL>redis.lpush(key, value)<EOL>

TODO: Add code to lpush to redis stack
        rpop when stack hits size 'X'


def redislog_callable(wrapped, dkwargs, hash_value=None, *args, **kwargs):

if "<STR_LIT>" not in dkwargs:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg)<EOL><DEDENT>event = dkwargs['<STR_LIT>']<EOL>if "<STR_LIT>" not in kwargs:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg)<EOL><DEDENT>owner = kwargs['<STR_LIT>']<EOL>if "<STR_LIT>" not in kwargs:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise TypeError(msg)<EOL><DEDENT>identifier = kwargs['<STR_LIT>']<EOL>senderobj = RedisLogSenderable(<EOL>wrapped, dkwargs, hash_value, WEBHOOK_ATTEMPTS, *args, **kwargs<EOL>)<EOL>try:<EOL><INDENT>senderobj.webhook_target = WebhookTarget.objects.get(<EOL>event=event,<EOL>owner=owner,<EOL>identifier=identifier<EOL>)<EOL><DEDENT>except WebhookTarget.DoesNotExist:<EOL><INDENT>return {"<STR_LIT:error>": "<STR_LIT>"}<EOL><DEDENT>senderobj.url = senderobj.webhook_target.target_url<EOL>senderobj.payload = senderobj.get_payload()<EOL>senderobj.payload['<STR_LIT>'] = getattr(kwargs['<STR_LIT>'], WEBHOOK_OWNER_FIELD)<EOL>senderobj.payload['<STR_LIT>'] = dkwargs['<STR_LIT>']<EOL>return senderobj.send()<EOL>

This is a synchronous sender callable that uses the Django ORM to store
    webhooks and Redis for the delivery log.

dkwargs argument requires the following key/values:

    :event: A string representing an event.

kwargs argument requires the following key/values

    :owner: The user who created/owns the event


if events is None:<EOL><INDENT>msg = "<STR_LIT>"<EOL>raise ImproperlyConfigured(msg)<EOL><DEDENT>try:<EOL><INDENT>choices = [(x, x) for x in events]<EOL><DEDENT>except TypeError:<EOL><INDENT>"""<STR_LIT>"""<EOL>msg = "<STR_LIT>"<EOL>raise ImproperlyConfigured(msg)<EOL><DEDENT>return choices<EOL>


def progressbar(njobs, finished, msg="<STR_LIT>", spacer="<STR_LIT:U+0020>"):

if njobs:<EOL><INDENT>progress = <NUM_LIT:100>*(finished / float(njobs))<EOL><DEDENT>else:<EOL><INDENT>progress = <NUM_LIT:100><EOL><DEDENT>hashes = '<STR_LIT:#>'*int(progress/<NUM_LIT>)<EOL>nohash = '<STR_LIT:U+0020>'*int(<NUM_LIT:20>-len(hashes))<EOL>if not ipyrad.__interactive__:<EOL><INDENT>msg = msg.rsplit("<STR_LIT:|>", <NUM_LIT:2>)[<NUM_LIT:0>]<EOL><DEDENT>args = [spacer, hashes+nohash, int(progress), msg]<EOL>print("<STR_LIT>".format(*args), end="<STR_LIT>")<EOL>sys.stdout.flush()<EOL>


<EOL>return seq.replace("<STR_LIT:A>", '<STR_LIT:t>').replace('<STR_LIT:T>', '<STR_LIT:a>').replace('<STR_LIT:C>', '<STR_LIT:g>').replace('<STR_LIT>', '<STR_LIT:c>').replace('<STR_LIT:n>', '<STR_LIT>').upper().replace("<STR_LIT>", "<STR_LIT:n>")<EOL>

returns a seq with complement. Preserves little n's for splitters.


try:<EOL><INDENT>r1file = tempfile.NamedTemporaryFile(mode='<STR_LIT:wb>', delete=False,<EOL>dir=data.dirs.edits,<EOL>suffix="<STR_LIT>")<EOL>r2file = tempfile.NamedTemporaryFile(mode='<STR_LIT:wb>', delete=False,<EOL>dir=data.dirs.edits,<EOL>suffix="<STR_LIT>")<EOL>r1dat = []<EOL>r2dat = []<EOL>for locus in clust:<EOL><INDENT>sname, seq = locus.split("<STR_LIT:\n>")<EOL>sname = "<STR_LIT:@>" + sname[<NUM_LIT:1>:]<EOL>r1, r2 = seq.split("<STR_LIT>")<EOL>r1dat.append("<STR_LIT>".format(sname, r1, "<STR_LIT:+>", "<STR_LIT:B>"*(len(r1))))<EOL>r2dat.append("<STR_LIT>".format(sname, r2, "<STR_LIT:+>", "<STR_LIT:B>"*(len(r2))))<EOL><DEDENT>r1file.write("<STR_LIT:\n>".join(r1dat))<EOL>r2file.write("<STR_LIT:\n>".join(r2dat))<EOL>r1file.close()<EOL>r2file.close()<EOL>merged_file = tempfile.NamedTemporaryFile(mode='<STR_LIT:wb>',<EOL>dir=data.dirs.edits,<EOL>suffix="<STR_LIT>").name<EOL>clust = []<EOL>merge_pairs(data, [(r1file.name, r2file.name)], merged_file, <NUM_LIT:0>, <NUM_LIT:1>)<EOL>with open(merged_file) as infile:<EOL><INDENT>quarts = itertools.izip(*[iter(infile)]*<NUM_LIT:4>)<EOL>while <NUM_LIT:1>:<EOL><INDENT>try:<EOL><INDENT>sname, seq, _, _ = quarts.next()<EOL>if not "<STR_LIT>" in sname.rsplit("<STR_LIT:;>", <NUM_LIT:1>)[<NUM_LIT:1>]:<EOL><INDENT>try:<EOL><INDENT>R1, R2 = seq.split("<STR_LIT>")<EOL>seq = R1 + "<STR_LIT>" + revcomp(R2)<EOL><DEDENT>except ValueError as inst:<EOL><INDENT>LOGGER.error("<STR_LIT>".format(sname, seq))<EOL>raise<EOL><DEDENT><DEDENT><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>sname = "<STR_LIT:>>" + sname[<NUM_LIT:1>:]<EOL>clust.extend([sname.strip(), seq.strip()])<EOL><DEDENT><DEDENT><DEDENT>except:<EOL><INDENT>LOGGER.info("<STR_LIT>")<EOL>raise<EOL><DEDENT>finally:<EOL><INDENT>for i in [r1file.name, r2file.name, merged_file]:<EOL><INDENT>if os.path.exists(i):<EOL><INDENT>log_level = logging.getLevelName(LOGGER.getEffectiveLevel())<EOL>os.remove(i)<EOL><DEDENT><DEDENT><DEDENT>return clust<EOL>

This is for pysam post-flight merging. The input is a cluster
for an individual locus. We have to split the clusters, write
R1 and R2 to files then call merge_pairs(). This is not ideal,
it's slow, but it works. This is the absolute worst way to do this,
it bounces all the files for each locus off the disk. I/O _hog_.


amb = amb.upper()<EOL>return DUCT.get(amb)<EOL>

This is copied from pyrad.alignable, and is referenced in
    several of the loci2*.py conversion modules. It duplicates some
    of the effort of unhetero(), but i guess it's fine for now. Probably
    could merge these two functions if you wanted to.


<EOL>seq = seq.replace("<STR_LIT:A>", '<STR_LIT:u>').replace('<STR_LIT:T>', '<STR_LIT:v>').replace('<STR_LIT:C>', '<STR_LIT:p>').replace('<STR_LIT>', '<STR_LIT:z>').replace('<STR_LIT:u>', '<STR_LIT:T>').replace('<STR_LIT:v>', '<STR_LIT:A>').replace('<STR_LIT:p>', '<STR_LIT>').replace('<STR_LIT:z>', '<STR_LIT:C>')<EOL>seq = seq.replace('<STR_LIT:R>', '<STR_LIT:u>').replace('<STR_LIT>', '<STR_LIT:v>').replace('<STR_LIT:Y>', '<STR_LIT:b>').replace('<STR_LIT:M>', '<STR_LIT:o>').replace('<STR_LIT:u>', '<STR_LIT:Y>').replace('<STR_LIT:v>', '<STR_LIT:M>').replace('<STR_LIT:b>', '<STR_LIT:R>').replace('<STR_LIT:o>', '<STR_LIT>')<EOL>seq = seq.replace('<STR_LIT:r>', '<STR_LIT:u>').replace('<STR_LIT:k>', '<STR_LIT:v>').replace('<STR_LIT:y>', '<STR_LIT:b>').replace('<STR_LIT:m>', '<STR_LIT:o>').replace('<STR_LIT:u>', '<STR_LIT:y>').replace('<STR_LIT:v>', '<STR_LIT:m>').replace('<STR_LIT:b>', '<STR_LIT:r>').replace('<STR_LIT:o>', '<STR_LIT:k>')<EOL>return seq<EOL>

returns complement of sequence including ambiguity characters,
    and saves lower case info for multiple hetero sequences


def fastq_touchup_for_vsearch_merge(read, outfile, reverse=False):

counts = <NUM_LIT:0><EOL>with open(outfile, '<STR_LIT:w>') as out:<EOL><INDENT>if read.endswith("<STR_LIT>"):<EOL><INDENT>fr1 = gzip.open(read, '<STR_LIT:rb>')<EOL><DEDENT>else:<EOL><INDENT>fr1 = open(read, '<STR_LIT:rb>')<EOL><DEDENT>quarts = itertools.izip(*[iter(fr1)]*<NUM_LIT:4>)<EOL>writing = []<EOL>while <NUM_LIT:1>:<EOL><INDENT>try:<EOL><INDENT>lines = quarts.next()<EOL><DEDENT>except StopIteration:<EOL><INDENT>break<EOL><DEDENT>if reverse:<EOL><INDENT>seq = lines[<NUM_LIT:1>].strip()[::-<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>seq = lines[<NUM_LIT:1>].strip()<EOL><DEDENT>writing.append("<STR_LIT>".join([<EOL>lines[<NUM_LIT:0>],<EOL>seq+"<STR_LIT:\n>",<EOL>lines[<NUM_LIT:2>],<EOL>"<STR_LIT:B>"*len(seq)<EOL>]))<EOL>counts += <NUM_LIT:1><EOL>if not counts % <NUM_LIT:1000>:<EOL><INDENT>out.write("<STR_LIT:\n>".join(writing)+"<STR_LIT:\n>")<EOL>writing = []<EOL><DEDENT><DEDENT>if writing:<EOL><INDENT>out.write("<STR_LIT:\n>".join(writing))<EOL><DEDENT><DEDENT>out.close()<EOL>fr1.close()<EOL>

option to change orientation of reads and sets Qscore to B


amb = amb.upper()<EOL>return AMBIGS.get(amb)<EOL>


def get_haploid_lik(errors, bfreqs, ustacks, counts):

hetero = <NUM_LIT:0.><EOL>if errors <= <NUM_LIT:0.>:<EOL><INDENT>score = np.exp(<NUM_LIT:100>)<EOL><DEDENT>else:<EOL><INDENT>lik1 = ((<NUM_LIT:1.>-hetero)*likelihood1(errors, bfreqs, ustacks)) <EOL>liks = lik1<EOL>logliks = np.log(liks[liks > <NUM_LIT:0>])*counts[liks > <NUM_LIT:0>]<EOL>score = -logliks.sum()<EOL><DEDENT>return score<EOL>


<EOL>sample, _, _, nhidepth, maxlen = recal_hidepth(data, sample)<EOL>clusters = gzip.open(sample.files.clusters)<EOL>pairdealer = itertools.izip(*[iter(clusters)]*<NUM_LIT:2>)<EOL>dims = (nhidepth, maxlen, <NUM_LIT:4>)<EOL>stacked = np.zeros(dims, dtype=np.uint64)<EOL>cutlens = [None, None]<EOL>try:<EOL><INDENT>cutlens[<NUM_LIT:0>] = len(data.paramsdict["<STR_LIT>"][<NUM_LIT:0>])<EOL>cutlens[<NUM_LIT:1>] = maxlen - len(data.paramsdict["<STR_LIT>"][<NUM_LIT:1>])<EOL><DEDENT>except TypeError:<EOL><INDENT>pass<EOL><DEDENT>nclust = <NUM_LIT:0><EOL>done = <NUM_LIT:0><EOL>while not done:<EOL><INDENT>try:<EOL><INDENT>done, chunk = clustdealer(pairdealer, <NUM_LIT:1>)<EOL><DEDENT>except IndexError:<EOL><INDENT>raise IPyradError("<STR_LIT>", chunk)<EOL><DEDENT>if chunk:<EOL><INDENT>piece = chunk[<NUM_LIT:0>].strip().split("<STR_LIT:\n>")<EOL>names = piece[<NUM_LIT:0>::<NUM_LIT:2>]<EOL>seqs = piece[<NUM_LIT:1>::<NUM_LIT:2>]<EOL>reps = [int(sname.split("<STR_LIT:;>")[-<NUM_LIT:2>][<NUM_LIT:5>:]) for sname in names]<EOL>sseqs = [list(seq) for seq in seqs]<EOL>arrayed = np.concatenate(<EOL>[[seq]*rep for seq, rep in zip(sseqs, reps)])<EOL>if arrayed.shape[<NUM_LIT:0>] >= data.paramsdict["<STR_LIT>"]:<EOL><INDENT>arrayed = arrayed[:<NUM_LIT>, cutlens[<NUM_LIT:0>]:cutlens[<NUM_LIT:1>]]<EOL>arrayed = arrayed[:, ~np.any(arrayed == "<STR_LIT:n>", axis=<NUM_LIT:0>)]<EOL>arrayed[arrayed == "<STR_LIT:->"] = "<STR_LIT:N>"<EOL>arrayed = arrayed[:, ~np.all(arrayed == "<STR_LIT:N>", axis=<NUM_LIT:0>)]<EOL>catg = np.array([np.sum(arrayed == i, axis=<NUM_LIT:0>) for i in list("<STR_LIT>")], <EOL>dtype=np.uint64).T<EOL>stacked[nclust, :catg.shape[<NUM_LIT:0>], :] = catg<EOL>nclust += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>newstack = stacked[stacked.sum(axis=<NUM_LIT:2>) > <NUM_LIT:0>]<EOL>assert not np.any(newstack.sum(axis=<NUM_LIT:1>) == <NUM_LIT:0>), "<STR_LIT>"<EOL>clusters.close()<EOL>return newstack<EOL>


def nget_diploid_lik(pstart, bfreqs, ustacks, counts):

hetero, errors = pstart<EOL>if (hetero <= <NUM_LIT:0.>) or (errors <= <NUM_LIT:0.>):<EOL><INDENT>score = np.exp(<NUM_LIT:100>)<EOL><DEDENT>else:<EOL><INDENT>lik1 = (<NUM_LIT:1.>-hetero) * likelihood1(errors, bfreqs, ustacks)<EOL>lik2 = (hetero) * nlikelihood2(errors, bfreqs, ustacks)<EOL>liks = lik1 + lik2<EOL>logliks = np.log(liks[liks > <NUM_LIT:0>]) * counts[liks > <NUM_LIT:0>]<EOL>score = -logliks.sum()<EOL><DEDENT>return score<EOL>


<EOL>tots = np.empty((ustacks.shape[<NUM_LIT:0>], <NUM_LIT:1>))<EOL>twos = np.empty((ustacks.shape[<NUM_LIT:0>], <NUM_LIT:6>))<EOL>thrs = np.empty((ustacks.shape[<NUM_LIT:0>], <NUM_LIT:6>, <NUM_LIT:2>))<EOL>for idx in xrange(ustacks.shape[<NUM_LIT:0>]):<EOL><INDENT>ust = ustacks[idx]<EOL>tot = ust.sum()<EOL>tots[idx] = tot<EOL>i = <NUM_LIT:0><EOL>for jdx in xrange(<NUM_LIT:4>):<EOL><INDENT>for kdx in xrange(<NUM_LIT:4>):<EOL><INDENT>if jdx < kdx:<EOL><INDENT>twos[idx][i] = tot - ust[jdx] - ust[kdx]<EOL>thrs[idx][i] = ust[jdx], ust[jdx] + ust[kdx]<EOL>i += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>return tots, twos, thrs<EOL>

JIT'd function builds large array that can be used to calc binom pmf


<EOL>majrdepth = data.paramsdict["<STR_LIT>"]<EOL>statdepth = data.paramsdict["<STR_LIT>"]    <EOL>maxlen = data._hackersonly["<STR_LIT>"]<EOL>if not hasattr(sample.stats_dfs.s3, "<STR_LIT>"):<EOL><INDENT>sample.stats_dfs.s3["<STR_LIT>"] = data.paramsdict["<STR_LIT>"]<EOL><DEDENT>if <NUM_LIT:1>: <EOL><INDENT>LOGGER.info("<STR_LIT>")<EOL>maxlens, depths = get_quick_depths(data, sample)<EOL>hidepths = depths >= majrdepth<EOL>stathidepths = depths >= statdepth<EOL>keepmj = depths[hidepths]<EOL>keepst = depths[stathidepths]<EOL>try:<EOL><INDENT>statlens = maxlens[stathidepths]<EOL>statlen = int(statlens.mean() + (<NUM_LIT>*statlens.std()))<EOL><DEDENT>except:<EOL><INDENT>raise IPyradError("<STR_LIT>")<EOL><DEDENT>LOGGER.info("<STR_LIT>", maxlens.shape, maxlens.mean(), maxlens.std())<EOL>maxlens = maxlens[hidepths]<EOL>maxlen = int(maxlens.mean() + (<NUM_LIT>*maxlens.std()))<EOL>sample.stats["<STR_LIT>"] = keepmj.shape[<NUM_LIT:0>]<EOL>sample.stats_dfs.s3["<STR_LIT>"] = keepmj.shape[<NUM_LIT:0>]        <EOL><DEDENT>return sample, keepmj.shape[<NUM_LIT:0>], maxlen, keepst.shape[<NUM_LIT:0>], statlen<EOL>

if mindepth setting were changed then 'clusters_hidepth' needs to be
recalculated. Check and recalculate if necessary.


<EOL>data.stats_dfs.s4 = data._build_stat("<STR_LIT>")<EOL>data.stats_files.s4 = os.path.join(data.dirs.clusts, <EOL>"<STR_LIT>")<EOL>with io.open(data.stats_files.s4, '<STR_LIT:w>') as outfile:<EOL><INDENT>data.stats_dfs.s4.to_string(outfile)<EOL><DEDENT>fails = data.stats[data.stats["<STR_LIT:state>"] == <NUM_LIT:3>].index.values<EOL>if fails:<EOL><INDENT>msg = """<STR_LIT>""".format(fails)<EOL>print(msg)<EOL><DEDENT>


<EOL>mapf = os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>")<EOL>umapf = os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>")<EOL>cmd1 = [ipyrad.bins.samtools, "<STR_LIT>", umapf]<EOL>proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>result1 = proc1.communicate()[<NUM_LIT:0>]<EOL>cmd2 = [ipyrad.bins.samtools, "<STR_LIT>", mapf]<EOL>proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>result2 = proc2.communicate()[<NUM_LIT:0>]<EOL>if "<STR_LIT>" in data.paramsdict["<STR_LIT>"]:<EOL><INDENT>sample.stats["<STR_LIT>"] = int(result1.split()[<NUM_LIT:0>]) / <NUM_LIT:2><EOL>sample.stats["<STR_LIT>"] = int(result2.split()[<NUM_LIT:0>]) / <NUM_LIT:2><EOL><DEDENT>else:<EOL><INDENT>sample.stats["<STR_LIT>"] = int(result1.split()[<NUM_LIT:0>])<EOL>sample.stats["<STR_LIT>"] = int(result2.split()[<NUM_LIT:0>])<EOL><DEDENT>sample_cleanup(data, sample)<EOL>

Get the number of mapped and unmapped reads for a sample
and update sample.stats


def mapreads(data, sample, nthreads, force):

LOGGER.info("<STR_LIT>", sample.name, nthreads)<EOL>derepfile = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>sample.files.dereps = [derepfile]<EOL>mumapfile = sample.files.unmapped_reads<EOL>umap1file = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>umap2file = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")        <EOL>if "<STR_LIT>" in data.paramsdict["<STR_LIT>"]:<EOL><INDENT>sample.files.split1 = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>sample.files.split2 = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>sample.files.dereps = [sample.files.split1, sample.files.split2]<EOL>split_merged_reads(sample.files.dereps, derepfile)<EOL><DEDENT>if "<STR_LIT>" in data._hackersonly["<STR_LIT>"]:<EOL><INDENT>cmd1 = [ipyrad.bins.smalt, "<STR_LIT>", <EOL>"<STR_LIT>", "<STR_LIT>", <EOL>"<STR_LIT>", str(max(<NUM_LIT:1>, nthreads)),<EOL>"<STR_LIT>", str(data.paramsdict['<STR_LIT>']), <EOL>"<STR_LIT>", os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>"),<EOL>"<STR_LIT>",<EOL>data.paramsdict['<STR_LIT>']<EOL>] + sample.files.dereps<EOL>cmd1_stdout = sps.PIPE<EOL>cmd1_stderr = sps.STDOUT<EOL><DEDENT>else:<EOL><INDENT>cmd1 = [ipyrad.bins.bwa, "<STR_LIT>",<EOL>"<STR_LIT>", str(max(<NUM_LIT:1>, nthreads)),<EOL>"<STR_LIT>",<EOL>data.paramsdict['<STR_LIT>']<EOL>] + sample.files.dereps<EOL>try:<EOL><INDENT>bwa_args = data._hackersonly["<STR_LIT>"].split()<EOL>bwa_args.reverse()<EOL>for arg in bwa_args:<EOL><INDENT>cmd1.insert(<NUM_LIT:2>, arg)<EOL><DEDENT><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT>cmd1_stdout = open(os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>"), '<STR_LIT:w>')<EOL>cmd1_stderr = None<EOL><DEDENT>cmd2 = [ipyrad.bins.samtools, "<STR_LIT>", <EOL>"<STR_LIT>", <EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>"), <EOL>os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>")]<EOL>cmd3 = [ipyrad.bins.samtools, "<STR_LIT>", <EOL>"<STR_LIT>", os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>"),<EOL>"<STR_LIT>", "<STR_LIT>", <EOL>"<STR_LIT>", sample.files.mapped_reads]<EOL>cmd4 = [ipyrad.bins.samtools, "<STR_LIT:index>", sample.files.mapped_reads]<EOL>cmd5 = [ipyrad.bins.samtools, "<STR_LIT>", "<STR_LIT>",<EOL>os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>")]<EOL>if '<STR_LIT>' in data.paramsdict["<STR_LIT>"]:<EOL><INDENT>if "<STR_LIT>" in data._hackersonly["<STR_LIT>"]:<EOL><INDENT>cmd1.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL>cmd1.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT>cmd2.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL>cmd2.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL>cmd5.insert(<NUM_LIT:2>, umap1file)<EOL>cmd5.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL>cmd5.insert(<NUM_LIT:2>, umap2file)<EOL>cmd5.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>cmd5.insert(<NUM_LIT:2>, mumapfile)<EOL>cmd5.insert(<NUM_LIT:2>, "<STR_LIT>")<EOL><DEDENT>LOGGER.debug("<STR_LIT:U+0020>".join(cmd1))<EOL>proc1 = sps.Popen(cmd1, stderr=cmd1_stderr, stdout=cmd1_stdout)<EOL>try:<EOL><INDENT>error1 = proc1.communicate()[<NUM_LIT:0>]<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>proc1.kill()<EOL><DEDENT>if proc1.returncode:<EOL><INDENT>raise IPyradWarningExit(error1)<EOL><DEDENT>LOGGER.debug("<STR_LIT:U+0020>".join(cmd2))<EOL>proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>LOGGER.debug("<STR_LIT:U+0020>".join(cmd3))<EOL>proc3 = sps.Popen(cmd3, stderr=sps.STDOUT, stdout=sps.PIPE, stdin=proc2.stdout)<EOL>error3 = proc3.communicate()[<NUM_LIT:0>]<EOL>if proc3.returncode:<EOL><INDENT>raise IPyradWarningExit(error3)<EOL><DEDENT>proc2.stdout.close()<EOL>LOGGER.debug("<STR_LIT:U+0020>".join(cmd4))<EOL>proc4 = sps.Popen(cmd4, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>error4 = proc4.communicate()[<NUM_LIT:0>]<EOL>if proc4.returncode:<EOL><INDENT>raise IPyradWarningExit(error4)<EOL><DEDENT>LOGGER.debug("<STR_LIT:U+0020>".join(cmd5))<EOL>proc5 = sps.Popen(cmd5, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>error5 = proc5.communicate()[<NUM_LIT:0>]<EOL>if proc5.returncode:<EOL><INDENT>raise IPyradWarningExit(error5)<EOL><DEDENT>if '<STR_LIT>' in data.paramsdict["<STR_LIT>"]:<EOL><INDENT>LOGGER.info("<STR_LIT>".format(umap1file, umap2file))<EOL>merge_pairs_after_refmapping(data, [(umap1file, umap2file)], mumapfile)<EOL><DEDENT>

Attempt to map reads to reference sequence. This reads in the fasta files
(samples.files.edits), and maps each read to the reference. Unmapped reads
are dropped right back in the de novo pipeline. Reads that map successfully
are processed and pushed downstream and joined with the rest of the data
post muscle_align.

Mapped reads end up in a sam file.


def fetch_cluster_se(data, samfile, chrom, rstart, rend):

<EOL>overlap_buffer = data._hackersonly["<STR_LIT>"]<EOL>rstart_buff = rstart + overlap_buffer<EOL>rend_buff = rend - overlap_buffer<EOL>if rstart_buff > rend_buff:<EOL><INDENT>tmp = rstart_buff<EOL>rstart_buff = rend_buff<EOL>rend_buff = tmp<EOL><DEDENT>if rstart_buff == rend_buff:<EOL><INDENT>rend_buff += <NUM_LIT:1><EOL><DEDENT>rdict = {}<EOL>clust = []<EOL>iterreg = []<EOL>iterreg = samfile.fetch(chrom, rstart_buff, rend_buff)<EOL>for read in iterreg:<EOL><INDENT>if read.qname not in rdict:<EOL><INDENT>rdict[read.qname] = read<EOL><DEDENT><DEDENT>sfunc = lambda x: int(x.split("<STR_LIT>")[<NUM_LIT:1>].split("<STR_LIT:;>")[<NUM_LIT:0>])<EOL>rkeys = sorted(rdict.keys(), key=sfunc, reverse=True)<EOL>try:<EOL><INDENT>read1 = rdict[rkeys[<NUM_LIT:0>]]<EOL><DEDENT>except ValueError:<EOL><INDENT>LOGGER.error("<STR_LIT>".format(rkeys[<NUM_LIT:0>], rdict))<EOL>return "<STR_LIT>"<EOL><DEDENT>poss = read1.get_reference_positions(full_length=True)<EOL>seed_r1start = min(poss)<EOL>seed_r1end = max(poss)<EOL>if read1.is_reverse:<EOL><INDENT>seq = revcomp(read1.seq)<EOL><DEDENT>else:<EOL><INDENT>seq = read1.seq<EOL><DEDENT>size = sfunc(rkeys[<NUM_LIT:0>])<EOL>clust.append("<STR_LIT>".format(chrom, seed_r1start, seed_r1end, size, seq))<EOL>if len(rkeys) > <NUM_LIT:1>:<EOL><INDENT>for key in rkeys[<NUM_LIT:1>:]:<EOL><INDENT>skip = False<EOL>try:<EOL><INDENT>read1 = rdict[key]<EOL><DEDENT>except ValueError:<EOL><INDENT>read1 = rdict[key][<NUM_LIT:0>]<EOL>skip = True<EOL><DEDENT>if not skip:<EOL><INDENT>poss = read1.get_reference_positions(full_length=True)<EOL>minpos = min(poss)<EOL>maxpos = max(poss)<EOL>if read1.is_reverse:<EOL><INDENT>seq = revcomp(read1.seq)<EOL><DEDENT>else:<EOL><INDENT>seq = read1.seq<EOL><DEDENT>size = sfunc(key)<EOL>clust.append("<STR_LIT>".format(chrom, minpos, maxpos, size, seq))<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>return clust<EOL>

Builds a single end cluster from the refmapped data.


LOGGER.debug("<STR_LIT>".format(fasta[<NUM_LIT:0>]))<EOL>if "<STR_LIT>" in fasta[<NUM_LIT:1>]:<EOL><INDENT>r1_len = len(fasta[<NUM_LIT:1>].split("<STR_LIT:\n>")[<NUM_LIT:1>].split("<STR_LIT>")[<NUM_LIT:0>])<EOL>r2_len = len(fasta[<NUM_LIT:1>].split("<STR_LIT:\n>")[<NUM_LIT:1>].split("<STR_LIT>")[<NUM_LIT:1>])<EOL>new_seq = fasta[<NUM_LIT:0>].split("<STR_LIT:\n>")[<NUM_LIT:1>][:r1_len]+("<STR_LIT>")+ revcomp(fasta[<NUM_LIT:0>].split("<STR_LIT:\n>")[<NUM_LIT:1>][-r2_len:])<EOL>fasta[<NUM_LIT:0>] = fasta[<NUM_LIT:0>].split("<STR_LIT:\n>")[<NUM_LIT:0>]+"<STR_LIT:\n>"+new_seq<EOL><DEDENT>LOGGER.debug("<STR_LIT>".format(fasta[<NUM_LIT:0>]))<EOL>return fasta<EOL>

If doing PE and R1/R2 don't overlap then the reference sequence
will be quite long and will cause indel hell during the
alignment stage. Here trim the reference sequence to the length
of the merged reads. Input is a list of alternating locus labels
and sequence data. The first locus label is the reference
sequence label and the first seq is the reference seq. Returns
the same list except with the reference sequence trimmed to
the length of the rad tags


def fetch_cluster_pairs(data, samfile, chrom, rstart, rend):

<EOL>rdict = {}<EOL>clust = []<EOL>iterreg = samfile.fetch(chrom, rstart, rend)<EOL>for read in iterreg:<EOL><INDENT>if read.qname not in rdict:<EOL><INDENT>rdict[read.qname] = [read]<EOL><DEDENT>else:<EOL><INDENT>rdict[read.qname].append(read)<EOL><DEDENT><DEDENT>sfunc = lambda x: int(x.split("<STR_LIT>")[<NUM_LIT:1>].split("<STR_LIT:;>")[<NUM_LIT:0>])<EOL>rkeys = sorted(rdict.keys(), key=sfunc, reverse=True)<EOL>try:<EOL><INDENT>read1, read2 = rdict[rkeys[<NUM_LIT:0>]]<EOL><DEDENT>except ValueError:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>poss = read1.get_reference_positions() + read2.get_reference_positions()<EOL>seed_r1start = min(poss)<EOL>seed_r2end = max(poss)<EOL>reads_overlap = False<EOL>if read1.is_reverse:<EOL><INDENT>if read2.aend > read1.get_blocks()[<NUM_LIT:0>][<NUM_LIT:0>]:<EOL><INDENT>reads_overlap = True<EOL>seq = read2.seq + "<STR_LIT>" + revcomp(read1.seq)<EOL><DEDENT>else:<EOL><INDENT>seq = read2.seq + "<STR_LIT>" + read1.seq<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if read1.aend > read2.get_blocks()[<NUM_LIT:0>][<NUM_LIT:0>]:<EOL><INDENT>reads_overlap = True<EOL>seq = read1.seq + "<STR_LIT>" + revcomp(read2.seq)<EOL><DEDENT>else:<EOL><INDENT>seq = read1.seq + "<STR_LIT>" + read2.seq<EOL><DEDENT><DEDENT>size = sfunc(rkeys[<NUM_LIT:0>])<EOL>clust.append("<STR_LIT>".format(chrom, seed_r1start, seed_r2end, size, seq))<EOL>if len(rkeys) > <NUM_LIT:1>:<EOL><INDENT>for key in rkeys[<NUM_LIT:1>:]:<EOL><INDENT>skip = False<EOL>try:<EOL><INDENT>read1, read2 = rdict[key]<EOL><DEDENT>except ValueError:<EOL><INDENT>read1 = rdict[key][<NUM_LIT:0>]<EOL>read2 = read1<EOL>skip = True<EOL><DEDENT>poss = read1.get_reference_positions() + read2.get_reference_positions()<EOL>minpos = min(poss)<EOL>maxpos = max(poss)<EOL>if read1.has_tag("<STR_LIT>") or read2.has_tag("<STR_LIT>"):<EOL><INDENT>skip = True<EOL><DEDENT>if (abs(minpos - seed_r1start) < <NUM_LIT:50>) and(abs(maxpos - seed_r2end) < <NUM_LIT:50>) and(not skip):<EOL><INDENT>if read1.is_reverse:<EOL><INDENT>if read2.aend > read1.get_blocks()[<NUM_LIT:0>][<NUM_LIT:0>]:<EOL><INDENT>reads_overlap = True<EOL>seq = read2.seq + "<STR_LIT>" + revcomp(read1.seq)<EOL><DEDENT>else:<EOL><INDENT>seq = read2.seq + "<STR_LIT>" + read1.seq<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if read1.aend > read2.get_blocks()[<NUM_LIT:0>][<NUM_LIT:0>]:<EOL><INDENT>reads_overlap = True<EOL>seq = read1.seq + "<STR_LIT>" + revcomp(read2.seq)<EOL><DEDENT>else:<EOL><INDENT>seq = read1.seq + "<STR_LIT>" + read2.seq<EOL><DEDENT><DEDENT>size = sfunc(key)<EOL>clust.append("<STR_LIT>".format(chrom, minpos, maxpos, size, seq))<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>if reads_overlap:<EOL><INDENT>if data._hackersonly["<STR_LIT>"]:<EOL><INDENT>clust = merge_after_pysam(data, clust)<EOL><DEDENT><DEDENT>return clust<EOL>

Builds a paired cluster from the refmapped data.


<EOL>sample.files.unmapped_reads = os.path.join(data.dirs.edits, <EOL>"<STR_LIT>".format(sample.name))<EOL>sample.files.mapped_reads = os.path.join(data.dirs.refmapping,<EOL>"<STR_LIT>".format(sample.name))<EOL>


<EOL>regions = bedtools_merge(data, sample).strip().split("<STR_LIT:\n>")<EOL>nregions = len(regions)<EOL>chunksize = (nregions / <NUM_LIT:10>) + (nregions % <NUM_LIT:10>)<EOL>LOGGER.debug("<STR_LIT>".format(nregions, chunksize))<EOL>idx = <NUM_LIT:0><EOL>tmpfile = os.path.join(data.tmpdir, sample.name+"<STR_LIT>")<EOL>for i in range(<NUM_LIT:11>):<EOL><INDENT>if os.path.exists(tmpfile.format(i)):<EOL><INDENT>os.remove(tmpfile.format(i))<EOL><DEDENT><DEDENT>fopen = open<EOL>if data.paramsdict["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>tmpfile = os.path.join(data.dirs.clusts, sample.name+"<STR_LIT>")<EOL>fopen = gzip.open<EOL><DEDENT>samfile = pysam.AlignmentFile(sample.files.mapped_reads, '<STR_LIT:rb>')<EOL>clusts = []<EOL>nclusts = <NUM_LIT:0><EOL>for region in regions:<EOL><INDENT>chrom, pos1, pos2 = region.split()<EOL>try:<EOL><INDENT>if "<STR_LIT>" in data.paramsdict["<STR_LIT>"]:<EOL><INDENT>clust = fetch_cluster_pairs(data, samfile, chrom, int(pos1), int(pos2))<EOL><DEDENT>else:<EOL><INDENT>clust = fetch_cluster_se(data, samfile, chrom, int(pos1), int(pos2))<EOL><DEDENT><DEDENT>except IndexError as inst:<EOL><INDENT>LOGGER.error("<STR_LIT>".format(chrom, pos1, pos2))<EOL>continue<EOL><DEDENT>if clust:<EOL><INDENT>clusts.append("<STR_LIT:\n>".join(clust))<EOL>nclusts += <NUM_LIT:1><EOL>if nclusts == chunksize:<EOL><INDENT>tmphandle = tmpfile.format(idx)<EOL>with fopen(tmphandle, '<STR_LIT:a>') as tmp:<EOL><INDENT>tmp.write("<STR_LIT>".join(clusts)+"<STR_LIT>")<EOL><DEDENT>idx += <NUM_LIT:1><EOL>nclusts = <NUM_LIT:0><EOL>clusts = []<EOL><DEDENT><DEDENT><DEDENT>if clusts:<EOL><INDENT>with fopen(tmpfile.format(idx), '<STR_LIT:a>') as tmp:<EOL><INDENT>tmp.write("<STR_LIT>".join(clusts)+"<STR_LIT>")<EOL><DEDENT>clusts = []<EOL><DEDENT>if not data.paramsdict["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>chunkfiles = glob.glob(os.path.join(data.tmpdir, sample.name+"<STR_LIT>"))<EOL>LOGGER.info("<STR_LIT>", chunkfiles)<EOL><DEDENT>samfile.close()<EOL>

1. Run bedtools to get all overlapping regions
2. Parse out reads from regions using pysam and dump into chunk files.
   We measure it out to create 10 chunk files per sample.
3. If we really wanted to speed this up, though it is pretty fast already,
   we could parallelize it since we can easily break the regions into
   a list of chunks.


<EOL>refseq_file = data.paramsdict['<STR_LIT>']<EOL>index_files = []<EOL>if "<STR_LIT>" in data._hackersonly["<STR_LIT>"]:<EOL><INDENT>index_files.extend(["<STR_LIT>", "<STR_LIT>"])<EOL><DEDENT>else:<EOL><INDENT>index_files.extend(["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"])<EOL><DEDENT>index_files.extend(["<STR_LIT>"])<EOL>if not force:<EOL><INDENT>if all([os.path.isfile(refseq_file+i) for i in index_files]):<EOL><INDENT>return<EOL><DEDENT><DEDENT>if "<STR_LIT>" in data._hackersonly["<STR_LIT>"]:<EOL><INDENT>cmd1 = [ipyrad.bins.smalt, "<STR_LIT:index>", <EOL>"<STR_LIT>", str(data._hackersonly["<STR_LIT>"]), <EOL>refseq_file, <EOL>refseq_file]<EOL><DEDENT>else:<EOL><INDENT>cmd1 = [ipyrad.bins.bwa, "<STR_LIT:index>", refseq_file]<EOL><DEDENT>LOGGER.info("<STR_LIT:U+0020>".join(cmd1))<EOL>proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>error1 = proc1.communicate()[<NUM_LIT:0>]<EOL>cmd2 = [ipyrad.bins.samtools, "<STR_LIT>", refseq_file]<EOL>LOGGER.info("<STR_LIT:U+0020>".join(cmd2))<EOL>proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)<EOL>error2 = proc2.communicate()[<NUM_LIT:0>]<EOL>if proc1.returncode:<EOL><INDENT>raise IPyradWarningExit(error1)<EOL><DEDENT>if error2:<EOL><INDENT>if "<STR_LIT>" in error2:<EOL><INDENT>raise IPyradWarningExit(NO_ZIP_BINS.format(refseq_file))<EOL><DEDENT>else:<EOL><INDENT>raise IPyradWarningExit(error2)<EOL><DEDENT><DEDENT>

Index the reference sequence, unless it already exists. Also make a mapping
of scaffolds to index numbers for later user in steps 5-6.


umap1file = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>umap2file = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>unmapped = os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>")<EOL>samplesam = os.path.join(data.dirs.refmapping, sample.name+"<STR_LIT>")<EOL>split1 = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>split2 = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>refmap_derep = os.path.join(data.dirs.edits, sample.name+"<STR_LIT>")<EOL>for f in [umap1file, umap2file, unmapped, samplesam, split1, split2, refmap_derep]:<EOL><INDENT>try:<EOL><INDENT>os.remove(f)<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>


<EOL>panel = Panel(ttree)          <EOL>if not kwargs.get("<STR_LIT:width>"):<EOL><INDENT>panel.kwargs["<STR_LIT:width>"] = min(<NUM_LIT:1000>, <NUM_LIT>*len(panel.tree))<EOL><DEDENT>if not kwargs.get("<STR_LIT>"):<EOL><INDENT>panel.kwargs["<STR_LIT>"] = panel.kwargs["<STR_LIT:width>"]<EOL><DEDENT>panel.kwargs.update(kwargs)<EOL>if panel.kwargs["<STR_LIT>"]:<EOL><INDENT>nnodes = sum(<NUM_LIT:1> for i in panel.tree.traverse()) - len(panel.tree)<EOL>supps = [int(panel.tree.search_nodes(idx=j)[<NUM_LIT:0>].support)for j in range(nnodes)]<EOL>if not panel.kwargs["<STR_LIT>"]:<EOL><INDENT>panel.kwargs["<STR_LIT>"] = <NUM_LIT:20><EOL><DEDENT>sizes = [panel.kwargs["<STR_LIT>"] for j in range(nnodes)]<EOL>supps += ["<STR_LIT>"] * len(panel.tree)<EOL>sizes += [<NUM_LIT:0>] * len(panel.tree)<EOL>panel.kwargs["<STR_LIT>"] = supps<EOL>panel.kwargs["<STR_LIT>"] = sizes<EOL>panel.kwargs["<STR_LIT>"] = True<EOL>if len(panel.tree.children) > <NUM_LIT:2>:<EOL><INDENT>supps[<NUM_LIT:0>] = "<STR_LIT>"<EOL>sizes[<NUM_LIT:0>] = <NUM_LIT:0><EOL><DEDENT><DEDENT>elif panel.kwargs.get("<STR_LIT>"):<EOL><INDENT>panel.kwargs["<STR_LIT>"] = panel.kwargs["<STR_LIT>"]<EOL>panel.kwargs["<STR_LIT>"] = True<EOL><DEDENT>else:<EOL><INDENT>panel.kwargs["<STR_LIT>"] = panel.node_labels.keys() <EOL><DEDENT>if print_args:<EOL><INDENT>print(panel.kwargs)<EOL><DEDENT>canvas = toyplot.Canvas(height=panel.kwargs['<STR_LIT>'], width=panel.kwargs['<STR_LIT:width>'])<EOL>axes = canvas.cartesian(bounds=("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"))    <EOL>axes.show = panel.kwargs["<STR_LIT>"]<EOL>panel._panel_tree(axes)<EOL>if panel.kwargs["<STR_LIT>"]:<EOL><INDENT>panel._panel_tip_labels(axes)<EOL><DEDENT>return canvas, axes, panel<EOL>


<EOL>share = np.zeros((lxs.shape[<NUM_LIT:0>], lxs.shape[<NUM_LIT:0>]), dtype=np.uint64)<EOL>names = range(lxs.shape[<NUM_LIT:0>])<EOL>for row in lxs:<EOL><INDENT>for samp1, samp2 in itertools.combinations(names, <NUM_LIT:2>):<EOL><INDENT>shared = lxs[samp1, lxs[samp2] > <NUM_LIT:0>].sum()<EOL>share[samp1, samp2] = shared<EOL><DEDENT><DEDENT>share += share.T<EOL>for row in xrange(len(names)):<EOL><INDENT>share[row, row] = lxs[row].sum()<EOL><DEDENT>return share<EOL>

fill a matrix with pairwise data sharing


<EOL>lxs = np.zeros((len(snames), len(loci)), dtype=np.uint64)<EOL>for loc in xrange(len(loci)):<EOL><INDENT>for seq in loci[loc].split("<STR_LIT:\n>"):<EOL><INDENT>if "<STR_LIT>" not in seq:<EOL><INDENT>lxs[snames.index(seq.split()[<NUM_LIT:0>][:]), loc] += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>return lxs, snames<EOL>

parse loci list and return presence/absence matrix
ordered by the tips on the tree or list of names.


<EOL>share = np.zeros((lxs.shape[<NUM_LIT:0>], lxs.shape[<NUM_LIT:0>]))<EOL>names = range(lxs.shape[<NUM_LIT:0>])<EOL>for row in lxs:<EOL><INDENT>for samp1, samp2 in itertools.combinations(names, <NUM_LIT:2>):<EOL><INDENT>shared = lxs[samp1, lxs[samp2] > <NUM_LIT:0>].sum()<EOL>share[samp1, samp2] = shared<EOL><DEDENT><DEDENT>for row in xrange(len(names)):<EOL><INDENT>share[row, row] = lxs[row].sum()<EOL><DEDENT>return share<EOL>


<EOL>iparg = "<STR_LIT>"<EOL>if "<STR_LIT>" in data._ipcluster["<STR_LIT>"]:<EOL><INDENT>iparg = "<STR_LIT>"<EOL><DEDENT>standard = """<STR_LIT>""".format(data._ipcluster["<STR_LIT>"], <EOL>data._ipcluster["<STR_LIT>"], <EOL>data._ipcluster["<STR_LIT>"],<EOL>data._ipcluster["<STR_LIT>"],<EOL>iparg)<EOL>try: <EOL><INDENT>LOGGER.info(shlex.split(standard))<EOL>subprocess.check_call(shlex.split(standard), <EOL>stderr=subprocess.STDOUT,<EOL>stdout=subprocess.PIPE)<EOL><DEDENT>except subprocess.CalledProcessError as inst:<EOL><INDENT>LOGGER.debug("<STR_LIT>")<EOL>raise<EOL><DEDENT>except Exception as inst:<EOL><INDENT>sys.exit("<STR_LIT>".format(inst))<EOL><DEDENT>


try: <EOL><INDENT>name = pinfo[str(param)][<NUM_LIT:0>].strip().split("<STR_LIT:U+0020>")[<NUM_LIT:1>]<EOL><DEDENT>except (KeyError, ValueError) as err:<EOL><INDENT>print("<STR_LIT>".format(param), err)<EOL>raise<EOL><DEDENT>return name<EOL>

Get the param name from the dict index value.


<EOL>if short:<EOL><INDENT>desc = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>desc = <NUM_LIT:0><EOL><DEDENT>try: <EOL><INDENT>description = pinfo[str(param)][desc]<EOL><DEDENT>except (KeyError, ValueError) as err:<EOL><INDENT>print("<STR_LIT>".format(param), err)<EOL>raise<EOL><DEDENT>return description<EOL>

Returns detailed information for the numbered parameter.
        Further information is available in the tutorial.
        Unlike params() this function doesn't deal well with *
        It only takes one parameter at a time and returns the desc


<EOL>returndict = OrderedDict([<EOL>("<STR_LIT:name>", self.name),<EOL>("<STR_LIT>", self.barcode),<EOL>("<STR_LIT>", self.files),<EOL>("<STR_LIT>", {<EOL>"<STR_LIT>": self.stats_dfs.s1.to_dict(),<EOL>"<STR_LIT>": self.stats_dfs.s2.to_dict(),                <EOL>"<STR_LIT>": self.stats_dfs.s3.to_dict(),<EOL>"<STR_LIT>": self.stats_dfs.s4.to_dict(),<EOL>"<STR_LIT>": self.stats_dfs.s5.to_dict(),<EOL>}),<EOL>("<STR_LIT>", self.stats.to_dict()),<EOL>("<STR_LIT>", self.depths)<EOL>])<EOL>return returndict<EOL>

Write to dict including data frames. All sample dicts
are combined in save() to dump JSON output


<EOL>parser = argparse.ArgumentParser(<EOL>formatter_class=argparse.RawDescriptionHelpFormatter,<EOL>epilog="""<STR_LIT>""")<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', action='<STR_LIT:version>', <EOL>version=str(pkg_resources.get_distribution('<STR_LIT>')))<EOL>parser.add_argument('<STR_LIT>', "<STR_LIT>", action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', "<STR_LIT>", action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', "<STR_LIT>", action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', "<STR_LIT>", action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', metavar='<STR_LIT>', dest="<STR_LIT>", type=str, <EOL>default=None, <EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', metavar='<STR_LIT>', dest="<STR_LIT>",<EOL>type=str, default=None,<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', metavar='<STR_LIT>', dest="<STR_LIT>",<EOL>type=str, default=None, nargs="<STR_LIT:*>",<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', metavar='<STR_LIT>', dest="<STR_LIT>",<EOL>default=None, nargs="<STR_LIT:*>",<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', metavar="<STR_LIT>", dest="<STR_LIT>",<EOL>type=str, default=None,<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT:-c>", metavar="<STR_LIT>", dest="<STR_LIT>",<EOL>type=int, default=<NUM_LIT:0>,<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", metavar="<STR_LIT>", dest="<STR_LIT>",<EOL>type=int, default=<NUM_LIT:2>,<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", metavar="<STR_LIT>", dest="<STR_LIT>",<EOL>type=str, nargs="<STR_LIT:?>", const="<STR_LIT:default>",<EOL>help="<STR_LIT>")<EOL>parser.add_argument("<STR_LIT>", metavar="<STR_LIT>", dest="<STR_LIT>",<EOL>type=str, nargs="<STR_LIT:*>", default=None, <EOL>help="<STR_LIT>")<EOL>if len(sys.argv) == <NUM_LIT:1>:<EOL><INDENT>parser.print_help()<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>args = parser.parse_args()<EOL>if not any(x in ["<STR_LIT>", "<STR_LIT>"] for x in vars(args).keys()):<EOL><INDENT>print("<STR_LIT>"+"<STR_LIT>")<EOL>parser.print_help()<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>return args<EOL>


import urllib2<EOL>from distutils.version import LooseVersion<EOL>header ="<STR_LIT>"+"<STR_LIT>".format(ip.__version__)+"<STR_LIT>"+"<STR_LIT>"<EOL>try:<EOL><INDENT>htmldat = urllib2.urlopen("<STR_LIT>").readlines()<EOL>curversion = next((x for x in htmldat if "<STR_LIT>" in x), None).split("<STR_LIT:>>")[<NUM_LIT:1>].split("<STR_LIT:<>")[<NUM_LIT:0>]<EOL>if LooseVersion(ip.__version__) < LooseVersion(curversion):<EOL><INDENT>msg = """<STR_LIT>""".format(curversion)<EOL>print(header + "<STR_LIT:\n>" + msg)<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>except Exception as inst:<EOL><INDENT>pass<EOL><DEDENT>

Test if there's a newer version and nag the user to upgrade.


<EOL>hosts = []<EOL>for eid in ipyclient.ids:<EOL><INDENT>engine = ipyclient[eid]<EOL>if not engine.outstanding:<EOL><INDENT>hosts.append(engine.apply(_socket.gethostname))<EOL><DEDENT><DEDENT>hosts = [i.get() for i in hosts]<EOL>result = []<EOL>for hostname in set(hosts):<EOL><INDENT>result.append("<STR_LIT>".format(spacer, hosts.count(hostname), hostname))<EOL><DEDENT>print("<STR_LIT:\n>".join(result))<EOL>

reports host and engine info for an ipyclient


_lconfig.dictConfig({<EOL>'<STR_LIT:version>': <NUM_LIT:1>,<EOL>'<STR_LIT>': False,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': "<STR_LIT>"+"<STR_LIT>"+"<STR_LIT>"+"<STR_LIT>"+"<STR_LIT>"<EOL>},<EOL>},<EOL>'<STR_LIT>': {<EOL>__name__: {<EOL>'<STR_LIT>':__loglevel__,<EOL>'<STR_LIT:class>':'<STR_LIT>',<EOL>'<STR_LIT:filename>':__debugfile__,<EOL>'<STR_LIT>':"<STR_LIT>",<EOL>'<STR_LIT>':'<STR_LIT>'<EOL>}<EOL>},<EOL>'<STR_LIT>':{<EOL>__name__: {<EOL>'<STR_LIT>': [__name__],<EOL>'<STR_LIT>': __loglevel__,<EOL>'<STR_LIT>': True<EOL>}<EOL>}<EOL>})<EOL>


if _os.path.exists(__debugflag__):<EOL><INDENT>_os.remove(__debugflag__)<EOL><DEDENT>__loglevel__ = "<STR_LIT>"<EOL>_LOGGER.info("<STR_LIT>")<EOL>_set_debug_dict(__loglevel__)<EOL>

turns off debugging by removing hidden tmp file


<EOL>if isinstance(obj, unicode):<EOL><INDENT>return obj.encode('<STR_LIT:utf-8>')<EOL><DEDENT>if isinstance(obj, list):<EOL><INDENT>return [_tup_and_byte(item) for item in obj]<EOL><DEDENT>if isinstance(obj, dict):<EOL><INDENT>if "<STR_LIT>" in obj:<EOL><INDENT>return tuple(_tup_and_byte(item) for item in obj["<STR_LIT>"])<EOL><DEDENT>else:<EOL><INDENT>return {<EOL>_tup_and_byte(key): _tup_and_byte(val) forkey, val in obj.iteritems()<EOL>}<EOL><DEDENT><DEDENT>return obj<EOL>


<EOL>datadict = OrderedDict([<EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"]),<EOL>("<STR_LIT>", dict(data.__dict__["<STR_LIT>"])),<EOL>("<STR_LIT>", data.__dict__["<STR_LIT>"])<EOL>])<EOL>


<EOL>if not imap:<EOL><INDENT>raise IPyradWarningExit(IMAP_REQUIRED)<EOL><DEDENT>if minmap:<EOL><INDENT>if minmap.keys() != imap.keys():<EOL><INDENT>raise IPyradWarningExit(KEYS_DIFFER)<EOL><DEDENT><DEDENT>if wdir:<EOL><INDENT>wdir = os.path.abspath(wdir)<EOL>if not os.path.exists(wdir):<EOL><INDENT>raise IPyradWarningExit("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>wdir = os.path.curdir<EOL><DEDENT>prog = '<STR_LIT>'<EOL>if isinstance(traits_df, pd.DataFrame):<EOL><INDENT>prog = '<STR_LIT>'<EOL><DEDENT>outfile = OPJ(wdir, "<STR_LIT>".format(name, prog))<EOL>mapfile = OPJ(wdir, "<STR_LIT>".format(name, prog))<EOL>fout = open(outfile, '<STR_LIT:w>')<EOL>fmap = open(mapfile, '<STR_LIT:w>')<EOL>with open(locifile, '<STR_LIT:r>') as infile:<EOL><INDENT>loci = infile.read().strip().split("<STR_LIT>")<EOL>nloci = len(loci)<EOL><DEDENT>samples = list(itertools.chain(*imap.values()))<EOL>nkept = <NUM_LIT:0><EOL>for iloc in xrange(nloci):<EOL><INDENT>lines = loci[iloc].split("<STR_LIT>")[<NUM_LIT:0>].split()<EOL>names = lines[::<NUM_LIT:2>]<EOL>names = ["<STR_LIT>"+i for i in names]<EOL>seqs = [list(i) for i in lines[<NUM_LIT:1>::<NUM_LIT:2>]]<EOL>seqlen = len(seqs[<NUM_LIT:0>])<EOL>skip = <NUM_LIT:0><EOL>if minmap:<EOL><INDENT>covd = {}<EOL>for group, vals in imap.items():<EOL><INDENT>covd[group] = sum(["<STR_LIT>"+i in names for i in vals])<EOL><DEDENT>if not all([covd[group] >= minmap[group] for group in minmap]):<EOL><INDENT>skip = <NUM_LIT:1><EOL><DEDENT><DEDENT>if maxloci:<EOL><INDENT>if nkept >= maxloci:<EOL><INDENT>skip = <NUM_LIT:1><EOL><DEDENT><DEDENT>if not skip:<EOL><INDENT>data = ["<STR_LIT>".format(i, "<STR_LIT>".join(k).replace("<STR_LIT:->", "<STR_LIT:N>")) for(i, k) in zip(names, seqs) if i[<NUM_LIT:1>:] in samples]<EOL>if data:<EOL><INDENT>fout.write("<STR_LIT>".format(len(data), seqlen, "<STR_LIT:\n>".join(data)))<EOL>nkept += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>fout.close()<EOL>data = ["<STR_LIT>".format(val, key) for keyin sorted(imap) for val in imap[key]]<EOL>fmap.write("<STR_LIT:\n>".join(data))<EOL>fmap.close()<EOL>write_ctl(name, imap, guidetree, nkept,<EOL>infer_sptree, infer_delimit, delimit_alg,<EOL>seed, burnin, nsample, sampfreq,<EOL>thetaprior, tauprior, traits_df, nu, kappa,<EOL>cleandata, useseqdata, usetraitdata, wdir,<EOL>finetune, verbose)<EOL>sys.stderr.write("<STR_LIT>".format(nkept, len(imap.keys()),<EOL>sum([len(i) for i in imap.values()])))<EOL>sys.stderr.write("<STR_LIT>".format(name, prog))<EOL>sys.stderr.write("<STR_LIT>".format(name, prog))<EOL>sys.stderr.write("<STR_LIT>".format(name, prog))<EOL>if isinstance(traits_df, pd.DataFrame):<EOL><INDENT>sys.stderr.write("<STR_LIT>".format(name, prog))<EOL><DEDENT>return os.path.abspath(<EOL>"<STR_LIT>".format(OPJ(wdir, name), prog))<EOL>

Converts loci file format to bpp file format, i.e., concatenated phylip-like
format, and produces imap and ctl input files for bpp.

Parameters:
-----------
name:
    A prefix name for output files that will be produced
locifile:
    A .loci file produced by ipyrad.
imap:
    A Python dictionary with 'species' names as keys, and lists of sample
    names for the values. Any sample that is not included in the imap
    dictionary will be filtered out of the data when converting the .loci
    file into the bpp formatted sequence file. Each species in the imap
    dictionary must also be present in the input 'guidetree'.
guidetree:
    A newick string species tree hypothesis [e.g., (((a,b),(c,d)),e);]
    All species in the imap dictionary must also be present in the guidetree

Optional parameters:
--------------------
infer_sptree:
    Default=0, only infer parameters on a fixed species tree. If 1, then the
    input tree is treated as a guidetree and tree search is employed to find
    the best tree. The results will include support values for the inferred
    topology.
infer_delimit:
    Default=0, no delimitation. If 1 then splits in the tree that separate
    'species' will be collapsed to test whether fewer species are a better
    fit to the data than the number in the input guidetree.
delimit_alg:
    Species delimitation algorithm. This is a tuple. The first value
    is the algorithm (0 or 1) and the following values are arguments
    for the given algorithm. See other ctl files for examples of what the
    delimitation line looks like. This is where you can enter the params
    (e.g., alpha, migration) for the two different algorithms.
    For example, the following args would produce the following ctl lines:
       alg=0, epsilon=5
       > delimit_alg = (0, 5)
       speciesdelimitation = 1 0 5

       alg=1, alpha=2, migration=1
       > delimit_alg = (1, 2, 1)
       speciesdelimitation = 1 1 2 1

       alg=1, alpha=2, migration=1, diagnosis=0, ?=1
       > delimit_alg = (1, 2, 1, 0, 1)
       speciesdelimitation = 1 1 2 1 0 1
seed:
    A random number seed at start of analysis.
burnin:
    Number of burnin generations in mcmc
nsample:
    Number of mcmc generations to run.
sampfreq:
    How often to sample from the mcmc chain.
thetaprior:
    Prior on theta (4Neu), gamma distributed. mean = a/b. e.g., (5, 5)
tauprior
    Prior on root tau, gamma distributed mean = a/b. Last number is
    dirichlet prior for other taus. e.g., (4, 2, 1)
traits_df:
    A pandas DataFrame with trait data properly formatted. This means only
    quantitative traits are included, and missing values are NaN.
    The first column contains sample names, with "Indiv" as the header.
    The following columns have a header row with trait names. This script
    will write a CSV trait file with trait values mean-standardized, with
    NaN replaced by "NA", and with sample not present in IMAP removed.
nu:
    A prior on phenotypic trait variance (0) for iBPP analysis.
kappa:
    A prior on phenotypic trait mean (0) for iBPP analysis.
useseqdata:
    If false inference proceeds without sequence data (can be used to test
    the effect of priors on the tree distributions).
usetraitdata:
    If false inference proceeds without trait data (can be used to test
    the effect of priors on the trait distributions).
cleandata:
    If 1 then sites with missing or hetero characters are removed.
wdir:
    A working directory to write files to.
finetune:
    See bpp documentation.
verbose:
    If verbose=1 the ctl file text will also be written to screen (stderr).


def write_ctl(name, imap, guidetree, nloci,<EOL>infer_sptree, infer_delimit, delimit_alg,<EOL>seed, burnin, nsample, sampfreq,<EOL>thetaprior, tauprior, traits_df, nu0, kappa0,<EOL>cleandata, useseqdata, usetraitdata, wdir,<EOL>finetune, verbose):

<EOL>ctl = []<EOL>if not guidetree.endswith("<STR_LIT:;>"):<EOL><INDENT>guidetree += "<STR_LIT:;>"<EOL><DEDENT>prog = '<STR_LIT>'<EOL>if isinstance(traits_df, pd.DataFrame):<EOL><INDENT>prog = '<STR_LIT>'<EOL><DEDENT>ctl.append("<STR_LIT>".format(seed))<EOL>ctl.append("<STR_LIT>".format(OPJ(wdir, name), prog))<EOL>ctl.append("<STR_LIT>".format(OPJ(wdir, name), prog))<EOL>ctl.append("<STR_LIT>".format(OPJ(wdir, name), prog))<EOL>ctl.append("<STR_LIT>".format(OPJ(wdir, name), prog))<EOL>if isinstance(traits_df, pd.DataFrame):<EOL><INDENT>ctl.append("<STR_LIT>".format(OPJ(wdir, name), prog))<EOL><DEDENT>ctl.append("<STR_LIT>".format(nloci))<EOL>ctl.append("<STR_LIT>".format(useseqdata))<EOL>ctl.append("<STR_LIT>".format(cleandata))<EOL>if infer_sptree:<EOL><INDENT>ctl.append("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>ctl.append("<STR_LIT>")<EOL><DEDENT>ctl.append("<STR_LIT>".format(infer_delimit, delimit_alg[<NUM_LIT:0>],<EOL>"<STR_LIT:U+0020>".join([str(i) for i in delimit_alg[<NUM_LIT:1>:]])))<EOL>if isinstance(traits_df, pd.DataFrame):<EOL><INDENT>try:<EOL><INDENT>traits_df.values.astype(float)<EOL><DEDENT>except Exception:<EOL><INDENT>raise IPyradWarningExit(PDREAD_ERROR)<EOL><DEDENT>samples = sorted(list(itertools.chain(*imap.values())))<EOL>didx = [list(traits_df.index).index(i) for i in traits_df.indexif i not in samples]<EOL>dtraits = traits_df.drop(traits_df.index[didx])<EOL>straits = dtraits.apply(lambda x: (x - x.mean()) / (x.std()))<EOL>ftraits = straits.fillna("<STR_LIT>")<EOL>traitdict = ftraits.T.to_dict("<STR_LIT:list>")<EOL>rev = {val:key for key in sorted(imap) for val in imap[key]}<EOL>traitfile = "<STR_LIT>".format(os.path.join(wdir, name), prog)<EOL>with open(traitfile, '<STR_LIT:w>') as tout:<EOL><INDENT>tout.write("<STR_LIT>")<EOL>tout.write("<STR_LIT:\t>".join(<EOL>['<STR_LIT>'] + list(ftraits.columns))+"<STR_LIT:\n>"<EOL>)<EOL>nindT = <NUM_LIT:0><EOL>for ikey in sorted(imap.keys()):<EOL><INDENT>samps = imap[ikey]<EOL>for samp in sorted(samps):<EOL><INDENT>if samp in traitdict:<EOL><INDENT>tout.write("<STR_LIT:\t>".join([samp, rev[samp]] +[str(i) for i in traitdict[samp]])+"<STR_LIT:\n>"<EOL>)<EOL>nindT += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>ctl.append("<STR_LIT>".format(traits_df.shape[<NUM_LIT:1>]))<EOL>ctl.append("<STR_LIT>".format(nindT))  <EOL>ctl.append("<STR_LIT>".format(usetraitdata))<EOL>ctl.append("<STR_LIT>".format(useseqdata))<EOL>ctl.append("<STR_LIT>".format(nu0))<EOL>ctl.append("<STR_LIT>".format(kappa0))<EOL>ctl.remove("<STR_LIT>".format(useseqdata))<EOL>ctl.remove("<STR_LIT>".format(infer_sptree))<EOL><DEDENT>nspecies = str(len(imap))<EOL>species = "<STR_LIT:U+0020>".join(sorted(imap))<EOL>ninds = "<STR_LIT:U+0020>".join([str(len(imap[i])) for i in sorted(imap)])<EOL>ctl.append("""<STR_LIT>""".format(nspecies, species, ninds, guidetree))<EOL>ctl.append("<STR_LIT>".format(*thetaprior))<EOL>ctl.append("<STR_LIT>".format(*tauprior))<EOL>ctl.append("<STR_LIT>".format("<STR_LIT:U+0020>".join([str(i) for i in finetune])))<EOL>ctl.append("<STR_LIT>")<EOL>ctl.append("<STR_LIT>".format(burnin))<EOL>ctl.append("<STR_LIT>".format(sampfreq))<EOL>ctl.append("<STR_LIT>".format(nsample))<EOL>with open("<STR_LIT>".format(OPJ(wdir, name), prog), '<STR_LIT:w>') as out:<EOL><INDENT>out.write("<STR_LIT:\n>".join(ctl))<EOL><DEDENT>if verbose:<EOL><INDENT>sys.stderr.write("<STR_LIT>"+"<STR_LIT:\n>".join(ctl)+"<STR_LIT>")<EOL><DEDENT>


counts = [Counter(col) for col in seqsamp.T if not ("<STR_LIT:->" in col or "<STR_LIT:N>" in col)]<EOL>pis = [i.most_common(<NUM_LIT:2>)[<NUM_LIT:1>][<NUM_LIT:1>] > <NUM_LIT:1> for i in counts if len(i.most_common(<NUM_LIT:2>))><NUM_LIT:1>]<EOL>if sum(pis) >= N:<EOL><INDENT>return sum(pis)<EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>


<EOL>if outdir:<EOL><INDENT>if not os.path.exists(outdir):<EOL><INDENT>os.makedirs(outdir)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>outdir = os.path.curdir<EOL><DEDENT>outdir = os.path.realpath(outdir)<EOL>outdir = os.path.join(outdir, "<STR_LIT>".format(name))<EOL>if not os.path.exists(outdir):<EOL><INDENT>os.makedirs(outdir)<EOL><DEDENT>else:<EOL><INDENT>ofiles = glob.glob(os.path.join(outdir, "<STR_LIT>"))<EOL>for ofile in ofiles:<EOL><INDENT>os.remove(ofile)<EOL><DEDENT><DEDENT>with open(locifile) as infile:<EOL><INDENT>loci = (i for i in infile.read().strip().split("<STR_LIT>"))<EOL><DEDENT>if not subsamples:<EOL><INDENT>with open(locifile) as infile:<EOL><INDENT>subs = set((i.split()[<NUM_LIT:0>] for i in infile.readlines() if "<STR_LIT>" not in i))<EOL><DEDENT><DEDENT>else:   <EOL><INDENT>subs = set(subsamples)<EOL><DEDENT>lens = len(subs)<EOL>nlocus = <NUM_LIT:0><EOL>for loc in loci:<EOL><INDENT>dat = loc.split("<STR_LIT:\n>")[:-<NUM_LIT:1>]<EOL>names = [i.split()[<NUM_LIT:0>] for i in dat]<EOL>seqs = np.array([list(i.split()[<NUM_LIT:1>]) for i in dat])<EOL>if len(set(names).intersection(set(subs))) == lens:<EOL><INDENT>seqsamp = seqs[[names.index(tax) for tax in subs]]<EOL>seqsamp = _resolveambig(seqsamp)<EOL>pis = _count_PIS(seqsamp, minSNPs)<EOL>if pis:<EOL><INDENT>nlocus += <NUM_LIT:1><EOL>copied = seqsamp.copy()<EOL>copied[copied == "<STR_LIT:->"] = "<STR_LIT:N>"<EOL>rmcol = np.all(copied == "<STR_LIT:N>", axis=<NUM_LIT:0>)<EOL>seqsamp = seqsamp[:, ~rmcol]<EOL>mdict = dict(list(zip(subs, [i.tostring() for i in seqsamp])))<EOL>nexmake(mdict, nlocus, outdir, mcmc_burnin, mcmc_ngen, mcmc_sample_freq)<EOL><DEDENT><DEDENT><DEDENT>print("<STR_LIT>".format(nlocus, outdir))<EOL>

Converts loci file format to multiple nexus formatted files, one for
each locus, and writes a mrbayes block in the nexus information. The
mrbayes block will be set to run 2 replicate chains, for [mcmc_ngen]
generations, skipping [burnin] steps, and sampling every
[mcmc_sample_freq] steps.

Parameters:
-----------
name: (str)
    A prefix name for output files that will be produced
locifile: (str)
    A .loci file produced by ipyrad.
maxloci: (int)
    Limit the number of loci to the first N loci with sufficient sampling
    to be included in the analysis.
minSNPs: (int)
    Only include loci that have at least N parsimony informative SNPs.
seed: (int)
    Random seed used for resolving ambiguities.
burnin: (int)
    mrbayes nexus block burnin parameter used for 'sump burnin' and 'sumt burnin'.
    The number of generations to skip before starting parameter and tree sampling.
mcmc_ngen: (int)
    mrbayes nexus block 'mcmc ngen' and 'mcmc printfreq' parameters. We don't really
    to have any info printed to screen, so these values are set equal. This is the
    length of the chains that will be run.
mcmc_sample_freq: (int)
    mrbayes nexus block 'mcmc samplefreq' parameter. The frequency of sampling from
    the mcmc chain.


<EOL>tre = toytree.tree(newick=self.results.tree)<EOL>tre.draw(<EOL>axes=axes,<EOL>use_edge_lengths=True,<EOL>tree_style='<STR_LIT:c>',<EOL>tip_labels_align=True,<EOL>edge_align_style={"<STR_LIT>": <NUM_LIT:1>}<EOL>);<EOL>for admix in self.results.admixture:<EOL><INDENT>pidx, pdist, cidx, cdist, weight = admix<EOL>a = _get_admix_point(tre, pidx, pdist)<EOL>b = _get_admix_point(tre, cidx, cdist)<EOL>mark = axes.plot(<EOL>a = (a[<NUM_LIT:0>], b[<NUM_LIT:0>]),<EOL>b = (a[<NUM_LIT:1>], b[<NUM_LIT:1>]),<EOL>style={"<STR_LIT>": <NUM_LIT:10>*weight,<EOL>"<STR_LIT>": <NUM_LIT>, <EOL>"<STR_LIT>": "<STR_LIT>"}<EOL>)<EOL>axes.scatterplot(<EOL>a = (b[<NUM_LIT:0>]),<EOL>b = (b[<NUM_LIT:1>]),<EOL>size=<NUM_LIT:8>,<EOL>title="<STR_LIT>".format(weight),<EOL>)<EOL><DEDENT>axes.y.show=False<EOL>axes.x.ticks.show=True<EOL>axes.x.label.text = "<STR_LIT>"<EOL>return axes<EOL>

Returns a treemix plot on a toyplot.axes object.


<EOL>subdict = {i:j for i, j in self.__dict__.iteritems() if i != "<STR_LIT>"}<EOL>newdict = copy.deepcopy(subdict)<EOL>newobj = Treemix(<EOL>data=newdict["<STR_LIT:data>"],<EOL>name=name,<EOL>workdir=newdict["<STR_LIT>"],<EOL>imap={i:j for i, j in newdict["<STR_LIT>"].items()},<EOL>mapfile=newdict['<STR_LIT>'],<EOL>minmap={i:j for i, j in newdict["<STR_LIT>"].items()},<EOL>seed=np.random.randint(<NUM_LIT:0>, int(<NUM_LIT>)),<EOL>)<EOL>for key, val in newobj.params.__dict__.iteritems():<EOL><INDENT>newobj.params.__setattr__(key, self.params.__getattribute__(key))<EOL><DEDENT>return newobj<EOL>

Returns a copy of the treemix object with the same parameter settings
but with the files attributes cleared, and with a new 'name' attribute.

Parameters
----------
name (str):
    A name for the new copied treemix bject that will be used for the
    output files created by the object.


proc = subprocess.Popen(<EOL>command_list,<EOL>stderr=subprocess.STDOUT, <EOL>stdout=subprocess.PIPE<EOL>)<EOL>comm = proc.communicate()<EOL>return comm<EOL>


return "<STR_LIT:U+0020>".join(self._command_list)<EOL>


self.results_table, self.results_boots = batch(self, ipyclient)<EOL>if not isinstance(self.results_table, list):<EOL><INDENT>self.results_table.nloci = np.nan_to_num(self.results_table.nloci).astype(int)<EOL><DEDENT>

Run a batch of dstat tests on a list of tests, where each test is
a dictionary mapping sample names to {p1 - p4} (and sometimes p5).
Parameters modifying the behavior of the run, such as the number
of bootstrap replicates (nboots) or the minimum coverage for
loci (mincov) can be set in {object}.params.

Parameters:
-----------
ipyclient (ipyparallel.Client object):
    An ipyparallel client object to distribute jobs to a cluster.


def _loci_to_arr(loci, taxdict, mindict):

<EOL>nloci = len(loci)<EOL>maxlen = np.max(np.array([len(locus.split("<STR_LIT:\n>")[<NUM_LIT:0>]) for locus in loci]))<EOL>keep = np.zeros(nloci, dtype=np.bool_)<EOL>arr = np.zeros((nloci, <NUM_LIT:4>, maxlen), dtype=np.float64)<EOL>if len(taxdict) == <NUM_LIT:5>:<EOL><INDENT>arr = np.zeros((nloci, <NUM_LIT:6>, maxlen), dtype=np.float64)<EOL><DEDENT>if isinstance(mindict, int):<EOL><INDENT>mindict = {i: mindict for i in taxdict}<EOL><DEDENT>elif isinstance(mindict, dict):<EOL><INDENT>mindict = {i: mindict[i] for i in taxdict}<EOL><DEDENT>else:<EOL><INDENT>mindict = {i: <NUM_LIT:1> for i in taxdict}<EOL><DEDENT>allowed_names = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>if any([i not in allowed_names for i in taxdict]):<EOL><INDENT>raise IPyradError("<STR_LIT>")<EOL><DEDENT>keys = sorted([i for i in taxdict.keys() if i[<NUM_LIT:0>] == '<STR_LIT:p>'])<EOL>outg = keys[-<NUM_LIT:1>]<EOL>for loc in xrange(nloci):<EOL><INDENT>lines = loci[loc].split("<STR_LIT:\n>")[:-<NUM_LIT:1>]<EOL>names = [i.split()[<NUM_LIT:0>] for i in lines]<EOL>seqs = np.array([list(i.split()[<NUM_LIT:1>]) for i in lines])<EOL>covs = [sum([j in names for j in taxdict[tax]]) >= mindict[tax]for tax in taxdict]<EOL>if all(covs):<EOL><INDENT>keep[loc] = True<EOL>refidx = np.where([i in taxdict[outg] for i in names])[<NUM_LIT:0>]<EOL>refseq = seqs[refidx].view(np.uint8)<EOL>ancestral = np.array([reftrick(refseq, GETCONS2)[:, <NUM_LIT:0>]])<EOL>iseq = _reffreq2(ancestral, refseq, GETCONS2)<EOL>arr[loc, -<NUM_LIT:1>, :iseq.shape[<NUM_LIT:1>]] = iseq <EOL>if len(taxdict) == <NUM_LIT:4>:<EOL><INDENT>for tidx, key in enumerate(keys[:-<NUM_LIT:1>]):<EOL><INDENT>nidx = np.where([i in taxdict[key] for i in names])[<NUM_LIT:0>]<EOL>sidx = seqs[nidx].view(np.uint8)<EOL>iseq = _reffreq2(ancestral, sidx, GETCONS2)<EOL>arr[loc, tidx, :iseq.shape[<NUM_LIT:1>]] = iseq<EOL><DEDENT><DEDENT>else:<EOL><INDENT>iseq = _reffreq2(ancestral, refseq, GETCONS2) <EOL>arr[loc, -<NUM_LIT:1>, :iseq.shape[<NUM_LIT:1>]] = iseq <EOL>nidx = np.where([i in taxdict['<STR_LIT>'] for i in names])[<NUM_LIT:0>]<EOL>sidx = seqs[nidx].view(np.uint8)<EOL>iseq = _reffreq2(ancestral, sidx, GETCONS2)<EOL>arr[loc, <NUM_LIT:0>, :iseq.shape[<NUM_LIT:1>]] = iseq<EOL>nidx = np.where([i in taxdict['<STR_LIT>'] for i in names])[<NUM_LIT:0>]<EOL>sidx = seqs[nidx].view(np.uint8)<EOL>iseq = _reffreq2(ancestral, sidx, GETCONS2)<EOL>arr[loc, <NUM_LIT:1>, :iseq.shape[<NUM_LIT:1>]] = iseq<EOL>nidx = np.where([i in taxdict['<STR_LIT>'] for i in names])[<NUM_LIT:0>]<EOL>nidy = np.where([i in taxdict['<STR_LIT>'] for i in names])[<NUM_LIT:0>]<EOL>sidx = seqs[nidx].view(np.uint8)<EOL>sidy = seqs[nidy].view(np.uint8)<EOL>xseq = _reffreq2(ancestral, sidx, GETCONS2)<EOL>yseq = _reffreq2(ancestral, sidy, GETCONS2)<EOL>mask3 = xseq != <NUM_LIT:0><EOL>mask4 = yseq != <NUM_LIT:0><EOL>xseq[mask4] = <NUM_LIT:0><EOL>yseq[mask3] = <NUM_LIT:0><EOL>arr[loc, <NUM_LIT:2>, :xseq.shape[<NUM_LIT:1>]] = xseq<EOL>arr[loc, <NUM_LIT:3>, :yseq.shape[<NUM_LIT:1>]] = yseq<EOL>nidx = nidx.tolist() + nidy.tolist()<EOL>sidx = seqs[nidx].view(np.uint8)<EOL>iseq = _reffreq2(ancestral, sidx, GETCONS2)<EOL>arr[loc, <NUM_LIT:4>, :iseq.shape[<NUM_LIT:1>]] = iseq<EOL><DEDENT><DEDENT><DEDENT>arr = arr[keep, :, :]<EOL>arr = masknulls(arr)<EOL>return arr, keep<EOL>

return a frequency array from a loci file for all loci with taxa from
taxdict and min coverage from mindict.


<EOL>self._decompose_tree(orient=orient, use_edge_lengths=use_edge_lengths)<EOL>dwargs = {}<EOL>dwargs["<STR_LIT>"] = show_tip_labels<EOL>dwargs["<STR_LIT>"] = show_node_support<EOL>dwargs.update(kwargs)<EOL>canvas, axes, panel = tree_panel_plot(self, print_args, **dwargs)<EOL>return canvas, axes, panel<EOL>

plot the tree using toyplot.graph.

Parameters:
-----------
    show_tip_labels: bool
        Show tip names from tree.
    use_edge_lengths: bool
        Use edge lengths from newick tree.
    show_node_support: bool
        Show support values at nodes using a set of default
        options.

    ...


<EOL>oldfiles = [self.files.qdump] +self.database.__dict__.values() +self.trees.__dict__.values()<EOL>for oldfile in oldfiles:<EOL><INDENT>if oldfile:<EOL><INDENT>if os.path.exists(oldfile):<EOL><INDENT>os.remove(oldfile)<EOL><DEDENT><DEDENT><DEDENT>oldcluster = copy.deepcopy(self._ipcluster)<EOL>self.__init__(<EOL>name=self.name, <EOL>data=self.files.data, <EOL>mapfile=self.files.mapfile,<EOL>workdir=self.dirs,<EOL>method=self.params.method,<EOL>guidetreefile=self.files.guidetreefile,<EOL>resolve=self._resolve, <EOL>nboots=self.params.nboots, <EOL>nquartets=self.params.nquartets, <EOL>initarr=True, <EOL>quiet=True,<EOL>cli=self.kwargs.get("<STR_LIT>")<EOL>)<EOL>self._ipcluster = oldcluster<EOL>

Remove all existing results files and reinit the h5 arrays
so that the tetrad object is just like fresh from a CLI start.


pool = tuple(iterable)<EOL>size = len(pool)<EOL>indices = random.sample(xrange(size), nquartets)<EOL>return tuple(pool[i] for i in indices)<EOL>

Random selection from itertools.combinations(iterable, r).
Use this if not sampling all possible quartets.


pool1 = tuple(iter1)<EOL>pool2 = tuple(iter2)<EOL>ind1 = random.sample(pool1, <NUM_LIT:2>)<EOL>ind2 = random.sample(pool2, <NUM_LIT:2>)<EOL>return tuple(ind1+ind2)<EOL>


def consensus_tree(trees, names=None, cutoff=<NUM_LIT:0.0>):

<EOL>namedict, clade_counts = _find_clades(trees, names=names)<EOL>fclade_counts = _filter_clades(clade_counts, cutoff)<EOL>consens_tree, _ = _build_trees(fclade_counts, namedict)<EOL>return consens_tree, clade_counts<EOL>

An extended majority rule consensus function for ete3.
Modelled on the similar function from scikit-bio tree module. If
cutoff=0.5 then it is a normal majority rule consensus, while if
cutoff=0.0 then subsequent non-conflicting clades are added to the tree.


<EOL>names = tre.get_leaves()<EOL>for name in names:<EOL><INDENT>name.name = self.samples[int(name.name)]<EOL><DEDENT>return tre.write(format=<NUM_LIT:9>)<EOL>

renames newick from numbers to sample names


def _inference(self, start, lbview, quiet=False):

<EOL>gen = xrange(self.checkpoint.arr, self.params.nquartets, self._chunksize)<EOL>njobs = sum(<NUM_LIT:1> for _ in gen)<EOL>jobiter = iter(gen)<EOL>LOGGER.info("<STR_LIT>",self._chunksize, self.checkpoint.arr, self.params.nquartets, njobs)<EOL>key = "<STR_LIT>".format(self.checkpoint.boots)<EOL>with h5py.File(self.database.output, '<STR_LIT>') as out:<EOL><INDENT>if key not in out["<STR_LIT>"].keys():<EOL><INDENT>out["<STR_LIT>"].create_dataset(key, <EOL>(self.params.nquartets, <NUM_LIT:4>), <EOL>dtype=np.uint32, <EOL>chunks=(self._chunksize, <NUM_LIT:4>))<EOL><DEDENT><DEDENT>elapsed = datetime.timedelta(seconds=int(time.time()-start))<EOL>if not self.checkpoint.boots:<EOL><INDENT>printstr = "<STR_LIT>"<EOL>if not quiet:<EOL><INDENT>progressbar(<NUM_LIT:1>, <NUM_LIT:0>, printstr.format(elapsed), spacer="<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>printstr = "<STR_LIT>"<EOL>if not quiet:<EOL><INDENT>progressbar(self.params.nboots, self.checkpoint.boots, <EOL>printstr.format(self.checkpoint.boots, elapsed), spacer="<STR_LIT>")<EOL><DEDENT><DEDENT>res = {}<EOL>for _ in xrange(njobs):<EOL><INDENT>qidx = jobiter.next()<EOL>LOGGER.info('<STR_LIT>', qidx)<EOL>with h5py.File(self.database.input, '<STR_LIT:r>') as inh5:<EOL><INDENT>smps = inh5["<STR_LIT>"][qidx:qidx+self._chunksize]<EOL>res[qidx] = lbview.apply(nworker, *[self, smps, TESTS])<EOL><DEDENT><DEDENT>done = <NUM_LIT:0><EOL>while <NUM_LIT:1>:<EOL><INDENT>curkeys = res.keys()<EOL>finished = [i.ready() for i in res.values()]<EOL>if any(finished):<EOL><INDENT>for ikey in curkeys:<EOL><INDENT>if res[ikey].ready():<EOL><INDENT>if res[ikey].successful():<EOL><INDENT>done += <NUM_LIT:1><EOL>results = res[ikey].get(<NUM_LIT:0>)<EOL>LOGGER.info("<STR_LIT:%s>", results[<NUM_LIT:1>])<EOL>self._insert_to_array(ikey, results) <EOL>del res[ikey]<EOL><DEDENT>else:<EOL><INDENT>raise IPyradWarningExit("""<STR_LIT>""".format(res[ikey].exception()))<EOL><DEDENT><DEDENT>try:<EOL><INDENT>qidx = jobiter.next()<EOL>with h5py.File(self.database.input, '<STR_LIT:r>') as inh5:<EOL><INDENT>smps = inh5["<STR_LIT>"][qidx:qidx+self._chunksize]<EOL><DEDENT>res[qidx] = lbview.apply(nworker, *[self, smps, TESTS])<EOL><DEDENT>except StopIteration:<EOL><INDENT>continue<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>time.sleep(<NUM_LIT>)<EOL><DEDENT>elapsed = datetime.timedelta(seconds=int(time.time()-start))<EOL>if not self.checkpoint.boots:<EOL><INDENT>if not quiet:<EOL><INDENT>progressbar(njobs, done, printstr.format(elapsed), spacer="<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if not quiet:<EOL><INDENT>progressbar(self.params.nboots, self.checkpoint.boots, <EOL>printstr.format(self.checkpoint.boots, elapsed), <EOL>spacer="<STR_LIT>")<EOL><DEDENT><DEDENT>if njobs == done:<EOL><INDENT>break<EOL><DEDENT><DEDENT>self._dump_qmc()<EOL>if not self.checkpoint.boots:<EOL><INDENT>self._run_qmc(<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>self._run_qmc(<NUM_LIT:1>)            <EOL><DEDENT>self.checkpoint.arr = <NUM_LIT:0><EOL>

Inference sends slices of jobs to the parallel engines for computing
and collects the results into the output hdf5 array as they finish.


@numba.jit('<STR_LIT>', nopython=True)<EOL>def subsample_snps_map(seqchunk, nmask, maparr):

<EOL>rmask = np.zeros(seqchunk.shape[<NUM_LIT:1>], dtype=np.bool_)<EOL>last_loc = -<NUM_LIT:1><EOL>for idx in xrange(maparr.shape[<NUM_LIT:0>]):<EOL><INDENT>if maparr[idx] != last_loc:<EOL><INDENT>if not nmask[idx]:<EOL><INDENT>rmask[idx] = True<EOL><DEDENT>last_loc = maparr[idx]<EOL><DEDENT><DEDENT>return rmask<EOL>

removes ncolumns from snparray prior to matrix calculation, and
subsamples 'linked' snps (those from the same RAD locus) such that
for these four samples only 1 SNP per locus is kept. This information
comes from the 'map' array (map file).


<EOL>io5 = h5py.File(self.database.output, '<STR_LIT:r>')<EOL>self.files.qdump = os.path.join(self.dirs, self.name+"<STR_LIT>")<EOL>LOGGER.info("<STR_LIT>", self.files.qdump)<EOL>outfile = open(self.files.qdump, '<STR_LIT:w>')<EOL>for idx in xrange(<NUM_LIT:0>, self.params.nquartets, self._chunksize):<EOL><INDENT>masked_quartets = io5["<STR_LIT>"][idx:idx+self._chunksize, :]<EOL>quarts = [list(j) for j in masked_quartets]<EOL>chunk = ["<STR_LIT>".format(*i) for i in quarts]<EOL>outfile.write("<STR_LIT:\n>".join(chunk)+"<STR_LIT:\n>")<EOL><DEDENT>outfile.close()<EOL>io5.close()<EOL>

Makes a reduced array that excludes quartets with no information and
prints the quartets and weights to a file formatted for wQMC


@numba.jit(nopython=True)<EOL>def fill_boot(seqarr, newboot, newmap, spans, loci):

<EOL>cidx = <NUM_LIT:0><EOL>for i in xrange(loci.shape[<NUM_LIT:0>]):<EOL><INDENT>x1 = spans[loci[i]][<NUM_LIT:0>]<EOL>x2 = spans[loci[i]][<NUM_LIT:1>]<EOL>cols = seqarr[:, x1:x2]<EOL>cord = np.random.choice(cols.shape[<NUM_LIT:1>], cols.shape[<NUM_LIT:1>], replace=False)<EOL>rcols = cols[:, cord]<EOL>newboot[:, cidx:cidx+cols.shape[<NUM_LIT:1>]] = rcols<EOL>newmap[cidx: cidx+cols.shape[<NUM_LIT:1>], <NUM_LIT:0>] = i+<NUM_LIT:1><EOL>cidx += cols.shape[<NUM_LIT:1>]<EOL><DEDENT>return newboot, newmap<EOL>


def run(self, force=<NUM_LIT:0>, verbose=<NUM_LIT:2>, ipyclient=None):

<EOL>if force:<EOL><INDENT>self.refresh()<EOL><DEDENT>inst = None<EOL>try:<EOL><INDENT>if not ipyclient:<EOL><INDENT>args = self._ipcluster.items() + [("<STR_LIT>", "<STR_LIT>")] <EOL>ipyclient = ip.core.parallel.get_client(**dict(args)) <EOL><DEDENT>if verbose == <NUM_LIT:2>:<EOL><INDENT>ip.cluster_info(ipyclient)<EOL><DEDENT>targets = get_targets(ipyclient)<EOL>lbview = ipyclient.load_balanced_view(targets=targets)<EOL>self._ipcluster["<STR_LIT>"] = ipyclient[:].apply(os.getpid).get_dict()<EOL>if not self._chunksize:<EOL><INDENT>if self.params.method != '<STR_LIT>':<EOL><INDENT>self._store_N_samples(ncpus=len(lbview))<EOL><DEDENT>else:<EOL><INDENT>self._store_equal_samples(ncpus=len(lbview))<EOL><DEDENT><DEDENT>start = time.time()            <EOL>if not self.trees.tree:<EOL><INDENT>if verbose:<EOL><INDENT>print("<STR_LIT>".format(self.params.nquartets))<EOL><DEDENT>self._inference(start, lbview, quiet=verbose == <NUM_LIT:0>)<EOL>if verbose:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if verbose:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT><DEDENT>start = time.time()<EOL>if self.params.nboots:<EOL><INDENT>if self.checkpoint.boots == self.params.nboots:<EOL><INDENT>if verbose:<EOL><INDENT>print("<STR_LIT>".format(self.params.nboots))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>while self.checkpoint.boots < self.params.nboots:<EOL><INDENT>if self.files.mapfile:<EOL><INDENT>self._sample_bootseq_array_map()<EOL><DEDENT>else:<EOL><INDENT>self._sample_bootseq_array() <EOL><DEDENT>self.checkpoint.boots += <NUM_LIT:1><EOL>self._inference(start, lbview, quiet=verbose == <NUM_LIT:0>)<EOL><DEDENT><DEDENT>if verbose:<EOL><INDENT>print("<STR_LIT>")<EOL><DEDENT><DEDENT>self.files.stats = os.path.join(self.dirs, self.name+"<STR_LIT>")<EOL>if not self.kwargs.get("<STR_LIT>"):<EOL><INDENT>self._compute_tree_stats(ipyclient)<EOL><DEDENT>else:<EOL><INDENT>self._finalize_stats(ipyclient)<EOL><DEDENT><DEDENT>except KeyboardInterrupt as inst:<EOL><INDENT>LOGGER.info("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL><DEDENT>except IPyradWarningExit as inst:<EOL><INDENT>LOGGER.info("<STR_LIT>", inst)<EOL>print(inst)<EOL><DEDENT>except Exception as inst:<EOL><INDENT>LOGGER.info("<STR_LIT>", inst)<EOL>print("<STR_LIT>".format(inst))<EOL><DEDENT>finally:<EOL><INDENT>try:<EOL><INDENT>self._save()                <EOL>if ipyclient:<EOL><INDENT>ipyclient.abort()<EOL>LOGGER.info("<STR_LIT>", self._ipcluster["<STR_LIT>"])<EOL>for engine_id, pid in self._ipcluster["<STR_LIT>"].items():<EOL><INDENT>LOGGER.info("<STR_LIT>", engine_id)<EOL>LOGGER.info("<STR_LIT>", pid)<EOL>LOGGER.info("<STR_LIT>", ipyclient.queue_status()[engine_id]["<STR_LIT>"])<EOL>if ipyclient.queue_status()[engine_id]["<STR_LIT>"]:<EOL><INDENT>LOGGER.info('<STR_LIT>'.format(engine_id, pid))<EOL>os.kill(pid, <NUM_LIT:2>)<EOL><DEDENT><DEDENT>time.sleep(<NUM_LIT:1>)<EOL>if '<STR_LIT>' in self._ipcluster["<STR_LIT>"]:<EOL><INDENT>LOGGER.info("<STR_LIT>")<EOL>ipyclient.shutdown(hub=True, block=False)<EOL>ipyclient.close()<EOL>LOGGER.info("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>if not ipyclient.outstanding:<EOL><INDENT>ipyclient.purge_everything()<EOL><DEDENT>else:<EOL><INDENT>ipyclient.shutdown(hub=True, block=False)<EOL>ipyclient.close()<EOL>print("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>if inst:<EOL><INDENT>raise inst<EOL><DEDENT><DEDENT>except Exception as inst2:<EOL><INDENT>print("<STR_LIT>".format(inst2))<EOL>LOGGER.error("<STR_LIT>", inst2)<EOL><DEDENT><DEDENT>

Run quartet inference on a SNP alignment and distribute work
across an ipyparallel cluster (ipyclient). Unless passed an
ipyclient explicitly, it looks for a running ipcluster instance
running from the defautl ("") profile, and will raise an exception
if one is not found within a set time limit. If not using the default
profile then you can set "profile" as an argument to the tetrad object.
Parameter settings influencing the run (e.g., nquartets, method) should
be set on the tetrad Class object itself.

Parameters
----------
force (bool):
    Overwrite results for an object with this name if they exist.
verbose (int):
    0=print nothing, 1=print progress bars, 2=print progress bars and
    print cluster info.
ipyclient (ipyparallel.Client object):
    Default is None (use running Default ipcluster instance). To use
    a different ipcluster instance start a Client class object
    and pass it in as an argument here.


if isinstance(data, unicode):<EOL><INDENT>return data.encode("<STR_LIT:utf-8>")<EOL><DEDENT>if isinstance(data, list):<EOL><INDENT>return [_byteify(item, ignore_dicts=True) for item in data]<EOL><DEDENT>if isinstance(data, dict) and not ignore_dicts:<EOL><INDENT>return {<EOL>_byteify(key, ignore_dicts=True): _byteify(value, ignore_dicts=True)<EOL>for key, value in data.iteritems()<EOL>}<EOL><DEDENT>return data<EOL>

converts unicode to utf-8 when reading in json files


<EOL>hosts = []<EOL>for eid in ipyclient.ids:<EOL><INDENT>engine = ipyclient[eid]<EOL>if not engine.outstanding:<EOL><INDENT>hosts.append(engine.apply(socket.gethostname))<EOL><DEDENT><DEDENT>hosts = [i.get() for i in hosts]<EOL>hostset = set(hosts)<EOL>hostzip = zip(hosts, ipyclient.ids)<EOL>hostdict = {host: [i[<NUM_LIT:1>] for i in hostzip if i[<NUM_LIT:0>] == host] for host in hostset}<EOL>targets = list(itertools.chain(*[hostdict[i][:<NUM_LIT:2>] for i in hostdict]))<EOL>return targets<EOL>

A function to find 2 engines per hostname on the ipyclient.
We'll assume that the CPUs are hyperthreaded, which is why
we grab two. If they are not then no foul. Two multi-threaded
jobs will be run on each of the 2 engines per host.


if (node.is_leaf() or node.is_root()):<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>if len(node.children) > <NUM_LIT:2>:<EOL><INDENT>down_r = node.children[<NUM_LIT:0>]<EOL>down_l = node.children[<NUM_LIT:1>]<EOL>for child in node.children[<NUM_LIT:2>:]:<EOL><INDENT>down_l += child<EOL><DEDENT><DEDENT>else:<EOL><INDENT>down_r, down_l = node.children<EOL><DEDENT>lendr = sum(<NUM_LIT:1> for i in down_r.iter_leaves())<EOL>lendl = sum(<NUM_LIT:1> for i in down_l.iter_leaves())<EOL>up_r = node.get_sisters()[<NUM_LIT:0>]<EOL>lenur = sum(<NUM_LIT:1> for i in up_r.iter_leaves())<EOL>lenul = tots - (lendr + lendl + lenur)<EOL>return lendr * lendl * lenur * lenul<EOL><DEDENT>

get total number of quartets possible for a split


return int(reduce(MUL, (Fraction(n-i, i+<NUM_LIT:1>) for i in range(k)), <NUM_LIT:1>))<EOL>

get the number of quartets as n-choose-k. This is used
    in equal splits to decide whether a split should be exhaustively sampled
    or randomly sampled. Edges near tips can be exhaustive while highly nested
    edges probably have too many quartets


compute_tree_stats(self, ipyclient)<EOL>

writes support values as edge labels on unrooted tree