con = sqlite3.connect(self.tmp_cookie_file)<EOL>cur = con.cursor()<EOL>try:<EOL><INDENT>cur.execute('<STR_LIT>'<EOL>'<STR_LIT>'.format(self.domain_name))<EOL><DEDENT>except sqlite3.OperationalError:<EOL><INDENT>cur.execute('<STR_LIT>'<EOL>'<STR_LIT>'.format(self.domain_name))<EOL><DEDENT>cj = http.cookiejar.CookieJar()<EOL>for item in cur.fetchall():<EOL><INDENT>host, path, secure, expires, name = item[:<NUM_LIT:5>]<EOL>value = self._decrypt(item[<NUM_LIT:5>], item[<NUM_LIT:6>])<EOL>c = create_cookie(host, path, secure, expires, name, value)<EOL>cj.set_cookie(c)<EOL><DEDENT>con.close()<EOL>return cj<EOL>


def _decrypt(self, value, encrypted_value):

if sys.platform == '<STR_LIT:win32>':<EOL><INDENT>return self._decrypt_windows_chrome(value, encrypted_value)<EOL><DEDENT>if value or (encrypted_value[:<NUM_LIT:3>] != b'<STR_LIT>'):<EOL><INDENT>return value<EOL><DEDENT>encrypted_value = encrypted_value[<NUM_LIT:3>:]<EOL>encrypted_value_half_len = int(len(encrypted_value) / <NUM_LIT:2>)<EOL>cipher = pyaes.Decrypter(pyaes.AESModeOfOperationCBC(self.key, self.iv))<EOL>decrypted = cipher.feed(encrypted_value[:encrypted_value_half_len])<EOL>decrypted += cipher.feed(encrypted_value[encrypted_value_half_len:])<EOL>decrypted += cipher.feed()<EOL>return decrypted.decode("<STR_LIT:utf-8>")<EOL>


import os<EOL>import ctypes<EOL>try:<EOL><INDENT>return os.geteuid() == <NUM_LIT:0><EOL><DEDENT>except AttributeError:<EOL><INDENT>return ctypes.windll.shell32.IsUserAnAdmin() != <NUM_LIT:0><EOL><DEDENT>return False<EOL>


def cmdclass(path, enable=None, user=None):

import warnings<EOL>from setuptools.command.install import install<EOL>from setuptools.command.develop import develop<EOL>from os.path import dirname, join, exists, realpath<EOL>from traceback import extract_stack<EOL>try:<EOL><INDENT>from notebook.nbextensions import install_nbextension<EOL>from notebook.services.config import ConfigManager<EOL><DEDENT>except ImportError:<EOL><INDENT>try:<EOL><INDENT>from IPython.html.nbextensions import install_nbextension<EOL>from IPython.html.services.config import ConfigManager<EOL><DEDENT>except ImportError:<EOL><INDENT>warnings.warn("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL>return {}<EOL><DEDENT><DEDENT>if user is None:<EOL><INDENT>user = not _is_root()<EOL><DEDENT>calling_file = extract_stack()[-<NUM_LIT:2>][<NUM_LIT:0>]<EOL>fullpath = realpath(calling_file)<EOL>if not exists(fullpath):<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT>extension_dir = join(dirname(fullpath), path)<EOL>def run_nbextension_install(develop):<EOL><INDENT>import sys<EOL>sysprefix = hasattr(sys, '<STR_LIT>')<EOL>if sysprefix:<EOL><INDENT>install_nbextension(<EOL>extension_dir, symlink=develop, sys_prefix=sysprefix)<EOL><DEDENT>else:<EOL><INDENT>install_nbextension(extension_dir, symlink=develop, user=user)<EOL><DEDENT>if enable is not None:<EOL><INDENT>print("<STR_LIT>")<EOL>cm = ConfigManager()<EOL>cm.update('<STR_LIT>', {"<STR_LIT>": {enable: True}})<EOL><DEDENT><DEDENT>class InstallCommand(install):<EOL><INDENT>def run(self):<EOL><INDENT>print("<STR_LIT>")<EOL>install.run(self)<EOL>print("<STR_LIT>")<EOL>run_nbextension_install(False)<EOL><DEDENT><DEDENT>class DevelopCommand(develop):<EOL><INDENT>def run(self):<EOL><INDENT>print("<STR_LIT>")<EOL>develop.run(self)<EOL>print("<STR_LIT>")<EOL>run_nbextension_install(True)<EOL><DEDENT><DEDENT>return {<EOL>'<STR_LIT>': InstallCommand,<EOL>'<STR_LIT>': DevelopCommand,<EOL>}<EOL>

Build nbextension cmdclass dict for the setuptools.setup method.

    Parameters
    ----------
    path: str
        Directory relative to the setup file that the nbextension code
        lives in.
    enable: [str=None]
        Extension to "enable".  Enabling an extension causes it to be loaded
        automatically by the IPython notebook.
    user: [bool=None]
        Whether or not the nbextension should be installed in user mode.
        If this is undefined, the script will install as user mode IF the
        installer is not sudo.

    Usage
    -----
    For automatic loading:
    # Assuming `./extension` is the relative path to the JS files and
    # `./extension/main.js` is the file that you want automatically loaded.
    setup(
        name='extension',
        ...
        cmdclass=cmdclass('extension', 'extension/main'),
    )

    For manual loading:
    # Assuming `./extension` is the relative path to the JS files.
    setup(
        name='extension',
        ...
        cmdclass=cmdclass('extension'),
    )


if not isinstance(containers, collections.Mapping):<EOL><INDENT>containers = dict((c.name, c) for c in containers)<EOL><DEDENT>container_links = dict((name, set(c.links.keys()))<EOL>for name, c in containers.items())<EOL>sorted_names = _resolve(container_links)<EOL>return [containers[name] for name in sorted_names]<EOL>

Sort a dictionary or list of containers into dependency order

    Returns a sequence


<EOL>count = <NUM_LIT:1><EOL>count_value = values.get('<STR_LIT:count>', <NUM_LIT:1>)<EOL>if isinstance(count_value, int):<EOL><INDENT>count = max(count_value, <NUM_LIT:1>)<EOL><DEDENT>def with_index(name, idx):<EOL><INDENT>if name and idx:<EOL><INDENT>return '<STR_LIT>' % (name, idx)<EOL><DEDENT>return name<EOL><DEDENT>def get_instance(n, idx=None):<EOL><INDENT>return BlockadeContainerConfig(<EOL>with_index(n, idx),<EOL>values['<STR_LIT:image>'],<EOL>command=values.get('<STR_LIT>'),<EOL>links=values.get('<STR_LIT>'),<EOL>volumes=values.get('<STR_LIT>'),<EOL>publish_ports=values.get('<STR_LIT>'),<EOL>expose_ports=values.get('<STR_LIT>'),<EOL>environment=values.get('<STR_LIT>'),<EOL>hostname=values.get('<STR_LIT>'),<EOL>dns=values.get('<STR_LIT>'),<EOL>start_delay=values.get('<STR_LIT>', <NUM_LIT:0>),<EOL>neutral=values.get('<STR_LIT>', False),<EOL>holy=values.get('<STR_LIT>', False),<EOL>container_name=with_index(values.get('<STR_LIT>'), idx),<EOL>cap_add=values.get('<STR_LIT>'))<EOL><DEDENT>if count == <NUM_LIT:1>:<EOL><INDENT>yield get_instance(name)<EOL><DEDENT>else:<EOL><INDENT>for idx in range(<NUM_LIT:1>, count+<NUM_LIT:1>):<EOL><INDENT>yield get_instance(name, idx)<EOL><DEDENT><DEDENT>

Convert a dictionary of configuration values
into a sequence of BlockadeContainerConfig instances


try:<EOL><INDENT>containers = values['<STR_LIT>']<EOL>parsed_containers = {}<EOL>for name, container_dict in containers.items():<EOL><INDENT>try:<EOL><INDENT>for cnt in BlockadeContainerConfig.from_dict(name, container_dict):<EOL><INDENT>if cnt.container_name:<EOL><INDENT>cname = cnt.container_name<EOL>existing = [c for c in parsed_containers.values() if c.container_name == cname]<EOL>if existing:<EOL><INDENT>raise BlockadeConfigError("<STR_LIT>" % (cname))<EOL><DEDENT><DEDENT>parsed_containers[cnt.name] = cnt<EOL><DEDENT><DEDENT>except Exception as err:<EOL><INDENT>raise BlockadeConfigError(<EOL>"<STR_LIT>" % (name, err))<EOL><DEDENT><DEDENT>network = values.get('<STR_LIT>')<EOL>if network:<EOL><INDENT>defaults = _DEFAULT_NETWORK_CONFIG.copy()<EOL>defaults.update(network)<EOL>network = defaults<EOL><DEDENT>else:<EOL><INDENT>network = _DEFAULT_NETWORK_CONFIG.copy()<EOL><DEDENT>return BlockadeConfig(parsed_containers, network=network)<EOL><DEDENT>except KeyError as err:<EOL><INDENT>raise BlockadeConfigError("<STR_LIT>" + str(err))<EOL><DEDENT>except Exception as err:<EOL><INDENT>raise BlockadeConfigError("<STR_LIT>" + str(err))<EOL><DEDENT>

Instantiate a BlockadeConfig instance based on
a given dictionary of configuration values


wait(lambda: len(multiprocessing.active_children()) == <NUM_LIT:0>)<EOL>

Wait for child processes to exit

    The testing system launches and terminates child processes, but
    doesn't wait for them to actually die. So in a few places we need
    this extra call


config = load_config(opts.config)<EOL>b = get_blockade(config, opts)<EOL>containers = b.create(verbose=opts.verbose, force=opts.force)<EOL>print_containers(containers, opts.json)<EOL>

Start the containers and link them together


config = load_config(opts.config)<EOL>b = get_blockade(config, opts)<EOL>b.destroy()<EOL>

Destroy all containers and restore networks


config = load_config(opts.config)<EOL>b = get_blockade(config, opts)<EOL>containers = b.status()<EOL>print_containers(containers, opts.json)<EOL>


__with_containers(opts, Blockade.start)<EOL>


kill_signal = opts.signal if hasattr(opts, '<STR_LIT>') else "<STR_LIT>"<EOL>__with_containers(opts, Blockade.kill, signal=kill_signal)<EOL>


__with_containers(opts, Blockade.stop)<EOL>


__with_containers(opts, Blockade.restart)<EOL>


__with_containers(opts, Blockade.flaky)<EOL>

Make the network flaky for some or all containers


__with_containers(opts, Blockade.slow)<EOL>

Make the network slow for some or all containers


__with_containers(opts, Blockade.fast)<EOL>

Restore network speed and reliability for some or all containers


__with_containers(opts, Blockade.duplicate)<EOL>

Introduce packet duplication into the network of some or all containers


config = load_config(opts.config)<EOL>b = get_blockade(config, opts)<EOL>if opts.random:<EOL><INDENT>if opts.partitions:<EOL><INDENT>raise BlockadeError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>b.random_partition()<EOL><DEDENT>else:<EOL><INDENT>partitions = []<EOL>for partition in opts.partitions:<EOL><INDENT>names = []<EOL>for name in partition.split("<STR_LIT:U+002C>"):<EOL><INDENT>name = name.strip()<EOL>if name:<EOL><INDENT>names.append(name)<EOL><DEDENT><DEDENT>partitions.append(names)<EOL><DEDENT>if not partitions:<EOL><INDENT>raise BlockadeError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>b.partition(partitions)<EOL><DEDENT>

Partition the network between containers

    Replaces any existing partitions outright. Any containers NOT specified
    in arguments will be globbed into a single implicit partition. For
    example if you have three containers: c1, c2, and c3 and you run:

        blockade partition c1

    The result will be a partition with just c1 and another partition with
    c2 and c3.

    Alternatively, --random may be specified, and zero or more random
    partitions will be generated by blockade.


config = load_config(opts.config)<EOL>b = get_blockade(config, opts)<EOL>b.join()<EOL>

Restore full networking between containers


config = load_config(opts.config)<EOL>b = get_blockade(config, opts)<EOL>puts(b.logs(opts.container).decode(encoding='<STR_LIT>'))<EOL>


if opts.data_dir is None:<EOL><INDENT>raise BlockadeError("<STR_LIT>")<EOL><DEDENT>rest.start(data_dir=opts.data_dir, port=opts.port, debug=opts.debug,<EOL>host_exec=get_host_exec())<EOL>


config = load_config(opts.config)<EOL>b = get_blockade(config, opts)<EOL>b.add_container(opts.containers)<EOL>

Add one or more existing Docker containers to a Blockade group


import blockade.version<EOL>puts("<STR_LIT>" + blockade.version.__version__)<EOL>


config = load_config(opts.config)<EOL>b = get_blockade(config, opts)<EOL>if opts.json:<EOL><INDENT>outf = None<EOL>_write = puts<EOL>if opts.output is not None:<EOL><INDENT>outf = open(opts.output, "<STR_LIT:w>")<EOL>_write = outf.write<EOL><DEDENT>try:<EOL><INDENT>delim = "<STR_LIT>"<EOL>logs = b.get_audit().read_logs(as_json=False)<EOL>_write('<STR_LIT>')<EOL>_write(os.linesep)<EOL>for l in logs:<EOL><INDENT>_write(delim + l)<EOL>delim = "<STR_LIT:U+002C>" + os.linesep<EOL><DEDENT>_write(os.linesep)<EOL>_write('<STR_LIT>')<EOL><DEDENT>finally:<EOL><INDENT>if opts.output is not None:<EOL><INDENT>outf.close()<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>puts(colored.blue(columns(["<STR_LIT>",         <NUM_LIT:10>],<EOL>["<STR_LIT>",        <NUM_LIT:16>],<EOL>["<STR_LIT>",         <NUM_LIT:8>],<EOL>["<STR_LIT>",          <NUM_LIT:16>],<EOL>["<STR_LIT>",       <NUM_LIT>])))<EOL>logs = b.get_audit().read_logs(as_json=True)<EOL>for l in logs:<EOL><INDENT>puts(columns([l['<STR_LIT>'],                          <NUM_LIT:10>],<EOL>[str([str(t) for t in l['<STR_LIT>']]), <NUM_LIT:16>],<EOL>[l['<STR_LIT:status>'],                          <NUM_LIT:8>],<EOL>[str(l['<STR_LIT>']),                 <NUM_LIT:16>],<EOL>[l['<STR_LIT:message>'],                        <NUM_LIT>]))<EOL><DEDENT><DEDENT>


result = {}<EOL>if not blockade_id:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>lines = self.get_chain_rules("<STR_LIT>")<EOL>for line in lines:<EOL><INDENT>parts = line.split()<EOL>if len(parts) < <NUM_LIT:4>:<EOL><INDENT>continue<EOL><DEDENT>try:<EOL><INDENT>partition_index = parse_partition_index(blockade_id, parts[<NUM_LIT:0>])<EOL><DEDENT>except ValueError:<EOL><INDENT>continue  <EOL><DEDENT>source = parts[<NUM_LIT:3>]<EOL>if source:<EOL><INDENT>result[source] = partition_index<EOL><DEDENT><DEDENT>return result<EOL>

Get a map of blockade chains IDs -> list of IPs targeted at them

        For figuring out which container is in which partition


def insert_rule(self, chain, src=None, dest=None, target=None):

if not chain:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if not target:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if not (src or dest):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>args = ["<STR_LIT>", chain]<EOL>if src:<EOL><INDENT>args += ["<STR_LIT>", src]<EOL><DEDENT>if dest:<EOL><INDENT>args += ["<STR_LIT>", dest]<EOL><DEDENT>args += ["<STR_LIT>", target]<EOL>self.call(*args)<EOL>


if not chain:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>self.call("<STR_LIT>", chain)<EOL>


<EOL>self.delete_blockade_rules(blockade_id)<EOL>self.delete_blockade_chains(blockade_id)<EOL>

Remove all iptables rules and chains related to this blockade


millisec = random.randint(self._start_min_delay, self._start_max_delay)<EOL>self._timer = threading.Timer(millisec / <NUM_LIT>, self.event_timeout)<EOL>self._timer.start()<EOL>


_logger.info("<STR_LIT>" % self._blockade_name)<EOL>self._do_blockade_event()<EOL>millisec = random.randint(self._run_min_time, self._run_max_time)<EOL>self._timer = threading.Timer(millisec / <NUM_LIT>, self.event_timeout)<EOL>self._timer.start()<EOL>


def _sm_stop_from_no_pain(self, *args, **kwargs):

<EOL>_logger.info("<STR_LIT>" % self._blockade_name)<EOL>self._timer.cancel()<EOL>

Stop chaos when there is no current blockade operation


def _sm_relieve_pain(self, *args, **kwargs):

_logger.info(<EOL>"<STR_LIT>" % self._blockade_name)<EOL>self._do_reset_all()<EOL>millisec = random.randint(self._start_min_delay, self._start_max_delay)<EOL>self._timer = threading.Timer(millisec/<NUM_LIT>, self.event_timeout)<EOL>self._timer.start()<EOL>

End the blockade event and return to a steady state


def _sm_stop_from_pain(self, *args, **kwargs):

_logger.info("<STR_LIT>" % self._blockade_name)<EOL>self._do_reset_all()<EOL>

Stop chaos while there is a blockade event in progress


if self._done_notification_func is not None:<EOL><INDENT>self._done_notification_func()<EOL><DEDENT>self._timer.cancel()<EOL>

Delete all state associated with the chaos session


def _sm_stale_timer(self, *args, **kwargs):

This is used when a cancel was called right before the timer fired but
after it was too late to cancel the timer.


<EOL>all_names = frozenset(c.name for c in containers if not c.holy)<EOL>holy_names = frozenset(c.name for c in containers if c.holy)<EOL>neutral_names = frozenset(c.name for c in containers if c.neutral)<EOL>partitions = [frozenset(p) for p in partitions]<EOL>unknown = set()<EOL>holy = set()<EOL>union = set()<EOL>for partition in partitions:<EOL><INDENT>unknown.update(partition - all_names - holy_names)<EOL>holy.update(partition - all_names)<EOL>union.update(partition)<EOL><DEDENT>if unknown:<EOL><INDENT>raise BlockadeError('<STR_LIT>' %<EOL>list(unknown))<EOL><DEDENT>if holy:<EOL><INDENT>raise BlockadeError('<STR_LIT>' %<EOL>list(holy))<EOL><DEDENT>leftover = all_names.difference(union)<EOL>if leftover:<EOL><INDENT>partitions.append(leftover)<EOL><DEDENT>if not neutral_names.issubset(leftover):<EOL><INDENT>partitions.append(neutral_names)<EOL><DEDENT>return partitions<EOL>

Validate the partitions of containers. If there are any containers
not in any partition, place them in an new partition.


return "<STR_LIT>" % self._blockade_id<EOL>

Generate blockade nework name based on the blockade_id


return deepcopy(self._containers)<EOL>


container = self._containers.get(name, None)<EOL>if not container is None:<EOL><INDENT>return container.get('<STR_LIT:id>', None)<EOL><DEDENT>return None<EOL>

Try to find the container ID with the specified name


self._containers = deepcopy(containers)<EOL>self.__write(containers, initialize=True)<EOL>

Initialize a new state file with the given contents.
This function fails in case the state file already exists.


return os.path.isfile(self._state_file)<EOL>

Checks whether a blockade state file already exists


self._containers = deepcopy(containers)<EOL>self.__write(containers, initialize=False)<EOL>

Update the current state file with the specified contents


try:<EOL><INDENT>with open(self._state_file) as f:<EOL><INDENT>state = yaml.safe_load(f)<EOL>self._containers = state['<STR_LIT>']<EOL><DEDENT><DEDENT>except (IOError, OSError) as err:<EOL><INDENT>if err.errno == errno.ENOENT:<EOL><INDENT>raise NotInitializedError("<STR_LIT>")<EOL><DEDENT>raise InconsistentStateError("<STR_LIT>"<EOL>+ str(err))<EOL><DEDENT>except Exception as err:<EOL><INDENT>raise InconsistentStateError("<STR_LIT>"<EOL>+ str(err))<EOL><DEDENT>

Try to load a blockade state file in the current directory


Try to remove the current state file and directory


if not cwd:<EOL><INDENT>cwd = os.getcwd()<EOL><DEDENT>parent_dir = os.path.abspath(cwd)<EOL>basename = os.path.basename(parent_dir).lower()<EOL>blockade_id = re.sub(r"<STR_LIT>", "<STR_LIT>", basename)<EOL>if not blockade_id:  <EOL><INDENT>blockade_id = "<STR_LIT:default>"<EOL><DEDENT>return blockade_id<EOL>

Generate a new blockade ID based on the CWD


try:<EOL><INDENT>os.makedirs(self._state_dir)<EOL><DEDENT>except OSError as err:<EOL><INDENT>if err.errno != errno.EEXIST:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>


try:<EOL><INDENT>os.remove(self._state_file)<EOL><DEDENT>except OSError as err:<EOL><INDENT>if err.errno not in (errno.EPERM, errno.ENOENT):<EOL><INDENT>raise<EOL><DEDENT><DEDENT>try:<EOL><INDENT>os.rmdir(self._state_dir)<EOL><DEDENT>except OSError as err:<EOL><INDENT>if err.errno not in (errno.ENOTEMPTY, errno.ENOENT):<EOL><INDENT>raise<EOL><DEDENT><DEDENT>

Try to delete the state.yml file and the folder .blockade


return dict(blockade_id=self._blockade_id,<EOL>containers=containers,<EOL>version=self._state_version)<EOL>

Convert blockade ID and container information into
a state dictionary object.


def __write(self, containers, initialize=True):

path = self._state_file<EOL>self._assure_dir()<EOL>try:<EOL><INDENT>flags = os.O_WRONLY | os.O_CREAT<EOL>if initialize:<EOL><INDENT>flags |= os.O_EXCL<EOL><DEDENT>with os.fdopen(os.open(path, flags), "<STR_LIT:w>") as f:<EOL><INDENT>yaml.safe_dump(self.__base_state(containers), f)<EOL><DEDENT><DEDENT>except OSError as err:<EOL><INDENT>if err.errno == errno.EEXIST:<EOL><INDENT>raise AlreadyInitializedError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % path)<EOL><DEDENT>raise<EOL><DEDENT>except Exception:<EOL><INDENT>self._state_delete()<EOL>raise<EOL><DEDENT>

Write the given state information into a file


if not options:<EOL><INDENT>return None<EOL><DEDENT>width = len(str(len(options)))<EOL>for x,option in enumerate(options):<EOL><INDENT>sys.stdout.write('<STR_LIT>'.format(x+<NUM_LIT:1>,option, width=width))<EOL><DEDENT>sys.stdout.write('<STR_LIT>'.format('<STR_LIT>', width=width+<NUM_LIT:1>))<EOL>sys.stdout.flush()<EOL>if sys.stdin.isatty():<EOL><INDENT>try:<EOL><INDENT>response = input().strip()<EOL><DEDENT>except (EOFError, KeyboardInterrupt):<EOL><INDENT>response = '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>sys.stdin = open("<STR_LIT>")<EOL>try:<EOL><INDENT>response = '<STR_LIT>'<EOL>while True:<EOL><INDENT>response += sys.stdin.read(<NUM_LIT:1>)<EOL>if response.endswith('<STR_LIT:\n>'):<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>except (EOFError, KeyboardInterrupt):<EOL><INDENT>sys.stdout.flush()<EOL>pass<EOL><DEDENT><DEDENT>try:<EOL><INDENT>response = int(response) - <NUM_LIT:1><EOL><DEDENT>except ValueError:<EOL><INDENT>return None<EOL><DEDENT>if response < <NUM_LIT:0> or response >= len(options):<EOL><INDENT>return None<EOL><DEDENT>return options[response]<EOL>

pass in a list of options, promt the user to select one, and return the selected option or None


f = utils.open_file_read(fname)<EOL>line = f.readline()<EOL>phylip_regex = re.compile('<STR_LIT>')<EOL>gbk_regex = re.compile('<STR_LIT>')<EOL>if line.startswith('<STR_LIT:>>'):<EOL><INDENT>seq = Fasta()<EOL>previous_lines[f] = line<EOL><DEDENT>elif line.startswith('<STR_LIT>'):<EOL><INDENT>seq = Fasta()<EOL>while not line.startswith('<STR_LIT:>>'):<EOL><INDENT>line = f.readline()<EOL>if not line:<EOL><INDENT>utils.close(f)<EOL>raise Error('<STR_LIT>' + fname + '<STR_LIT:">')<EOL><DEDENT><DEDENT>seq = Fasta()<EOL>previous_lines[f] = line<EOL><DEDENT>elif line.startswith('<STR_LIT>') and line[<NUM_LIT:5>] != '<STR_LIT:U+0020>':<EOL><INDENT>seq = Embl()<EOL>previous_lines[f] = line<EOL><DEDENT>elif gbk_regex.search(line):<EOL><INDENT>seq = Embl()<EOL>previous_lines[f] = line<EOL><DEDENT>elif line.startswith('<STR_LIT:@>'):<EOL><INDENT>seq = Fastq()<EOL>previous_lines[f] = line<EOL><DEDENT>elif phylip_regex.search(line):<EOL><INDENT>number_of_seqs, bases_per_seq = line.strip().split()<EOL>number_of_seqs = int(number_of_seqs)<EOL>bases_per_seq = int(bases_per_seq)<EOL>got_blank_line = False<EOL>first_line = line<EOL>seq_lines = []<EOL>while <NUM_LIT:1>:<EOL><INDENT>line = f.readline()<EOL>if line == '<STR_LIT>':<EOL><INDENT>break<EOL><DEDENT>elif line == '<STR_LIT:\n>':<EOL><INDENT>got_blank_line = True<EOL><DEDENT>else:<EOL><INDENT>seq_lines.append(line.rstrip())<EOL><DEDENT><DEDENT>utils.close(f)<EOL>if len(seq_lines) == <NUM_LIT:1> or len(seq_lines) == number_of_seqs:<EOL><INDENT>sequential = True<EOL><DEDENT>elif seq_lines[<NUM_LIT:0>][<NUM_LIT:10>] != '<STR_LIT:U+0020>' and seq_lines[<NUM_LIT:1>][<NUM_LIT:10>] == '<STR_LIT:U+0020>':<EOL><INDENT>sequential = True<EOL><DEDENT>else:<EOL><INDENT>sequential = False<EOL><DEDENT>if sequential:<EOL><INDENT>current_id = None<EOL>current_seq = '<STR_LIT>'<EOL>for line in seq_lines:<EOL><INDENT>if len(current_seq) == bases_per_seq or len(current_seq) == <NUM_LIT:0>:<EOL><INDENT>if current_id is not None:<EOL><INDENT>yield Fasta(current_id, current_seq.replace('<STR_LIT:->', '<STR_LIT>'))<EOL><DEDENT>current_seq = '<STR_LIT>'<EOL>current_id, new_bases = line[<NUM_LIT:0>:<NUM_LIT:10>].rstrip(), line.rstrip()[<NUM_LIT:10>:]<EOL><DEDENT>else:<EOL><INDENT>new_bases = line.rstrip()<EOL><DEDENT>current_seq += new_bases.replace('<STR_LIT:U+0020>','<STR_LIT>')<EOL><DEDENT>yield Fasta(current_id, current_seq.replace('<STR_LIT:->', '<STR_LIT>'))<EOL><DEDENT>else:<EOL><INDENT>if seq_lines[number_of_seqs + <NUM_LIT:1>][<NUM_LIT:0>] == '<STR_LIT:U+0020>':<EOL><INDENT>first_gap_pos = seq_lines[<NUM_LIT:0>].find('<STR_LIT:U+0020>')<EOL>end_of_gap = first_gap_pos<EOL>while seq_lines[<NUM_LIT:0>][end_of_gap] == '<STR_LIT:U+0020>':<EOL><INDENT>end_of_gap += <NUM_LIT:1><EOL><DEDENT>first_seq_base = end_of_gap<EOL><DEDENT>else:<EOL><INDENT>first_seq_base = <NUM_LIT:10><EOL><DEDENT>seqs = []<EOL>for i in range(number_of_seqs):<EOL><INDENT>name, bases = seq_lines[i][<NUM_LIT:0>:first_seq_base].rstrip(), seq_lines[i][first_seq_base:]<EOL>seqs.append(Fasta(name, bases))<EOL><DEDENT>for i in range(number_of_seqs, len(seq_lines)):<EOL><INDENT>seqs[i%number_of_seqs].seq += seq_lines[i]<EOL><DEDENT>for fa in seqs:<EOL><INDENT>fa.seq = fa.seq.replace('<STR_LIT:U+0020>','<STR_LIT>').replace('<STR_LIT:->','<STR_LIT>')<EOL>yield fa<EOL><DEDENT><DEDENT>return<EOL><DEDENT>elif line == '<STR_LIT>':<EOL><INDENT>utils.close(f)<EOL>return<EOL><DEDENT>else:<EOL><INDENT>utils.close(f)<EOL>raise Error('<STR_LIT>' + fname + '<STR_LIT>' + line.rstrip())<EOL><DEDENT>try:<EOL><INDENT>while seq.get_next_from_file(f, read_quals):<EOL><INDENT>yield seq<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>utils.close(f)<EOL><DEDENT>

Iterates over a FASTA or FASTQ file, yielding the next sequence in the file until there are no more sequences


return Fasta(self.id, self.seq[start:end])<EOL>

Returns Fasta object with the same name, of the bases from start to end, but not including end


try:<EOL><INDENT>a = self.id.rsplit('<STR_LIT:.>', <NUM_LIT:1>)<EOL>if a[<NUM_LIT:1>].startswith('<STR_LIT:p>'):<EOL><INDENT>dir = '<STR_LIT>'<EOL><DEDENT>elif a[<NUM_LIT:1>].startswith('<STR_LIT:q>'):<EOL><INDENT>dir = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>dir = '<STR_LIT>'<EOL><DEDENT>return {'<STR_LIT>': a[<NUM_LIT:0>], '<STR_LIT>': dir, '<STR_LIT>':a[<NUM_LIT:1>]}<EOL><DEDENT>except:<EOL><INDENT>raise Error('<STR_LIT>', self.id)<EOL><DEDENT>

Gets the prefix and suffix of an name of a capillary read, e.g. xxxxx.p1k or xxxx.q1k. Returns a tuple (prefix, suffx)


s = list(self.seq)<EOL>for i in range(len(s)):<EOL><INDENT>if s[i] in redundant_nts:<EOL><INDENT>s[i] = '<STR_LIT>'.join(redundant_nts[s[i]])<EOL><DEDENT><DEDENT>seqs = []<EOL>for x in itertools.product(*s):<EOL><INDENT>seqs.append(Fasta(self.id + '<STR_LIT:.>' + str(len(seqs) + <NUM_LIT:1>), '<STR_LIT>'.join(x)))<EOL><DEDENT>return seqs<EOL>

Assumes sequence is nucleotides. Returns list of all combinations of redundant nucleotides. e.g. R is A or G, so CRT would have combinations CAT and CGT


self.id = self.id.split()[<NUM_LIT:0>]<EOL>

Removes everything in the name after the first whitespace character


if self.id.endswith('<STR_LIT>') or self.id.endswith('<STR_LIT>'):<EOL><INDENT>self.id = self.id[:-<NUM_LIT:2>]<EOL><DEDENT>

Removes any trailing /1 or /2 off the end of the name


self.seq = self.seq.translate(str.maketrans("<STR_LIT>", "<STR_LIT>"))[::-<NUM_LIT:1>]<EOL>


def is_all_Ns(self, start=<NUM_LIT:0>, end=None):

if end is not None:<EOL><INDENT>if start > end:<EOL><INDENT>raise Error('<STR_LIT>')<EOL><DEDENT>end += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>end = len(self)<EOL><DEDENT>if len(self) == <NUM_LIT:0>:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>return re.search('<STR_LIT>', self.seq[start:end]) is None<EOL><DEDENT>

Returns true if the sequence is all Ns (upper or lower case)


self.seq = self.seq.strip('<STR_LIT>')<EOL>

Removes any leading or trailing N or n characters from the sequence


self.seq = self.seq.replace(old, new)<EOL>

Replaces all occurrences of 'old' with 'new


self.seq = re.sub(r'''<STR_LIT>''', '<STR_LIT:N>', self.seq)<EOL>

Replace all non acgt characters with an N (case insensitive)


def replace_interval(self, start, end, new):

if start > end or start > len(self) - <NUM_LIT:1> or end > len(self) - <NUM_LIT:1>:<EOL><INDENT>raise Error('<STR_LIT>' + str(start) + '<STR_LIT:->' + str(end) + '<STR_LIT>' + self.id)<EOL><DEDENT>self.seq = self.seq[<NUM_LIT:0>:start] + new + self.seq[end + <NUM_LIT:1>:]<EOL>

Replaces the sequence from start to end with the sequence "new


def gaps(self, min_length = <NUM_LIT:1>):

gaps = []<EOL>regex = re.compile('<STR_LIT>', re.IGNORECASE)<EOL>for m in regex.finditer(self.seq):<EOL><INDENT>if m.span()[<NUM_LIT:1>] - m.span()[<NUM_LIT:0>] + <NUM_LIT:1> >= min_length:<EOL><INDENT>gaps.append(intervals.Interval(m.span()[<NUM_LIT:0>], m.span()[<NUM_LIT:1>] - <NUM_LIT:1>))<EOL><DEDENT><DEDENT>return gaps<EOL>

Finds the positions of all gaps in the sequence that are at least min_length long. Returns a list of Intervals. Coords are zero-based


<EOL>gaps = self.gaps()<EOL>if len(gaps) == <NUM_LIT:0>:<EOL><INDENT>return [intervals.Interval(<NUM_LIT:0>, len(self) - <NUM_LIT:1>)]<EOL><DEDENT>coords = [<NUM_LIT:0>]<EOL>for g in gaps:<EOL><INDENT>if g.start == <NUM_LIT:0>:<EOL><INDENT>coords = [g.end + <NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>coords += [g.start - <NUM_LIT:1>, g.end + <NUM_LIT:1>]<EOL><DEDENT><DEDENT>if coords[-<NUM_LIT:1>] < len(self):<EOL><INDENT>coords.append(len(self) - <NUM_LIT:1>)<EOL><DEDENT>return [intervals.Interval(coords[i], coords[i+<NUM_LIT:1>]) for i in range(<NUM_LIT:0>, len(coords)-<NUM_LIT:1>,<NUM_LIT:2>)]<EOL>

Finds coords of contigs, i.e. everything that's not a gap (N or n). Returns a list of Intervals. Coords are zero-based


def orfs(self, frame=<NUM_LIT:0>, revcomp=False):

assert frame in [<NUM_LIT:0>,<NUM_LIT:1>,<NUM_LIT:2>]<EOL>if revcomp:<EOL><INDENT>self.revcomp()<EOL><DEDENT>aa_seq = self.translate(frame=frame).seq.rstrip('<STR_LIT:X>')<EOL>if revcomp:<EOL><INDENT>self.revcomp()<EOL><DEDENT>orfs = _orfs_from_aa_seq(aa_seq)<EOL>for i in range(len(orfs)):<EOL><INDENT>if revcomp:<EOL><INDENT>start = len(self) - (orfs[i].end * <NUM_LIT:3> + <NUM_LIT:3>) - frame<EOL>end = len(self) - (orfs[i].start * <NUM_LIT:3>) - <NUM_LIT:1> - frame<EOL><DEDENT>else:<EOL><INDENT>start = orfs[i].start * <NUM_LIT:3> + frame<EOL>end = orfs[i].end * <NUM_LIT:3> + <NUM_LIT:2> + frame<EOL><DEDENT>orfs[i] = intervals.Interval(start, end)<EOL><DEDENT>return orfs<EOL>

Returns a list of ORFs that the sequence has, starting on the given
           frame. Each returned ORF is an interval.Interval object.
           If revomp=True, then finds the ORFs of the reverse complement
           of the sequence.


orfs = []<EOL>for frame in [<NUM_LIT:0>,<NUM_LIT:1>,<NUM_LIT:2>]:<EOL><INDENT>for revcomp in [False, True]:<EOL><INDENT>orfs.extend([(t, revcomp) for t in self.orfs(frame=frame, revcomp=revcomp) if len(t)>=min_length])<EOL><DEDENT><DEDENT>return sorted(orfs, key=lambda t:t[<NUM_LIT:0>])<EOL>

Finds all open reading frames in the sequence, that are at least as
           long as min_length. Includes ORFs on the reverse strand.
           Returns a list of ORFs, where each element is a tuple:
           (interval.Interval, bool)
           where bool=True means on the reverse strand


if len(self) %<NUM_LIT:3> != <NUM_LIT:0> or len(self) < <NUM_LIT:6>:<EOL><INDENT>return False<EOL><DEDENT>orfs = self.orfs()<EOL>complete_orf = intervals.Interval(<NUM_LIT:0>, len(self) - <NUM_LIT:1>)<EOL>for orf in orfs:<EOL><INDENT>if orf == complete_orf:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

Returns true iff length is >= 6, is a multiple of 3, and there is exactly one stop codon in the sequence and it is at the end


return self.is_complete_orf()and len(self) >= <NUM_LIT:6>and len(self) %<NUM_LIT:3> == <NUM_LIT:0>and self.seq[<NUM_LIT:0>:<NUM_LIT:3>].upper() in genetic_codes.starts[genetic_code]<EOL>

Returns true iff: length >=6, length is a multiple of 3, first codon is start, last codon is a stop and has no other stop codons


for reverse in [True, False]:<EOL><INDENT>for frame in range(<NUM_LIT:3>):<EOL><INDENT>new_seq = copy.copy(self)<EOL>if reverse:<EOL><INDENT>new_seq.revcomp()<EOL><DEDENT>new_seq.seq = new_seq[frame:]<EOL>if len(new_seq) % <NUM_LIT:3>:<EOL><INDENT>new_seq.seq = new_seq.seq[:-(len(new_seq) % <NUM_LIT:3>)]<EOL><DEDENT>new_aa_seq = new_seq.translate()<EOL>if len(new_aa_seq) >= <NUM_LIT:2> and new_seq[<NUM_LIT:0>:<NUM_LIT:3>] in genetic_codes.starts[genetic_code] and new_aa_seq[-<NUM_LIT:1>] == '<STR_LIT:*>' and '<STR_LIT:*>' not in new_aa_seq[:-<NUM_LIT:1>]:<EOL><INDENT>strand = '<STR_LIT:->' if reverse else '<STR_LIT:+>'<EOL>return new_seq, strand, frame<EOL><DEDENT><DEDENT><DEDENT>return None<EOL>

Tries to make into a gene sequence. Tries all three reading frames and both strands. Returns a tuple (new sequence, strand, frame) if it was successful. Otherwise returns None.


self.seq = self.seq[start:len(self.seq) - end]<EOL>

Removes first 'start'/'end' bases off the start/end of the sequence


if len(self) != len(qual_scores):<EOL><INDENT>raise Error('<STR_LIT>', self.id)<EOL><DEDENT>return Fastq(self.id, self.seq, '<STR_LIT>'.join([chr(max(<NUM_LIT:0>, min(x, <NUM_LIT>)) + <NUM_LIT>) for x in qual_scores]))<EOL>

Returns a Fastq object. qual_scores expected to be a list of numbers, like you would get in a .qual file


seq = self.seq.upper()<EOL>search_string = search_string.upper()<EOL>pos = <NUM_LIT:0><EOL>found = seq.find(search_string, pos)<EOL>hits = []<EOL>while found != -<NUM_LIT:1>:<EOL><INDENT>hits.append((found, '<STR_LIT:+>'))<EOL>pos = found + <NUM_LIT:1><EOL>found = seq.find(search_string, pos)<EOL><DEDENT>pos = <NUM_LIT:0><EOL>search_string = Fasta('<STR_LIT:x>', search_string)<EOL>search_string.revcomp()<EOL>search_string = search_string.seq<EOL>found = seq.find(search_string, pos)<EOL>while found != -<NUM_LIT:1>:<EOL><INDENT>hits.append((found, '<STR_LIT:->'))<EOL>pos = found + <NUM_LIT:1><EOL>found = seq.find(search_string, pos)<EOL><DEDENT>return hits<EOL>

Finds every occurrence (including overlapping ones) of the search_string, including on the reverse strand. Returns a list where each element is a tuple (position, strand) where strand is in ['-', '+']. Positions are zero-based


return Fasta(self.id, '<STR_LIT>'.join([genetic_codes.codes[genetic_code].get(self.seq[x:x+<NUM_LIT:3>].upper(), '<STR_LIT:X>') for x in range(frame, len(self)-<NUM_LIT:1>-frame, <NUM_LIT:3>)]))<EOL>

Returns a Fasta sequence, translated into amino acids. Starts translating from 'frame', where frame expected to be 0,1 or 2


gc_total = <NUM_LIT:0.0><EOL>num_bases = <NUM_LIT:0.0><EOL>n_tuple = tuple('<STR_LIT>')<EOL>accepted_bases = tuple('<STR_LIT>')<EOL>for base, count in Counter(self.seq).items():<EOL><INDENT>if base not in n_tuple:<EOL><INDENT>num_bases += count<EOL>if base in accepted_bases:  <EOL><INDENT>gc_total += count<EOL><DEDENT><DEDENT><DEDENT>gc_content = gc_total / num_bases<EOL>if not as_decimal:  <EOL><INDENT>gc_content *= <NUM_LIT:100><EOL><DEDENT>return gc_content<EOL>

Returns the GC content for the sequence.
        Notes:
            This method ignores N when calculating the length of the sequence.
            It does not, however ignore other ambiguous bases. It also only
            includes the ambiguous base S (G or C). In this sense the method is
            conservative with its calculation.

        Args:
            as_decimal (bool): Return the result as a decimal. Setting to False
            will return as a percentage. i.e for the sequence GCAT it will
            return 0.5 by default and 50.00 if set to False.

        Returns:
            float: GC content calculated as the number of G, C, and S divided
            by the number of (non-N) bases (length).


return Fastq(self.id, self.seq[start:end], self.qual[start:end])<EOL>

Returns Fastq object with the same name, of the bases from start to end, but not including end


super().revcomp()<EOL>self.qual = self.qual[::-<NUM_LIT:1>]<EOL>


super().trim(start, end)<EOL>self.qual = self.qual[start:len(self.qual) - end]<EOL>


<EOL>i = <NUM_LIT:0><EOL>while i < len(self) and self.seq[i] in '<STR_LIT>':<EOL><INDENT>i += <NUM_LIT:1><EOL><DEDENT>self.seq = self.seq[i:]<EOL>self.qual = self.qual[i:]<EOL>self.seq = self.seq.rstrip('<STR_LIT>')<EOL>self.qual = self.qual[:len(self.seq)]<EOL>

Removes any leading or trailing N or n characters from the sequence


def replace_interval(self, start, end, new, qual_string):

if len(new) != len(qual_string):<EOL><INDENT>raise Error('<STR_LIT>')<EOL><DEDENT>super().replace_interval(start, end, new)<EOL>self.qual = self.qual[<NUM_LIT:0>:start] + qual_string + self.qual[end + <NUM_LIT:1>:]<EOL>


fa = super().translate()<EOL>return Fastq(fa.id, fa.seq, '<STR_LIT:I>'*len(fa.seq))<EOL>


f = utils.open_file_write(outfile)<EOL>for seq in sequences.file_reader(infile):<EOL><INDENT>seq.replace_non_acgt()<EOL>print(seq, file=f)<EOL><DEDENT>utils.close(f)<EOL>

Replace every non-acgtn (case insensitve) character with an N


def caf_to_fastq(infile, outfile, min_length=<NUM_LIT:0>, trim=False):

caf_reader = caf.file_reader(infile)<EOL>fout = utils.open_file_write(outfile)<EOL>for c in caf_reader:<EOL><INDENT>if trim:<EOL><INDENT>if c.clip_start is not None and c.clip_end is not None:<EOL><INDENT>c.seq.seq = c.seq.seq[c.clip_start:c.clip_end + <NUM_LIT:1>]<EOL>c.seq.qual = c.seq.qual[c.clip_start:c.clip_end + <NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>', c.id, file=sys.stderr)<EOL><DEDENT><DEDENT>if len(c.seq) >= min_length:<EOL><INDENT>print(c.seq, file=fout)<EOL><DEDENT><DEDENT>utils.close(fout)<EOL>

Convert a CAF file to fastq. Reads shorter than min_length are not output. If clipping information is in the CAF file (with a line Clipping QUAL ...) and trim=True, then trim the reads


seq_reader = sequences.file_reader(infile)<EOL>n = <NUM_LIT:0><EOL>for seq in seq_reader:<EOL><INDENT>n += <NUM_LIT:1><EOL><DEDENT>return n<EOL>

Returns the number of sequences in a file


def interleave(infile_1, infile_2, outfile, suffix1=None, suffix2=None):

seq_reader_1 = sequences.file_reader(infile_1)<EOL>seq_reader_2 = sequences.file_reader(infile_2)<EOL>f_out = utils.open_file_write(outfile)<EOL>for seq_1 in seq_reader_1:<EOL><INDENT>try:<EOL><INDENT>seq_2 = next(seq_reader_2)<EOL><DEDENT>except:<EOL><INDENT>utils.close(f_out)<EOL>raise Error('<STR_LIT>', seq_1.id, '<STR_LIT>')<EOL><DEDENT>if suffix1 is not None and not seq_1.id.endswith(suffix1):<EOL><INDENT>seq_1.id += suffix1<EOL><DEDENT>if suffix2 is not None and not seq_2.id.endswith(suffix2):<EOL><INDENT>seq_2.id += suffix2<EOL><DEDENT>print(seq_1, file=f_out)<EOL>print(seq_2, file=f_out)<EOL><DEDENT>try:<EOL><INDENT>seq_2 = next(seq_reader_2)<EOL><DEDENT>except:<EOL><INDENT>seq_2 = None<EOL><DEDENT>if seq_2 is not None:<EOL><INDENT>utils.close(f_out)<EOL>raise Error('<STR_LIT>', seq_2.id, '<STR_LIT>')<EOL><DEDENT>utils.close(f_out)<EOL>

Makes interleaved file from two sequence files. If used, will append suffix1 onto end
    of every sequence name in infile_1, unless it already ends with suffix1. Similar for sufffix2.


def make_random_contigs(contigs, length, outfile, name_by_letters=False, prefix='<STR_LIT>', seed=None, first_number=<NUM_LIT:1>):

random.seed(a=seed)<EOL>fout = utils.open_file_write(outfile)<EOL>letters = list('<STR_LIT>')<EOL>letters_index = <NUM_LIT:0><EOL>for i in range(contigs):<EOL><INDENT>if name_by_letters:<EOL><INDENT>name = letters[letters_index]<EOL>letters_index += <NUM_LIT:1><EOL>if letters_index == len(letters):<EOL><INDENT>letters_index = <NUM_LIT:0><EOL><DEDENT><DEDENT>else:<EOL><INDENT>name = str(i + first_number)<EOL><DEDENT>fa = sequences.Fasta(prefix + name, '<STR_LIT>'.join([random.choice('<STR_LIT>') for x in range(length)]))<EOL>print(fa, file=fout)<EOL><DEDENT>utils.close(fout)<EOL>

Makes a multi fasta file of random sequences, all the same length


total = <NUM_LIT:0><EOL>count = <NUM_LIT:0><EOL>seq_reader = sequences.file_reader(infile)<EOL>for seq in seq_reader:<EOL><INDENT>total += len(seq)<EOL>count += <NUM_LIT:1><EOL>if limit is not None and count >= limit:<EOL><INDENT>break<EOL><DEDENT><DEDENT>assert count > <NUM_LIT:0><EOL>return total / count<EOL>

Returns the mean length of the sequences in the input file. By default uses all sequences. To limit to the first N sequences, use limit=N


def merge_to_one_seq(infile, outfile, seqname='<STR_LIT>'):

seq_reader = sequences.file_reader(infile)<EOL>seqs = []<EOL>for seq in seq_reader:<EOL><INDENT>seqs.append(copy.copy(seq))<EOL><DEDENT>new_seq = '<STR_LIT>'.join([seq.seq for seq in seqs])<EOL>if type(seqs[<NUM_LIT:0>]) == sequences.Fastq:<EOL><INDENT>new_qual = '<STR_LIT>'.join([seq.qual for seq in seqs])<EOL>seqs[:] = []<EOL>merged = sequences.Fastq(seqname, new_seq, new_qual)<EOL><DEDENT>else:<EOL><INDENT>merged = sequences.Fasta(seqname, new_seq)<EOL>seqs[:] = []<EOL><DEDENT>f = utils.open_file_write(outfile)<EOL>print(merged, file=f)<EOL>utils.close(f)<EOL>

Takes a multi fasta or fastq file and writes a new file that contains just one sequence, with the original sequences catted together, preserving their order


def scaffolds_to_contigs(infile, outfile, number_contigs=False):

seq_reader = sequences.file_reader(infile)<EOL>fout = utils.open_file_write(outfile)<EOL>for seq in seq_reader:<EOL><INDENT>contigs = seq.contig_coords()<EOL>counter = <NUM_LIT:1><EOL>for contig in contigs:<EOL><INDENT>if number_contigs:<EOL><INDENT>name = seq.id + '<STR_LIT:.>' + str(counter)<EOL>counter += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>name = '<STR_LIT:.>'.join([seq.id, str(contig.start + <NUM_LIT:1>), str(contig.end + <NUM_LIT:1>)])<EOL><DEDENT>print(sequences.Fasta(name, seq[contig.start:contig.end+<NUM_LIT:1>]), file=fout)<EOL><DEDENT><DEDENT>utils.close(fout)<EOL>

Makes a file of contigs from scaffolds by splitting at every N.
       Use number_contigs=True to add .1, .2, etc onto end of each
       contig, instead of default to append coordinates.


def sort_by_size(infile, outfile, smallest_first=False):

seqs = {}<EOL>file_to_dict(infile, seqs)<EOL>seqs = list(seqs.values())<EOL>seqs.sort(key=lambda x: len(x), reverse=not smallest_first)<EOL>fout = utils.open_file_write(outfile)<EOL>for seq in seqs:<EOL><INDENT>print(seq, file=fout)<EOL><DEDENT>utils.close(fout)<EOL>

Sorts input sequence file by biggest sequence first, writes sorted output file. Set smallest_first=True to have smallest first


seqs = {}<EOL>file_to_dict(infile, seqs)<EOL>fout = utils.open_file_write(outfile)<EOL>for name in sorted(seqs):<EOL><INDENT>print(seqs[name], file=fout)<EOL><DEDENT>utils.close(fout)<EOL>

Sorts input sequence file by sort -d -k1,1, writes sorted output file.


def to_fastg(infile, outfile, circular=None):

if circular is None:<EOL><INDENT>to_circularise = set()<EOL><DEDENT>elif type(circular) is not set:<EOL><INDENT>f = utils.open_file_read(circular)<EOL>to_circularise = set([x.rstrip() for x in f.readlines()])<EOL>utils.close(f)<EOL><DEDENT>else:<EOL><INDENT>to_circularise = circular<EOL><DEDENT>seq_reader = sequences.file_reader(infile)<EOL>fout = utils.open_file_write(outfile)<EOL>nodes = <NUM_LIT:1><EOL>for seq in seq_reader:<EOL><INDENT>new_id = '<STR_LIT:_>'.join([<EOL>'<STR_LIT>', str(nodes),<EOL>'<STR_LIT>', str(len(seq)),<EOL>'<STR_LIT>', '<STR_LIT:1>',<EOL>'<STR_LIT>', seq.id<EOL>])<EOL>if seq.id in to_circularise:<EOL><INDENT>seq.id = new_id + '<STR_LIT::>' + new_id + '<STR_LIT:;>'<EOL>print(seq, file=fout)<EOL>seq.revcomp()<EOL>seq.id = new_id + "<STR_LIT>" + new_id + "<STR_LIT>"<EOL>print(seq, file=fout)<EOL><DEDENT>else:<EOL><INDENT>seq.id = new_id + '<STR_LIT:;>'<EOL>print(seq, file=fout)<EOL>seq.revcomp()<EOL>seq.id = new_id + "<STR_LIT>"<EOL>print(seq, file=fout)<EOL><DEDENT>nodes += <NUM_LIT:1><EOL><DEDENT>utils.close(fout)<EOL>

Writes a FASTG file in SPAdes format from input file. Currently only whether or not a sequence is circular is supported. Put circular=set of ids, or circular=filename to make those sequences circular in the output. Puts coverage=1 on all contigs


positions = {}<EOL>total_length = <NUM_LIT:0><EOL>f = utils.open_file_read(fai_file)<EOL>for line in f:<EOL><INDENT>try:<EOL><INDENT>(name, length) = line.rstrip().split()[:<NUM_LIT:2>]<EOL>length = int(length)<EOL><DEDENT>except:<EOL><INDENT>raise Error('<STR_LIT>' + fai_file + '<STR_LIT:\n>' + line)<EOL><DEDENT>positions[name] = total_length<EOL>total_length += length<EOL><DEDENT>utils.close(f)<EOL>return positions<EOL>

Returns a dictionary of positions of the start of each sequence, as
       if all the sequences were catted into one sequence.
       eg if file has three sequences, seq1 10bp, seq2 30bp, seq3 20bp, then
       the output would be: {'seq1': 0, 'seq2': 10, 'seq3': 40}


def split_by_base_count(infile, outfiles_prefix, max_bases, max_seqs=None):

seq_reader = sequences.file_reader(infile)<EOL>base_count = <NUM_LIT:0><EOL>file_count = <NUM_LIT:1><EOL>seq_count = <NUM_LIT:0><EOL>fout = None<EOL>if max_seqs is None:<EOL><INDENT>max_seqs = float('<STR_LIT>')<EOL><DEDENT>for seq in seq_reader:<EOL><INDENT>if base_count == <NUM_LIT:0>:<EOL><INDENT>fout = utils.open_file_write(outfiles_prefix + '<STR_LIT:.>' + str(file_count))<EOL>file_count += <NUM_LIT:1><EOL><DEDENT>if base_count + len(seq) > max_bases or seq_count >= max_seqs:<EOL><INDENT>if base_count == <NUM_LIT:0>:<EOL><INDENT>print(seq, file=fout)<EOL>utils.close(fout)<EOL><DEDENT>else:<EOL><INDENT>utils.close(fout)<EOL>fout = utils.open_file_write(outfiles_prefix + '<STR_LIT:.>' + str(file_count))<EOL>print(seq, file=fout)<EOL>base_count = len(seq)<EOL>file_count += <NUM_LIT:1><EOL>seq_count = <NUM_LIT:1><EOL><DEDENT><DEDENT>else:<EOL><INDENT>base_count += len(seq)<EOL>seq_count += <NUM_LIT:1><EOL>print(seq, file=fout)<EOL><DEDENT><DEDENT>utils.close(fout)<EOL>

Splits a fasta/q file into separate files, file size determined by number of bases.

    Puts <= max_bases in each split file The exception is a single sequence >=max_bases
    is put in its own file.  This does not split sequences.


def split_by_fixed_size(infile, outfiles_prefix, chunk_size, tolerance, skip_if_all_Ns=False):

file_count = <NUM_LIT:1><EOL>coords = []<EOL>small_sequences = []  <EOL>seq_reader = sequences.file_reader(infile)<EOL>f_coords = utils.open_file_write(outfiles_prefix + '<STR_LIT>')<EOL>for seq in seq_reader:<EOL><INDENT>if skip_if_all_Ns and seq.is_all_Ns():<EOL><INDENT>continue<EOL><DEDENT>if len(seq) < chunk_size:<EOL><INDENT>small_sequences.append(copy.copy(seq))<EOL><DEDENT>elif len(seq) <= chunk_size + tolerance:<EOL><INDENT>f = utils.open_file_write(outfiles_prefix + '<STR_LIT:.>' + str(file_count))<EOL>print(seq, file=f)<EOL>utils.close(f)<EOL>file_count += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>chunks = [(x,x+chunk_size) for x in range(<NUM_LIT:0>, len(seq), chunk_size)]<EOL>if chunks[-<NUM_LIT:1>][<NUM_LIT:1>] - <NUM_LIT:1> > len(seq):<EOL><INDENT>chunks[-<NUM_LIT:1>] = (chunks[-<NUM_LIT:1>][<NUM_LIT:0>], len(seq))<EOL><DEDENT>if len(chunks) > <NUM_LIT:1> and (chunks[-<NUM_LIT:1>][<NUM_LIT:1>] - chunks[-<NUM_LIT:1>][<NUM_LIT:0>]) <= tolerance:<EOL><INDENT>chunks[-<NUM_LIT:2>] = (chunks[-<NUM_LIT:2>][<NUM_LIT:0>], chunks[-<NUM_LIT:1>][<NUM_LIT:1>])<EOL>chunks.pop()<EOL><DEDENT>offset = <NUM_LIT:0><EOL>for chunk in chunks:<EOL><INDENT>if not(skip_if_all_Ns and seq.is_all_Ns(start=chunk[<NUM_LIT:0>], end=chunk[<NUM_LIT:1>]-<NUM_LIT:1>)):<EOL><INDENT>f = utils.open_file_write(outfiles_prefix + '<STR_LIT:.>' + str(file_count))<EOL>chunk_id = seq.id + '<STR_LIT::>' + str(chunk[<NUM_LIT:0>]+<NUM_LIT:1>) + '<STR_LIT:->' + str(chunk[<NUM_LIT:1>])<EOL>print(sequences.Fasta(chunk_id, seq[chunk[<NUM_LIT:0>]:chunk[<NUM_LIT:1>]]), file=f)<EOL>print(chunk_id, seq.id, offset, sep='<STR_LIT:\t>', file=f_coords)<EOL>utils.close(f)<EOL>file_count += <NUM_LIT:1><EOL><DEDENT>offset += chunk[<NUM_LIT:1>] - chunk[<NUM_LIT:0>]<EOL><DEDENT><DEDENT><DEDENT>if len(small_sequences):<EOL><INDENT>f = utils.open_file_write(outfiles_prefix + '<STR_LIT:.>' + str(file_count))<EOL>file_count += <NUM_LIT:1><EOL>base_count = <NUM_LIT:0><EOL>for seq in small_sequences:<EOL><INDENT>if base_count > <NUM_LIT:0> and base_count + len(seq) > chunk_size + tolerance:<EOL><INDENT>utils.close(f)<EOL>f = utils.open_file_write(outfiles_prefix + '<STR_LIT:.>' + str(file_count))<EOL>file_count += <NUM_LIT:1><EOL>base_count = <NUM_LIT:0><EOL><DEDENT>print(seq, file=f)<EOL>base_count += len(seq)<EOL><DEDENT>utils.close(f)<EOL><DEDENT>

Splits  fasta/q file into separate files, with up to (chunk_size + tolerance) bases in each file


def split_by_fixed_size_onefile(infile, outfile, chunk_size, tolerance, skip_if_all_Ns=False):

seq_reader = sequences.file_reader(infile)<EOL>f_out = utils.open_file_write(outfile)<EOL>for seq in seq_reader:<EOL><INDENT>for i in range(<NUM_LIT:0>, len(seq), chunk_size):<EOL><INDENT>if i + chunk_size + tolerance >= len(seq):<EOL><INDENT>end = len(seq)<EOL><DEDENT>else:<EOL><INDENT>end = i + chunk_size<EOL><DEDENT>subseq = seq.subseq(i, end)<EOL>if not (skip_if_all_Ns and subseq.is_all_Ns()):<EOL><INDENT>subseq.id += '<STR_LIT:.>' + str(i+<NUM_LIT:1>) + '<STR_LIT:_>' + str(end)<EOL>print(subseq, file=f_out)<EOL><DEDENT>if end == len(seq):<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>utils.close(f_out)<EOL>

Splits each sequence in infile into chunks of fixed size, last chunk can be up to
       (chunk_size + tolerance) in length


f = utils.open_file_read(infile)<EOL>try:<EOL><INDENT>lengths = sorted([int(line.split('<STR_LIT:\t>')[<NUM_LIT:1>]) for line in f], reverse=True)<EOL><DEDENT>except:<EOL><INDENT>raise Error('<STR_LIT>' + infile)<EOL><DEDENT>utils.close(f)<EOL>stats = {}<EOL>if len(lengths) > <NUM_LIT:0>:<EOL><INDENT>stats['<STR_LIT>'] = max(lengths)<EOL>stats['<STR_LIT>'] = min(lengths)<EOL>stats['<STR_LIT>'] = sum(lengths)<EOL>stats['<STR_LIT>'] = stats['<STR_LIT>'] / len(lengths)<EOL>stats['<STR_LIT>'] = len(lengths)<EOL>cumulative_length = <NUM_LIT:0><EOL>for length in lengths:<EOL><INDENT>cumulative_length += length<EOL>if cumulative_length >= <NUM_LIT:0.5> * stats['<STR_LIT>']:<EOL><INDENT>stats['<STR_LIT>'] = length<EOL>break<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>stats = {x: <NUM_LIT:0> for x in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>')}<EOL><DEDENT>return stats<EOL>

Returns dictionary of length stats from an fai file. Keys are: longest, shortest, mean, total_length, N50, number


seq_reader = sequences.file_reader(infile)<EOL>f_out = utils.open_file_write(outfile)<EOL>for sequence in seq_reader:<EOL><INDENT>print("<STR_LIT>" + sequence.id, file=f_out)<EOL>print("<STR_LIT>" + sequence.seq, file=f_out)<EOL>print("<STR_LIT:=>", file=f_out)<EOL><DEDENT>utils.close(f_out)<EOL>

Converts input sequence file into a "Boulder-IO format", as used by primer3


if len(l1) == <NUM_LIT:0> or len(l2) == <NUM_LIT:0>:<EOL><INDENT>return []<EOL><DEDENT>out = []<EOL>l2_pos = <NUM_LIT:0><EOL>for l in l1:<EOL><INDENT>while l2_pos < len(l2) and l2[l2_pos].end < l.start:<EOL><INDENT>l2_pos += <NUM_LIT:1><EOL><DEDENT>if l2_pos == len(l2):<EOL><INDENT>break<EOL><DEDENT>while l2_pos < len(l2) and l.intersects(l2[l2_pos]):<EOL><INDENT>out.append(l.intersection(l2[l2_pos]))<EOL>l2_pos += <NUM_LIT:1><EOL><DEDENT>l2_pos = max(<NUM_LIT:0>, l2_pos - <NUM_LIT:1>)<EOL><DEDENT>return out<EOL>

Returns intersection of two lists.  Assumes the lists are sorted by start positions


i = <NUM_LIT:0><EOL>l.sort()<EOL>while i < len(l) - <NUM_LIT:1>:<EOL><INDENT>u = l[i].union(l[i+<NUM_LIT:1>])<EOL>if u is not None:<EOL><INDENT>l[i] = u<EOL>l.pop(i+<NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>i += <NUM_LIT:1><EOL><DEDENT><DEDENT>

Sorts list, merges any overlapping intervals, and also adjacent intervals. e.g.
       [0,1], [1,2] would be merge to [0,.2].


i = <NUM_LIT:0><EOL>l.sort()<EOL>while i < len(l) - <NUM_LIT:1>:<EOL><INDENT>if l[i+<NUM_LIT:1>].contains(l[i]):<EOL><INDENT>l.pop(i)<EOL><DEDENT>elif l[i].contains(l[i+<NUM_LIT:1>]):<EOL><INDENT>l.pop(i+<NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>i += <NUM_LIT:1><EOL><DEDENT><DEDENT>

Sorts list in place, then removes any intervals that are completely
       contained inside another interval


return sum([len(x) for x in l])<EOL>

Returns total length of intervals from a list


if self.start <= p <= self.end:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>return min(abs(self.start - p), abs(self.end - p))<EOL><DEDENT>

Returns the distance from the point to the interval. Zero if the point lies inside the interval.


return self.start <= i.end and i.start <= self.end<EOL>

Returns true iff this interval intersects the interval i


return self.start <= i.start and i.end <= self.end<EOL>

Returns true iff this interval contains the interval i


if self.intersects(i) or self.end + <NUM_LIT:1> == i.start or i.end + <NUM_LIT:1> == self.start:<EOL><INDENT>return Interval(min(self.start, i.start), max(self.end, i.end))<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>

If intervals intersect, returns their union, otherwise returns None


return Interval(min(self.start, i.start), max(self.end, i.end))<EOL>

Like union, but ignores whether the two intervals intersect or not


if self.intersects(i):<EOL><INDENT>return Interval(max(self.start, i.start), min(self.end, i.end))<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>

If intervals intersect, returns their intersection, otherwise returns None


attrs = session.eval('<STR_LIT>' % name, nout=<NUM_LIT:1>).ravel().tolist()<EOL>methods = session.eval('<STR_LIT>' % name, nout=<NUM_LIT:1>).ravel().tolist()<EOL>ref = weakref.ref(session)<EOL>doc = _DocDescriptor(ref, name)<EOL>values = dict(__doc__=doc, _name=name, _ref=ref, _attrs=attrs,<EOL>__module__='<STR_LIT>')<EOL>for method in methods:<EOL><INDENT>doc = _MethodDocDescriptor(ref, name, method)<EOL>cls_name = '<STR_LIT>' % (name, method)<EOL>method_values = dict(__doc__=doc)<EOL>method_cls = type(str(cls_name),<EOL>(OctaveUserClassMethod,), method_values)<EOL>values[method] = method_cls(ref, method, name)<EOL><DEDENT>for attr in attrs:<EOL><INDENT>values[attr] = OctaveUserClassAttr(ref, attr, attr)<EOL><DEDENT>return type(str(name), (OctaveUserClass,), values)<EOL>

Make an Octave class for a given class name


return OctaveVariablePtr(weakref.ref(session), name, name)<EOL>

Make a pointer instance for a given variable by name.


addr = self._address = '<STR_LIT>' % (self._name, id(self))<EOL>self._ref().feval(self._name, *inputs, store_as=addr, **kwargs)<EOL>

Create a new instance with the user class constructor.


instance = OctaveUserClass.__new__(cls)<EOL>instance._address = '<STR_LIT>' % (instance._name, id(instance))<EOL>instance._ref().push(instance._address, value)<EOL>return instance<EOL>

This is how an instance is created when we read a
           MatlabObject from a MAT file.


if not isinstance(instance, OctaveUserClass) or not instance._attrs:<EOL><INDENT>return dict()<EOL><DEDENT>dtype = []<EOL>values = []<EOL>for attr in instance._attrs:<EOL><INDENT>dtype.append((str(attr), object))<EOL>values.append(getattr(instance, attr))<EOL><DEDENT>struct = np.array([tuple(values)], dtype)<EOL>return MatlabObject(struct, instance._name)<EOL>


return OctavePtr(instance._ref, instance._name, instance._address)<EOL>


script = """<STR_LIT>"""<EOL>if not PY2:<EOL><INDENT>script = script.replace('<STR_LIT>', '<STR_LIT:input>')<EOL><DEDENT>for line in script.strip().split('<STR_LIT:\n>'):<EOL><INDENT>line = line.strip()<EOL>if not '<STR_LIT>' in line:<EOL><INDENT>time.sleep(delay)<EOL>print("<STR_LIT>".format(line))<EOL>time.sleep(delay)<EOL><DEDENT>if not interactive:<EOL><INDENT>if '<STR_LIT>' in line or '<STR_LIT>' in line or '<STR_LIT>' in line:<EOL><INDENT>line = '<STR_LIT>'<EOL><DEDENT><DEDENT>exec(line)<EOL><DEDENT>

Play a demo script showing most of the oct2py api features.

Parameters
==========
delay : float
    Time between each command in seconds.


for key, type_ in zip(keys, types):<EOL><INDENT>if not type(base[key]) == type_:<EOL><INDENT>try:<EOL><INDENT>assert type_(base[key]) == base[key], key<EOL><DEDENT>except ValueError:<EOL><INDENT>assert np.allclose(type_(base[key]), base[key])<EOL><DEDENT><DEDENT><DEDENT>

Perform type checking of the values

Parameters
==========
base : dict
    Sub-dictionary we are accessing.
keys : array-like
    List of keys to test in base.
types : array-like
    List of expected return types for the keys.


if isinstance(val1, list):<EOL><INDENT>for (subval1, subval2) in zip(val1, val2):<EOL><INDENT>if isinstance(subval1, list):<EOL><INDENT>self.nested_equal(subval1, subval2)<EOL><DEDENT>elif isinstance(subval1, np.ndarray):<EOL><INDENT>np.allclose(subval1, subval2)<EOL><DEDENT>else:<EOL><INDENT>assert subval1 == subval2<EOL><DEDENT><DEDENT><DEDENT>elif isinstance(val1, np.ndarray):<EOL><INDENT>np.allclose(val1, np.array(val2))<EOL><DEDENT>elif isinstance(val1, (str, unicode)):<EOL><INDENT>assert val1 == val2<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>assert (np.alltrue(np.isnan(val1)) and<EOL>np.alltrue(np.isnan(val2)))<EOL><DEDENT>except (AssertionError, NotImplementedError):<EOL><INDENT>assert val1 == val2<EOL><DEDENT><DEDENT>

Test for equality in a nested list or ndarray


def helper(self, outgoing, expected_type=None):

incoming = self.oc.roundtrip(outgoing)<EOL>if expected_type is None:<EOL><INDENT>expected_type = type(outgoing)<EOL><DEDENT>self.nested_equal(incoming, outgoing)<EOL>try:<EOL><INDENT>assert type(incoming) == expected_type<EOL><DEDENT>except AssertionError:<EOL><INDENT>if type(incoming) == np.float32 and expected_type == np.float64:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>

Use roundtrip.m to make sure the data goes out and back intact.

Parameters
==========
outgoing : object
    Object to send to Octave.


def helper(self, outgoing, incoming=None, expected_type=None):

if incoming is None:<EOL><INDENT>incoming = self.oc.roundtrip(outgoing)<EOL><DEDENT>if not expected_type:<EOL><INDENT>for out_type, _, in_type in TYPE_CONVERSIONS:<EOL><INDENT>if out_type == type(outgoing):<EOL><INDENT>expected_type = in_type<EOL>break<EOL><DEDENT><DEDENT><DEDENT>if not expected_type:<EOL><INDENT>expected_type = np.ndarray<EOL><DEDENT>try:<EOL><INDENT>assert incoming == outgoing<EOL><DEDENT>except ValueError:<EOL><INDENT>assert np.allclose(np.array(incoming), np.array(outgoing))<EOL><DEDENT>if type(incoming) != expected_type:<EOL><INDENT>incoming = self.oc.roundtrip(outgoing)<EOL>assert expected_type(incoming) == incoming<EOL><DEDENT>

Uses roundtrip.m to make sure the data goes out and back intact.

Parameters
==========
outgoing : object
    Object to send to Octave
incoming : object, optional
    Object already retreived from Octave


if not sys.stdin.encoding:<EOL><INDENT>sys.stdin = codecs.getreader('<STR_LIT:utf-8>')(sys.stdin)<EOL><DEDENT>cls.ip = get_ipython()<EOL>cls.ip.magic('<STR_LIT>')<EOL>cls.ip.ex('<STR_LIT>')<EOL>cls.svgs_generated = <NUM_LIT:0><EOL>

Set up an IPython session just once.
        It'd be safer to set it up for each test, but for now,
        I'm mimicking the IPython team's logic.


super(OctaveMagics, self).__init__(shell)<EOL>self._oct = oct2py.octave<EOL>self._display = display<EOL>

Parameters
----------
shell : IPython shell


import os<EOL>if os.name == '<STR_LIT>':<EOL><INDENT>os.system('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>os.system('<STR_LIT>')<EOL>os.system('<STR_LIT>')<EOL><DEDENT>octave.restart()<EOL>

Kill all octave instances (cross-platform).

    This will restart the "octave" instance.  If you have instantiated
    Any other Oct2Py objects, you must restart them.


print("<STR_LIT>".format(nthreads,<EOL>datetime.datetime.now()))<EOL>threads = []<EOL>for i in range(nthreads):<EOL><INDENT>thread = ThreadClass()<EOL>thread.setDaemon(True)<EOL>thread.start()<EOL>threads.append(thread)<EOL><DEDENT>for thread in threads:<EOL><INDENT>thread.join()<EOL><DEDENT>print('<STR_LIT>'.format(datetime.datetime.now()))<EOL>

Start a number of threads and verify each has a unique Octave session.

Parameters
==========
nthreads : int
    Number of threads to use.

Raises
======
Oct2PyError
    If the thread does not sucessfully demonstrate independence.


octave = Oct2Py()<EOL>octave.push('<STR_LIT:name>', self.getName())<EOL>name = octave.pull('<STR_LIT:name>')<EOL>now = datetime.datetime.now()<EOL>print("<STR_LIT>".format(self.getName(), name, now))<EOL>octave.exit()<EOL>try:<EOL><INDENT>assert self.getName() == name<EOL><DEDENT>except AssertionError:  <EOL><INDENT>raise Oct2PyError('<STR_LIT>')<EOL><DEDENT>return<EOL>

Create a unique instance of Octave and verify namespace uniqueness.

Raises
======
Oct2PyError
    If the thread does not sucessfully demonstrate independence


def __init__(self, executable=None, logger=None, timeout=None,<EOL>oned_as='<STR_LIT>', temp_dir=None, convert_to_float=True,<EOL>backend=None):

self._oned_as = oned_as<EOL>self._executable = executable<EOL>self._engine = None<EOL>self._logger = None<EOL>self.logger = logger<EOL>self.timeout = timeout<EOL>self.backend = backend<EOL>self.temp_dir = temp_dir or tempfile.mkdtemp()<EOL>self.convert_to_float = convert_to_float<EOL>self._user_classes = dict()<EOL>self._function_ptrs = dict()<EOL>self.restart()<EOL>atexit.register(self._cleanup)<EOL>


return self._logger<EOL>

The logging instance used by the session.


if not self._engine:<EOL><INDENT>self.restart()<EOL><DEDENT>return self<EOL>

Return octave object, restart session if necessary


def __exit__(self, type, value, traceback):

self.exit()<EOL>


if self._engine:<EOL><INDENT>self._engine.repl.terminate()<EOL><DEDENT>self._engine = None<EOL>

Quits this octave session and cleans up.


def push(self, name, var, timeout=None, verbose=True):

if isinstance(name, (str, unicode)):<EOL><INDENT>name = [name]<EOL>var = [var]<EOL><DEDENT>for (n, v) in zip(name, var):<EOL><INDENT>self.feval('<STR_LIT>', '<STR_LIT>', n, v, nout=<NUM_LIT:0>, timeout=timeout,<EOL>verbose=verbose)<EOL><DEDENT>

Put a variable or variables into the Octave session.

Parameters
----------
name : str or list
    Name of the variable(s).
var : object or list
    The value(s) to pass.
timeout : float
    Time to wait for response from Octave (per line).
**kwargs: Deprecated kwargs, ignored.

Examples
--------
>>> from oct2py import octave
>>> y = [1, 2]
>>> octave.push('y', y)
>>> octave.pull('y')
array([[ 1.,  2.]])
>>> octave.push(['x', 'y'], ['spam', [1, 2, 3, 4]])
>>> octave.pull(['x', 'y'])  # doctest: +SKIP
[u'spam', array([[1, 2, 3, 4]])]

Notes
-----
Integer type arguments will be converted to floating point
unless `convert_to_float=False`.


def pull(self, var, timeout=None, verbose=True):

if isinstance(var, (str, unicode)):<EOL><INDENT>var = [var]<EOL><DEDENT>outputs = []<EOL>for name in var:<EOL><INDENT>exist = self._exist(name)<EOL>if exist == <NUM_LIT:1>:<EOL><INDENT>outputs.append(self.feval('<STR_LIT>', '<STR_LIT>', name,<EOL>timeout=timeout, verbose=verbose))<EOL><DEDENT>else:<EOL><INDENT>outputs.append(self.get_pointer(name, timeout=timeout))<EOL><DEDENT><DEDENT>if len(outputs) == <NUM_LIT:1>:<EOL><INDENT>return outputs[<NUM_LIT:0>]<EOL><DEDENT>return outputs<EOL>

Retrieve a value or values from the Octave session.

Parameters
----------
var : str or list
    Name of the variable(s) to retrieve.
timeout : float, optional.
    Time to wait for response from Octave (per line).
**kwargs: Deprecated kwargs, ignored.

Returns
-------
out : object
    Object returned by Octave.

Raises
------
Oct2PyError
    If the variable does not exist in the Octave session.

Examples
--------
  >>> from oct2py import octave
  >>> y = [1, 2]
  >>> octave.push('y', y)
  >>> octave.pull('y')
  array([[ 1.,  2.]])
  >>> octave.push(['x', 'y'], ['spam', [1, 2, 3, 4]])
  >>> octave.pull(['x', 'y'])  # doctest: +SKIP
  [u'spam', array([[1, 2, 3, 4]])]


def get_pointer(self, name, timeout=None):

exist = self._exist(name)<EOL>isobject = self._isobject(name, exist)<EOL>if exist == <NUM_LIT:0>:<EOL><INDENT>raise Oct2PyError('<STR_LIT>' % name)<EOL><DEDENT>elif exist == <NUM_LIT:1>:<EOL><INDENT>return _make_variable_ptr_instance(self, name)<EOL><DEDENT>elif isobject:<EOL><INDENT>return self._get_user_class(name)<EOL><DEDENT>elif exist in [<NUM_LIT:2>, <NUM_LIT:3>, <NUM_LIT:5>]:<EOL><INDENT>return self._get_function_ptr(name)<EOL><DEDENT>raise Oct2PyError('<STR_LIT>' % name)<EOL>

Get a pointer to a named object in the Octave workspace.

        Parameters
        ----------
        name: str
            The name of the object in the Octave workspace.
        timemout: float, optional.
            Time to wait for response from Octave (per line).

        Examples
        --------
        >>> from oct2py import octave
        >>> octave.eval('foo = [1, 2];')
        >>> ptr = octave.get_pointer('foo')
        >>> ptr.value
        array([[ 1.,  2.]])
        >>> ptr.address
        'foo'
        >>> # Can be passed as an argument
        >>> octave.disp(ptr)  # doctest: +SKIP
        1  2

        >>> from oct2py import octave
        >>> sin = octave.get_pointer('sin')  # equivalent to `octave.sin`
        >>> sin.address
        '@sin'
        >>> x = octave.quad(sin, 0, octave.pi())
        >>> x
        2.0

        Notes
        -----
        Pointers can be passed to `feval` or dynamic functions as function arguments.  A pointer passed as a nested value will be passed by value instead.

        Raises
        ------
        Oct2PyError
            If the variable does not exist in the Octave session or is of
            unknown type.

        Returns
        -------
        A variable, object, user class, or function pointer as appropriate.


def extract_figures(self, plot_dir, remove=False):

figures = self._engine.extract_figures(plot_dir, remove)<EOL>return figures<EOL>

Extract the figures in the directory to IPython display objects.

        Parameters
        ----------
        plot_dir: str
            The plot dir where the figures were created.
        remove: bool, optional.
            Whether to remove the plot directory after saving.


def feval(self, func_path, *func_args, **kwargs):

if not self._engine:<EOL><INDENT>raise Oct2PyError('<STR_LIT>')<EOL><DEDENT>nout = kwargs.get('<STR_LIT>', None)<EOL>if nout is None:<EOL><INDENT>nout = <NUM_LIT:1><EOL><DEDENT>plot_dir = kwargs.get('<STR_LIT>')<EOL>settings = dict(backend='<STR_LIT>' if plot_dir else self.backend,<EOL>format=kwargs.get('<STR_LIT>'),<EOL>name=kwargs.get('<STR_LIT>'),<EOL>width=kwargs.get('<STR_LIT>'),<EOL>height=kwargs.get('<STR_LIT>'),<EOL>resolution=kwargs.get('<STR_LIT>'))<EOL>self._engine.plot_settings = settings<EOL>dname = osp.dirname(func_path)<EOL>fname = osp.basename(func_path)<EOL>func_name, ext = osp.splitext(fname)<EOL>if ext and not ext == '<STR_LIT>':<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>if func_name == '<STR_LIT>':<EOL><INDENT>raise Oct2PyError('<STR_LIT>' +<EOL>'<STR_LIT>')<EOL><DEDENT>stream_handler = kwargs.get('<STR_LIT>')<EOL>verbose = kwargs.get('<STR_LIT>', True)<EOL>store_as = kwargs.get('<STR_LIT>', '<STR_LIT>')<EOL>timeout = kwargs.get('<STR_LIT>', self.timeout)<EOL>if not stream_handler:<EOL><INDENT>stream_handler = self.logger.info if verbose else self.logger.debug<EOL><DEDENT>return self._feval(func_name, func_args, dname=dname, nout=nout,<EOL>timeout=timeout, stream_handler=stream_handler,<EOL>store_as=store_as, plot_dir=plot_dir)<EOL>

Run a function in Octave and return the result.

        Parameters
        ----------
        func_path: str
            Name of function to run or a path to an m-file.
        func_args: object, optional
            Args to send to the function.
        nout: int, optional
            Desired number of return arguments, defaults to 1.
        store_as: str, optional
            If given, saves the result to the given Octave variable name
            instead of returning it.
        verbose : bool, optional
            Log Octave output at INFO level.  If False, log at DEBUG level.
        stream_handler: callable, optional
            A function that is called for each line of output from the
            evaluation.
        timeout: float, optional
            The timeout in seconds for the call.
        plot_dir: str, optional
            If specificed, save the session's plot figures to the plot
            directory instead of displaying the plot window.
        plot_name : str, optional
            Saved plots will start with `plot_name` and
            end with "_%%.xxx' where %% is the plot number and
            xxx is the `plot_format`.
        plot_format: str, optional
            The format in which to save the plot.
        plot_width: int, optional
            The plot with in pixels.
        plot_height: int, optional
            The plot height in pixels.

        Notes
        -----
        The function arguments passed follow Octave calling convention, not
        Python. That is, all values must be passed as a comma separated list,
        not using `x=foo` assignment.

        Examples
        --------
        >>> from oct2py import octave
        >>> cell = octave.feval('cell', 10, 10, 10)
        >>> cell.shape
        (10, 10, 10)

        >>> from oct2py import octave
        >>> x = octave.feval('linspace', 0, octave.pi() / 2)
        >>> x.shape
        (1, 100)

        >>> from oct2py import octave
        >>> x = octave.feval('svd', octave.hilb(3))
        >>> x
        array([[ 1.40831893],
               [ 0.12232707],
               [ 0.00268734]])
        >>> # specify three return values
        >>> (u, v, d) = octave.feval('svd', octave.hilb(3), nout=3)
        >>> u.shape
        (3, 3)

        Returns
        -------
        The Python value(s) returned by the Octave function call.


def eval(self, cmds, verbose=True, timeout=None, stream_handler=None,<EOL>temp_dir=None, plot_dir=None, plot_name='<STR_LIT>', plot_format='<STR_LIT>',<EOL>plot_width=None, plot_height=None, plot_res=None,<EOL>nout=<NUM_LIT:0>, **kwargs):

if isinstance(cmds, (str, unicode)):<EOL><INDENT>cmds = [cmds]<EOL><DEDENT>prev_temp_dir = self.temp_dir<EOL>self.temp_dir = temp_dir or self.temp_dir<EOL>prev_log_level = self.logger.level<EOL>if kwargs.get('<STR_LIT>') is False:<EOL><INDENT>self.logger.setLevel(logging.WARN)<EOL><DEDENT>for name in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>if name not in kwargs:<EOL><INDENT>continue<EOL><DEDENT>msg = '<STR_LIT>'<EOL>warnings.warn(msg % name, stacklevel=<NUM_LIT:2>)<EOL><DEDENT>return_both = kwargs.pop('<STR_LIT>', False)<EOL>lines = []<EOL>if return_both and not stream_handler:<EOL><INDENT>stream_handler = lines.append<EOL><DEDENT>ans = None<EOL>for cmd in cmds:<EOL><INDENT>resp = self.feval('<STR_LIT>', '<STR_LIT>', cmd,<EOL>nout=nout, timeout=timeout,<EOL>stream_handler=stream_handler,<EOL>verbose=verbose, plot_dir=plot_dir,<EOL>plot_name=plot_name, plot_format=plot_format,<EOL>plot_width=plot_width, plot_height=plot_height,<EOL>plot_res=plot_res)<EOL>if resp is not None:<EOL><INDENT>ans = resp<EOL><DEDENT><DEDENT>self.temp_dir = prev_temp_dir<EOL>self.logger.setLevel(prev_log_level)<EOL>if return_both:<EOL><INDENT>return '<STR_LIT:\n>'.join(lines), ans<EOL><DEDENT>return ans<EOL>

Evaluate an Octave command or commands.

Parameters
----------
cmds : str or list
    Commands(s) to pass to Octave.
verbose : bool, optional
     Log Octave output at INFO level.  If False, log at DEBUG level.
stream_handler: callable, optional
    A function that is called for each line of output from the
    evaluation.
timeout : float, optional
    Time to wait for response from Octave (per line).  If not given,
    the instance `timeout` is used.
nout : int, optional.
    The desired number of returned values, defaults to 0.  If nout
    is 0, the `ans` will be returned as the return value.
temp_dir: str, optional
    If specified, the session's MAT files will be created in the
    directory, otherwise a the instance `temp_dir` is used.
    a shared memory (tmpfs) path.
plot_dir: str, optional
    If specificed, save the session's plot figures to the plot
    directory instead of displaying the plot window.
plot_name : str, optional
    Saved plots will start with `plot_name` and
    end with "_%%.xxx' where %% is the plot number and
    xxx is the `plot_format`.
plot_format: str, optional
    The format in which to save the plot (PNG by default).
plot_width: int, optional
    The plot with in pixels.
plot_height: int, optional
    The plot height in pixels.
plot_res: int, optional
    The plot resolution in pixels per inch.
**kwargs Deprectated kwargs.

Examples
--------
>>> from oct2py import octave
>>> octave.eval('disp("hello")') # doctest: +SKIP
hello
>>> x = octave.eval('round(quad(@sin, 0, pi/2));')
>>> x
1.0

>>> a = octave.eval('disp("hello");1;')  # doctest: +SKIP
hello
>>> a = octave.eval('disp("hello");1;', verbose=False)
>>> a
1.0

>>> from oct2py import octave
>>> lines = []
>>> octave.eval('for i = 1:3; disp(i);end', \
                stream_handler=lines.append)
>>> lines  # doctest: +SKIP
[' 1', ' 2', ' 3']

Returns
-------
out : object
    Octave "ans" variable, or None.

Notes
-----
The deprecated `log` kwarg will temporarily set the `logger` level to
`WARN`.  Using the `logger` settings directly is preferred.
The deprecated `return_both` kwarg will still work, but the preferred
method is to use the `stream_handler`.  If `stream_handler` is given,
the `return_both` kwarg will be honored but will give an empty string
as the reponse.

Raises
------
Oct2PyError
    If the command(s) fail.


if self._engine:<EOL><INDENT>self._engine.repl.terminate()<EOL><DEDENT>executable = self._executable<EOL>if executable:<EOL><INDENT>os.environ['<STR_LIT>'] = executable<EOL><DEDENT>if '<STR_LIT>' not in os.environ and '<STR_LIT>' in os.environ:<EOL><INDENT>os.environ['<STR_LIT>'] = os.environ['<STR_LIT>']<EOL><DEDENT>self._engine = OctaveEngine(stdin_handler=self._handle_stdin,<EOL>logger=self.logger)<EOL>self._engine.eval('<STR_LIT>' % HERE.replace(osp.sep, '<STR_LIT:/>'))<EOL>

Restart an Octave session in a clean state


def _feval(self, func_name, func_args=(), dname='<STR_LIT>', nout=<NUM_LIT:0>,<EOL>timeout=None, stream_handler=None, store_as='<STR_LIT>', plot_dir=None):

engine = self._engine<EOL>if engine is None:<EOL><INDENT>raise Oct2PyError('<STR_LIT>')<EOL><DEDENT>out_file = osp.join(self.temp_dir, '<STR_LIT>')<EOL>out_file = out_file.replace(osp.sep, '<STR_LIT:/>')<EOL>in_file = osp.join(self.temp_dir, '<STR_LIT>')<EOL>in_file = in_file.replace(osp.sep, '<STR_LIT:/>')<EOL>func_args = list(func_args)<EOL>ref_indices = []<EOL>for (i, value) in enumerate(func_args):<EOL><INDENT>if isinstance(value, OctavePtr):<EOL><INDENT>ref_indices.append(i + <NUM_LIT:1>)<EOL>func_args[i] = value.address<EOL><DEDENT><DEDENT>ref_indices = np.array(ref_indices)<EOL>req = dict(func_name=func_name, func_args=tuple(func_args),<EOL>dname=dname or '<STR_LIT>', nout=nout,<EOL>store_as=store_as or '<STR_LIT>',<EOL>ref_indices=ref_indices)<EOL>write_file(req, out_file, oned_as=self._oned_as,<EOL>convert_to_float=self.convert_to_float)<EOL>engine.stream_handler = stream_handler or self.logger.info<EOL>if timeout is None:<EOL><INDENT>timeout = self.timeout<EOL><DEDENT>try:<EOL><INDENT>engine.eval('<STR_LIT>' % (out_file, in_file),<EOL>timeout=timeout)<EOL><DEDENT>except KeyboardInterrupt as e:<EOL><INDENT>stream_handler(engine.repl.interrupt())<EOL>raise<EOL><DEDENT>except TIMEOUT:<EOL><INDENT>stream_handler(engine.repl.interrupt())<EOL>raise Oct2PyError('<STR_LIT>')<EOL><DEDENT>except EOF:<EOL><INDENT>stream_handler(engine.repl.child.before)<EOL>self.restart()<EOL>raise Oct2PyError('<STR_LIT>')<EOL><DEDENT>resp = read_file(in_file, self)<EOL>if resp['<STR_LIT>']:<EOL><INDENT>msg = self._parse_error(resp['<STR_LIT>'])<EOL>raise Oct2PyError(msg)<EOL><DEDENT>result = resp['<STR_LIT:result>'].ravel().tolist()<EOL>if isinstance(result, list) and len(result) == <NUM_LIT:1>:<EOL><INDENT>result = result[<NUM_LIT:0>]<EOL><DEDENT>if (isinstance(result, Cell) and<EOL>result.size == <NUM_LIT:1> and<EOL>isinstance(result[<NUM_LIT:0>], string_types) and<EOL>result[<NUM_LIT:0>] == '<STR_LIT>'):<EOL><INDENT>result = None<EOL><DEDENT>if plot_dir:<EOL><INDENT>self._engine.make_figures(plot_dir)<EOL><DEDENT>return result<EOL>

Run the given function with the given args.


self.logger.debug(err)<EOL>stack = err.get('<STR_LIT>', [])<EOL>if not err['<STR_LIT:message>'].startswith('<STR_LIT>'):<EOL><INDENT>err['<STR_LIT:message>'] = '<STR_LIT>' + err['<STR_LIT:message>']<EOL><DEDENT>errmsg = '<STR_LIT>' % err['<STR_LIT:message>']<EOL>if not isinstance(stack, StructArray):<EOL><INDENT>return errmsg<EOL><DEDENT>errmsg += '<STR_LIT>'<EOL>for item in stack[:-<NUM_LIT:1>]:<EOL><INDENT>errmsg += '<STR_LIT>' % item<EOL>try:<EOL><INDENT>errmsg += '<STR_LIT>' % item<EOL><DEDENT>except Exception:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return errmsg<EOL>

Create a traceback for an Octave evaluation error.


return input(line.replace(STDIN_PROMPT, '<STR_LIT>'))<EOL>

Handle a stdin request from the session.


doc = '<STR_LIT>' % name<EOL>engine = self._engine<EOL>doc = engine.eval('<STR_LIT>' % name, silent=True)<EOL>if '<STR_LIT>' in doc.lower():<EOL><INDENT>raise Oct2PyError(doc)<EOL><DEDENT>if '<STR_LIT>' in doc.lower():<EOL><INDENT>doc = engine.eval('<STR_LIT>' % name, silent=True)<EOL>doc = '<STR_LIT:\n>'.join(doc.splitlines()[:<NUM_LIT:3>])<EOL><DEDENT>default = self.feval.__doc__<EOL>default = '<STR_LIT:U+0020>' + default[default.find('<STR_LIT>'):]<EOL>default = '<STR_LIT:\n>'.join([line[<NUM_LIT:8>:] for line in default.splitlines()])<EOL>doc = '<STR_LIT:\n>'.join(doc.splitlines())<EOL>doc = '<STR_LIT:\n>' + doc + '<STR_LIT>' + default<EOL>doc += '<STR_LIT>'<EOL>doc += '<STR_LIT>'<EOL>doc += '<STR_LIT>'<EOL>doc += '<STR_LIT>'<EOL>doc += '<STR_LIT>'<EOL>doc += '<STR_LIT>'<EOL>doc += '<STR_LIT>'<EOL>return doc<EOL>

Get the documentation of an Octave procedure or object.

Parameters
----------
name : str
    Function name to search for.

Returns
-------
out : str
  Documentation string.

Raises
------
Oct2PyError
   If the procedure or object function has a syntax error.


cmd = '<STR_LIT>' % name<EOL>resp = self._engine.eval(cmd, silent=True).strip()<EOL>exist = int(resp.split()[-<NUM_LIT:1>])<EOL>if exist == <NUM_LIT:0>:<EOL><INDENT>msg = '<STR_LIT>'<EOL>raise Oct2PyError(msg % name)<EOL><DEDENT>return exist<EOL>

Test whether a name exists and return the name code.

        Raises an error when the name does not exist.


if exist in [<NUM_LIT:2>, <NUM_LIT:5>]:<EOL><INDENT>return False<EOL><DEDENT>cmd = '<STR_LIT>' % name<EOL>resp = self._engine.eval(cmd, silent=True).strip()<EOL>return resp == '<STR_LIT>'<EOL>


func = _make_function_ptr_instance<EOL>self._function_ptrs.setdefault(name, func(self, name))<EOL>return self._function_ptrs[name]<EOL>

Get or create a function pointer of the given name.


self._user_classes.setdefault(name, _make_user_class(self, name))<EOL>return self._user_classes[name]<EOL>

Get or create a user class of the given type.


self.exit()<EOL>workspace = osp.join(os.getcwd(), '<STR_LIT>')<EOL>if osp.exists(workspace):<EOL><INDENT>os.remove(workspace)<EOL><DEDENT>


<EOL>if attr.startswith('<STR_LIT>'):<EOL><INDENT>return super(Oct2Py, self).__getattr__(attr)<EOL><DEDENT>if attr[-<NUM_LIT:1>] == "<STR_LIT:_>":<EOL><INDENT>name = attr[:-<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>name = attr<EOL><DEDENT>if self._engine is None:<EOL><INDENT>raise Oct2PyError('<STR_LIT>')<EOL><DEDENT>exist = self._exist(name)<EOL>if exist not in [<NUM_LIT:2>, <NUM_LIT:3>, <NUM_LIT:5>, <NUM_LIT>]:<EOL><INDENT>msg = '<STR_LIT>'<EOL>raise Oct2PyError(msg % name)<EOL><DEDENT>if name == '<STR_LIT>':<EOL><INDENT>raise Oct2PyError('<STR_LIT>' +<EOL>'<STR_LIT>')<EOL><DEDENT>if self._isobject(name, exist):<EOL><INDENT>obj = self._get_user_class(name)<EOL><DEDENT>else:<EOL><INDENT>obj = self._get_function_ptr(name)<EOL><DEDENT>setattr(self, attr, obj)<EOL>return obj<EOL>

Automatically creates a wapper to an Octave function or object.

        Adapted from the mlabwrap project.


if name is None:<EOL><INDENT>name = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>name = '<STR_LIT>' + name<EOL><DEDENT>log = logging.getLogger(name)<EOL>log.setLevel(logging.INFO)<EOL>return log<EOL>

Return a console logger.

    Output may be sent to the logger using the `debug`, `info`, `warning`,
    `error` and `critical` methods.

    Parameters
    ----------
    name : str
        Name of the log.

    References
    ----------
    .. [1] Logging facility for Python,
           http://docs.python.org/library/logging.html


try:<EOL><INDENT>handler = logging.StreamHandler(stream=sys.stdout)<EOL><DEDENT>except TypeError:  <EOL><INDENT>handler = logging.StreamHandler(strm=sys.stdout)<EOL><DEDENT>log = get_log()<EOL>log.addHandler(handler)<EOL>log.setLevel(logging.INFO)<EOL>log.propagate = False<EOL>


test = SpeedCheck()<EOL>test.run()<EOL>

Checks the speed penalty of the Python to Octave bridge.

    Uses timeit to test the raw execution of a Octave command,
    Then tests progressively larger array passing.


self.octave = Oct2Py()<EOL>self.array = []<EOL>

Create our Octave instance and initialize the data array


Run a fast Octave command and see how long it takes.


self.octave.push('<STR_LIT:x>', self.array)<EOL>

Create a large matrix and load it into the octave session.


Retrieve the large matrix from the octave session


print('<STR_LIT>')<EOL>print('<STR_LIT:*>' * <NUM_LIT:20>)<EOL>time.sleep(<NUM_LIT:1>)<EOL>print('<STR_LIT>')<EOL>avg = timeit.timeit(self.raw_speed, number=<NUM_LIT:10>) / <NUM_LIT:10><EOL>print('<STR_LIT>'.format(avg * <NUM_LIT>))<EOL>sides = [<NUM_LIT:1>, <NUM_LIT:10>, <NUM_LIT:100>, <NUM_LIT:1000>]<EOL>runs = [<NUM_LIT:10>, <NUM_LIT:10>, <NUM_LIT:10>, <NUM_LIT:5>]<EOL>for (side, nruns) in zip(sides, runs):<EOL><INDENT>self.array = np.reshape(np.arange(side ** <NUM_LIT:2>), (-<NUM_LIT:1>))<EOL>print('<STR_LIT>'.format(side, side))<EOL>avg = timeit.timeit(self.large_array_put, number=nruns) / nruns<EOL>print('<STR_LIT>'.format(avg * <NUM_LIT>))<EOL>print('<STR_LIT>'.format(side, side))<EOL>avg = timeit.timeit(self.large_array_get, number=nruns) / nruns<EOL>print('<STR_LIT>'.format(avg * <NUM_LIT>))<EOL><DEDENT>self.octave.exit()<EOL>print('<STR_LIT:*>' * <NUM_LIT:20>)<EOL>print('<STR_LIT>')<EOL>

Perform the Oct2Py speed analysis.

        Uses timeit to test the raw execution of an Octave command,
        Then tests progressively larger array passing.


try:<EOL><INDENT>data = loadmat(path, struct_as_record=True)<EOL><DEDENT>except UnicodeDecodeError as e:<EOL><INDENT>raise Oct2PyError(str(e))<EOL><DEDENT>out = dict()<EOL>for (key, value) in data.items():<EOL><INDENT>out[key] = _extract(value, session)<EOL><DEDENT>return out<EOL>


def write_file(obj, path, oned_as='<STR_LIT>', convert_to_float=True):

data = _encode(obj, convert_to_float)<EOL>try:<EOL><INDENT>with _WRITE_LOCK:<EOL><INDENT>savemat(path, data, appendmat=False, oned_as=oned_as,<EOL>long_field_names=True)<EOL><DEDENT><DEDENT>except KeyError:  <EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT>

Save a Python object to an Octave file on the given path.


<EOL>if isinstance(data, list):<EOL><INDENT>return [_extract(d, session) for d in data]<EOL><DEDENT>if not isinstance(data, np.ndarray):<EOL><INDENT>return data<EOL><DEDENT>if isinstance(data, MatlabObject):<EOL><INDENT>cls = session._get_user_class(data.classname)<EOL>return cls.from_value(data)<EOL><DEDENT>if data.dtype.names:<EOL><INDENT>if data.size == <NUM_LIT:1>:<EOL><INDENT>return _create_struct(data, session)<EOL><DEDENT>return StructArray(data, session)<EOL><DEDENT>if data.dtype.kind == '<STR_LIT:O>':<EOL><INDENT>return Cell(data, session)<EOL><DEDENT>if data.size == <NUM_LIT:1>:<EOL><INDENT>return data.item()<EOL><DEDENT>if data.size == <NUM_LIT:0>:<EOL><INDENT>if data.dtype.kind in '<STR_LIT>':<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>return []<EOL><DEDENT>return data<EOL>

Convert the Octave values to values suitable for Python.


out = Struct()<EOL>for name in data.dtype.names:<EOL><INDENT>item = data[name]<EOL>if isinstance(item, np.ndarray) and item.dtype.kind == '<STR_LIT:O>':<EOL><INDENT>item = item.squeeze().tolist()<EOL><DEDENT>out[name] = _extract(item, session)<EOL><DEDENT>return out<EOL>


ctf = convert_to_float<EOL>if isinstance(data, (OctaveVariablePtr)):<EOL><INDENT>return _encode(data.value, ctf)<EOL><DEDENT>if isinstance(data, OctaveUserClass):<EOL><INDENT>return _encode(OctaveUserClass.to_value(data), ctf)<EOL><DEDENT>if isinstance(data, (OctaveFunctionPtr, MatlabFunction)):<EOL><INDENT>raise Oct2PyError('<STR_LIT>')<EOL><DEDENT>if isinstance(data, MatlabObject):<EOL><INDENT>view = data.view(np.ndarray)<EOL>out = MatlabObject(data, data.classname)<EOL>for name in out.dtype.names:<EOL><INDENT>out[name] = _encode(view[name], ctf)<EOL><DEDENT>return out<EOL><DEDENT>if isinstance(data, (DataFrame, Series)):<EOL><INDENT>return _encode(data.values, ctf)<EOL><DEDENT>if isinstance(data, dict):<EOL><INDENT>out = dict()<EOL>for (key, value) in data.items():<EOL><INDENT>out[key] = _encode(value, ctf)<EOL><DEDENT>return out<EOL><DEDENT>if data is None:<EOL><INDENT>return np.NaN<EOL><DEDENT>if isinstance(data, set):<EOL><INDENT>return _encode(list(data), ctf)<EOL><DEDENT>if isinstance(data, list):<EOL><INDENT>if _is_simple_numeric(data):<EOL><INDENT>return _encode(np.array(data), ctf)<EOL><DEDENT>return _encode(tuple(data), ctf)<EOL><DEDENT>if isinstance(data, tuple):<EOL><INDENT>obj = np.empty(len(data), dtype=object)<EOL>for (i, item) in enumerate(data):<EOL><INDENT>obj[i] = _encode(item, ctf)<EOL><DEDENT>return obj<EOL><DEDENT>if isinstance(data, spmatrix):<EOL><INDENT>return data.astype(np.float64)<EOL><DEDENT>if not isinstance(data, np.ndarray):<EOL><INDENT>return data<EOL><DEDENT>if data.dtype.kind in '<STR_LIT>':<EOL><INDENT>out = np.empty(data.size, dtype=data.dtype)<EOL>for (i, item) in enumerate(data.ravel()):<EOL><INDENT>if data.dtype.names:<EOL><INDENT>for name in data.dtype.names:<EOL><INDENT>out[i][name] = _encode(item[name], ctf)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>out[i] = _encode(item, ctf)<EOL><DEDENT><DEDENT>return out.reshape(data.shape)<EOL><DEDENT>if data.dtype.name == '<STR_LIT>':<EOL><INDENT>return data.astype(np.complex128)<EOL><DEDENT>if ctf and data.dtype.kind in '<STR_LIT>':<EOL><INDENT>return data.astype(np.float64)<EOL><DEDENT>return data<EOL>

Convert the Python values to values suitable to send to Octave.


for item in data:<EOL><INDENT>if isinstance(item, set):<EOL><INDENT>item = list(item)<EOL><DEDENT>if isinstance(item, list):<EOL><INDENT>if not _is_simple_numeric(item):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>elif not isinstance(item, (int, float, complex)):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>

Test if a list contains simple numeric data.


value = np.asarray(value)<EOL>if (value.shape[value.ndim - <NUM_LIT:1>] == <NUM_LIT:1>):<EOL><INDENT>value = value.squeeze(axis=value.ndim - <NUM_LIT:1>)<EOL><DEDENT>value = np.atleast_1d(value)<EOL>if not session:<EOL><INDENT>return value.view(cls)<EOL><DEDENT>obj = np.empty(value.size, dtype=value.dtype).view(cls)<EOL>for (i, item) in enumerate(value.ravel()):<EOL><INDENT>for name in value.dtype.names:<EOL><INDENT>obj[i][name] = _extract(item[name], session)<EOL><DEDENT><DEDENT>return obj.reshape(value.shape)<EOL>

Create a struct array from a value and optional Octave session.


return self.dtype.names<EOL>


attr = np.recarray.__getattribute__(self, attr)<EOL>if isinstance(attr, np.ndarray) and attr.dtype.kind == '<STR_LIT:O>':<EOL><INDENT>return Cell(attr)<EOL><DEDENT>return attr<EOL>

Return object arrays as cells and all other values unchanged.


item = np.recarray.__getitem__(self, item)<EOL>if isinstance(item, np.ndarray) and item.dtype.kind == '<STR_LIT:O>':<EOL><INDENT>return Cell(item)<EOL><DEDENT>return item<EOL>


value = np.asarray(value, dtype=object)<EOL>if (value.shape[value.ndim - <NUM_LIT:1>] == <NUM_LIT:1>):<EOL><INDENT>value = value.squeeze(axis=value.ndim - <NUM_LIT:1>)<EOL><DEDENT>value = np.atleast_1d(value)<EOL>if not session:<EOL><INDENT>return value.view(cls)<EOL><DEDENT>obj = np.empty(value.size, dtype=object).view(cls)<EOL>for (i, item) in enumerate(value.ravel()):<EOL><INDENT>obj[i] = _extract(item, session)<EOL><DEDENT>return obj.reshape(value.shape)<EOL>

Create a cell array from a value and optional Octave session.


if "<STR_LIT:user>" in context:<EOL><INDENT>user = context["<STR_LIT:user>"]<EOL>if user.is_authenticated():<EOL><INDENT>qs = Inbox.objects.select_related("<STR_LIT:message>").filter(user=user)<EOL>return {<EOL>"<STR_LIT>": qs[:num_elements],<EOL>"<STR_LIT:count>": qs.count(),<EOL>}<EOL><DEDENT><DEDENT>

Renders a list of unread stored messages for the current user


if "<STR_LIT:user>" in context:<EOL><INDENT>user = context["<STR_LIT:user>"]<EOL>if user.is_authenticated():<EOL><INDENT>return Inbox.objects.select_related("<STR_LIT:message>").filter(user=user).count()<EOL><DEDENT><DEDENT>


if "<STR_LIT:user>" in context:<EOL><INDENT>user = context["<STR_LIT:user>"]<EOL>if user.is_authenticated():<EOL><INDENT>qs = MessageArchive.objects.select_related("<STR_LIT:message>").filter(user=user)<EOL>return {<EOL>"<STR_LIT>": qs[:num_elements],<EOL>"<STR_LIT:count>": qs.count(),<EOL>}<EOL><DEDENT><DEDENT>

Renders a list of archived messages for the current user


from .settings import stored_messages_settings<EOL>backend = stored_messages_settings.STORAGE_BACKEND()<EOL>backend.inbox_purge(request.user)<EOL>return Response({"<STR_LIT:message>": "<STR_LIT>"})<EOL>

Mark all messages as read (i.e. delete from inbox) for current logged in user


@detail_route(methods=['<STR_LIT:POST>'])<EOL><INDENT>def read(self, request, pk=None):<DEDENT>

from .settings import stored_messages_settings<EOL>backend = stored_messages_settings.STORAGE_BACKEND()<EOL>try:<EOL><INDENT>backend.inbox_delete(request.user, pk)<EOL><DEDENT>except MessageDoesNotExist as e:<EOL><INDENT>return Response(e.message, status='<STR_LIT>')<EOL><DEDENT>return Response({'<STR_LIT:status>': '<STR_LIT>'})<EOL>

Mark the message as read (i.e. delete from inbox)


messages, all_retrieved = super(StorageMixin, self)._get(*args, **kwargs)<EOL>if self.user.is_authenticated():<EOL><INDENT>inbox_messages = self.backend.inbox_list(self.user)<EOL><DEDENT>else:<EOL><INDENT>inbox_messages = []<EOL><DEDENT>return messages + inbox_messages, all_retrieved<EOL>

Retrieve unread messages for current user, both from the inbox and
from other storages


def add(self, level, message, extra_tags='<STR_LIT>'):

if not message:<EOL><INDENT>return<EOL><DEDENT>level = int(level)<EOL>if level < self.level:<EOL><INDENT>return<EOL><DEDENT>if level not in stored_messages_settings.STORE_LEVELS or self.user.is_anonymous():<EOL><INDENT>return super(StorageMixin, self).add(level, message, extra_tags)<EOL><DEDENT>self.added_new = True<EOL>m = self.backend.create_message(level, message, extra_tags)<EOL>self.backend.archive_store([self.user], m)<EOL>self._queued_messages.append(m)<EOL>

If the message level was configured for being stored and request.user
is not anonymous, save it to the database. Otherwise, let some other
class handle the message.

Notice: controls like checking the message is not empty and the level
is above the filter need to be performed here, but it could happen
they'll be performed again later if the message does not need to be
stored.


def _store(self, messages, response, *args, **kwargs):

contrib_messages = []<EOL>if self.user.is_authenticated():<EOL><INDENT>if not messages:<EOL><INDENT>self.backend.inbox_purge(self.user)<EOL><DEDENT>else:<EOL><INDENT>for m in messages:<EOL><INDENT>try:<EOL><INDENT>self.backend.inbox_store([self.user], m)<EOL><DEDENT>except MessageTypeNotSupported:<EOL><INDENT>contrib_messages.append(m)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>super(StorageMixin, self)._store(contrib_messages, response, *args, **kwargs)<EOL>

persistent messages are already in the database inside the 'archive',
so we can say they're already "stored".
Here we put them in the inbox, or remove from the inbox in case the
messages were iterated.

messages contains only new msgs if self.used==True
else contains both new and unread messages


for message in messages:<EOL><INDENT>if not self.backend.can_handle(message):<EOL><INDENT>message._prepare()<EOL><DEDENT><DEDENT>

Like the base class method, prepares a list of messages for storage
but avoid to do this for `models.Message` instances.


if isinstance(val, six.string_types):<EOL><INDENT>return import_from_string(val, setting_name)<EOL><DEDENT>elif isinstance(val, (list, tuple)):<EOL><INDENT>return [import_from_string(item, setting_name) for item in val]<EOL><DEDENT>return val<EOL>

If the given setting is a string import notation,
then perform the necessary import or imports.


try:<EOL><INDENT>parts = val.split('<STR_LIT:.>')<EOL>module_path, class_name = '<STR_LIT:.>'.join(parts[:-<NUM_LIT:1>]), parts[-<NUM_LIT:1>]<EOL>module = importlib.import_module(module_path)<EOL>return getattr(module, class_name)<EOL><DEDENT>except ImportError as e:<EOL><INDENT>msg = "<STR_LIT>" % (val, setting_name,<EOL>e.__class__.__name__, e)<EOL>raise ImportError(msg)<EOL><DEDENT>

Attempt to import a class from a string representation.


return json.dumps(msg_instance._asdict(), cls=DjangoJSONEncoder)<EOL>

Dump a Message instance into a JSON string


return Message(**json.loads(force_text(json_msg)))<EOL>

Return a Message instance built from data contained in a JSON string


ret = []<EOL>for msg_json in self.client.lrange(key, <NUM_LIT:0>, -<NUM_LIT:1>):<EOL><INDENT>ret.append(self._fromJSON(msg_json))<EOL><DEDENT>return ret<EOL>


def create_message(self, level, msg_text, extra_tags='<STR_LIT>', date=None, url=None):

if not date:<EOL><INDENT>now = timezone.now()<EOL><DEDENT>else:<EOL><INDENT>now = date<EOL><DEDENT>r = now.isoformat()<EOL>if now.microsecond:<EOL><INDENT>r = r[:<NUM_LIT>] + r[<NUM_LIT>:]<EOL><DEDENT>if r.endswith('<STR_LIT>'):<EOL><INDENT>r = r[:-<NUM_LIT:6>] + '<STR_LIT>'<EOL><DEDENT>fingerprint = r + msg_text<EOL>msg_id = hashlib.sha256(fingerprint.encode('<STR_LIT:ascii>', '<STR_LIT:ignore>')).hexdigest()<EOL>return Message(id=msg_id, message=msg_text, level=level, tags=extra_tags, date=r, url=url)<EOL>

Message instances are namedtuples of type `Message`.
The date field is already serialized in datetime.isoformat ECMA-262 format


def create_message(self, level, msg_text, extra_tags, date=None):

raise NotImplementedError()<EOL>

Create and return a `Message` instance.
Instance types depend on backends implementation.

Params:
    `level`: message level (see django.contrib.messages)
    `msg_text`: what you think it is
    `extra_tags`: see django.contrib.messages
    `date`: a DateTime (optional)

Return:
    `Message` instance


raise NotImplementedError()<EOL>

Retrieve all the messages in `user`'s Inbox.

Params:
    `user`: Django User instance

Return:
    An iterable containing `Message` instances


raise NotImplementedError()<EOL>

Delete all the messages in `user`'s Inbox.

Params:
    user: Django User instance

Return:
    None


def inbox_store(self, users, msg_instance):

raise NotImplementedError()<EOL>

Store a `Message` instance in the inbox for a list
of users.

Params:
    users: a list or iterable containing Django User instances
    msg_instance: Message instance to persist in inbox

Return:
    None

Raise:
    MessageTypeNotSupported if `msg_instance` cannot be managed by current backend


raise NotImplementedError()<EOL>

Remove a `Message` instance from `user`'s inbox.

Params:
    user: Django User instance
    msg_id: Message identifier

Return:
    None

Raise:
    MessageDoesNotExist if msg_id was not found


Retrieve a `Message` instance from `user`'s inbox.

Params:
    user: Django User instance
    msg_id: Message identifier

Return:
    A `Message` instance

Raise:
    MessageDoesNotExist if msg_id was not found


def archive_store(self, users, msg_instance):

raise NotImplementedError()<EOL>

Store a `Message` instance in the archive for a list
of users.

Params:
    users: a list or iterable containing Django User instances
    msg_instance: Message instance to persist in archive

Return:
    None

Raise:
    MessageTypeNotSupported if `msg_instance` cannot be managed by current backend


raise NotImplementedError()<EOL>

Retrieve all the messages in `user`'s archive.

Params:
    user: Django User instance

Return:
    An iterable containing `Message` instances


raise NotImplementedError()<EOL>

Determine if this backend can handle messages
of the same type of `msg_instance`.

Params:
    `msg_instance`: `Message` instance

Return:
    True if type is correct, False otherwise


raise NotImplementedError()<EOL>

Remove messages that have been expired.

Params:
    None

Return:
   None


raise NotImplementedError()<EOL>

Clear all backend data.
Warning: heavily destructive! Here for convenience, not used by the API anyway.

Params:
    None

Return:
    None


def add_message_for(users, level, message_text, extra_tags='<STR_LIT>', date=None, url=None, fail_silently=False):

BackendClass = stored_messages_settings.STORAGE_BACKEND<EOL>backend = BackendClass()<EOL>m = backend.create_message(level, message_text, extra_tags, date, url)<EOL>backend.archive_store(users, m)<EOL>backend.inbox_store(users, m)<EOL>

Send a message to a list of users without passing through `django.contrib.messages`

:param users: an iterable containing the recipients of the messages
:param level: message level
:param message_text: the string containing the message
:param extra_tags: like the Django api, a string containing extra tags for the message
:param date: a date, different than the default timezone.now
:param url: an optional url
:param fail_silently: not used at the moment


def broadcast_message(level, message_text, extra_tags='<STR_LIT>', date=None, url=None, fail_silently=False):

from django.contrib.auth import get_user_model<EOL>users = get_user_model().objects.all()<EOL>add_message_for(users, level, message_text, extra_tags=extra_tags, date=date, url=url, fail_silently=fail_silently)<EOL>

Send a message to all users aka broadcast.

:param level: message level
:param message_text: the string containing the message
:param extra_tags: like the Django api, a string containing extra tags for the message
:param date: a date, different than the default timezone.now
:param url: an optional url
:param fail_silently: not used at the moment


BackendClass = stored_messages_settings.STORAGE_BACKEND<EOL>backend = BackendClass()<EOL>backend.inbox_delete(user, message)<EOL>

Mark message instance as read for user.
Returns True if the message was `unread` and thus actually marked as `read` or False in case
it is already `read` or it does not exist at all.

:param user: user instance for the recipient
:param message: a Message instance to mark as read


BackendClass = stored_messages_settings.STORAGE_BACKEND<EOL>backend = BackendClass()<EOL>backend.inbox_purge(user)<EOL>

Mark all message instances for a user as read.

:param user: user instance for the recipient


init_py = open(os.path.join(package, '<STR_LIT>')).read()<EOL>return re.search("<STR_LIT>", init_py).group(<NUM_LIT:1>)<EOL>

Return package version as listed in `__version__` in `init.py`.