try:<EOL><INDENT>msg_ids = self.db.get_history()<EOL><DEDENT>except Exception as e:<EOL><INDENT>content = error.wrap_exception()<EOL><DEDENT>else:<EOL><INDENT>content = dict(status='<STR_LIT>', history=msg_ids)<EOL><DEDENT>self.session.send(self.query, "<STR_LIT>", content=content,<EOL>parent=msg, ident=client_id)<EOL>

Get a list of all msg_ids in our DB records


content = msg['<STR_LIT:content>']<EOL>query = content.get('<STR_LIT>', {})<EOL>keys = content.get('<STR_LIT>', None)<EOL>buffers = []<EOL>empty = list()<EOL>try:<EOL><INDENT>records = self.db.find_records(query, keys)<EOL><DEDENT>except Exception as e:<EOL><INDENT>content = error.wrap_exception()<EOL><DEDENT>else:<EOL><INDENT>if keys is not None:<EOL><INDENT>buffer_lens = [] if '<STR_LIT>' in keys else None<EOL>result_buffer_lens = [] if '<STR_LIT>' in keys else None<EOL><DEDENT>else:<EOL><INDENT>buffer_lens = None<EOL>result_buffer_lens = None<EOL><DEDENT>for rec in records:<EOL><INDENT>b = rec.pop('<STR_LIT>', empty) or empty<EOL>if buffer_lens is not None:<EOL><INDENT>buffer_lens.append(len(b))<EOL>buffers.extend(b)<EOL><DEDENT>rb = rec.pop('<STR_LIT>', empty) or empty<EOL>if result_buffer_lens is not None:<EOL><INDENT>result_buffer_lens.append(len(rb))<EOL>buffers.extend(rb)<EOL><DEDENT><DEDENT>content = dict(status='<STR_LIT>', records=records, buffer_lens=buffer_lens,<EOL>result_buffer_lens=result_buffer_lens)<EOL><DEDENT>self.session.send(self.query, "<STR_LIT>", content=content,<EOL>parent=msg, ident=client_id,<EOL>buffers=buffers)<EOL>

Perform a raw query on the task record database.


d = {}<EOL>keys = self._keys if keys is None else keys<EOL>for key in keys:<EOL><INDENT>d[key] = None<EOL><DEDENT>return d<EOL>


cursor = self._db.execute("<STR_LIT>"%self.table)<EOL>lines = cursor.fetchall()<EOL>if not lines:<EOL><INDENT>return True<EOL><DEDENT>types = {}<EOL>keys = []<EOL>for line in lines:<EOL><INDENT>keys.append(line[<NUM_LIT:1>])<EOL>types[line[<NUM_LIT:1>]] = line[<NUM_LIT:2>]<EOL><DEDENT>if self._keys != keys:<EOL><INDENT>self.log.warn('<STR_LIT>')<EOL>return False<EOL><DEDENT>for key in self._keys:<EOL><INDENT>if types[key] != self._types[key]:<EOL><INDENT>self.log.warn(<EOL>'<STR_LIT>'%(key,types[key],self._types[key])<EOL>)<EOL>return False<EOL><DEDENT><DEDENT>return True<EOL>

Ensure that an incorrect table doesn't exist

        If a bad (old) table does exist, return False


<EOL>sqlite3.register_adapter(dict, _adapt_dict)<EOL>sqlite3.register_converter('<STR_LIT>', _convert_dict)<EOL>sqlite3.register_adapter(list, _adapt_bufs)<EOL>sqlite3.register_converter('<STR_LIT>', _convert_bufs)<EOL>dbfile = os.path.join(self.location, self.filename)<EOL>self._db = sqlite3.connect(dbfile, detect_types=sqlite3.PARSE_DECLTYPES,<EOL>cached_statements=<NUM_LIT:64>)<EOL>first_table = previous_table = self.table<EOL>i=<NUM_LIT:0><EOL>while not self._check_table():<EOL><INDENT>i+=<NUM_LIT:1><EOL>self.table = first_table+'<STR_LIT>'%i<EOL>self.log.warn(<EOL>"<STR_LIT>"%<EOL>(previous_table, self.table)<EOL>)<EOL>previous_table = self.table<EOL><DEDENT>self._db.execute("""<STR_LIT>"""%self.table)<EOL>self._db.commit()<EOL>

Connect to the database and get new session number.


return [ d[key] for key in self._keys ]<EOL>

turn a mongodb-style record dict into a list.


def _list_to_dict(self, line, keys=None):

keys = self._keys if keys is None else keys<EOL>d = self._defaults(keys)<EOL>for key,value in zip(keys, line):<EOL><INDENT>d[key] = value<EOL><DEDENT>return d<EOL>


expressions = []<EOL>args = []<EOL>skeys = set(check.keys())<EOL>skeys.difference_update(set(self._keys))<EOL>skeys.difference_update(set(['<STR_LIT>', '<STR_LIT>']))<EOL>if skeys:<EOL><INDENT>raise KeyError("<STR_LIT>"%skeys)<EOL><DEDENT>for name,sub_check in check.iteritems():<EOL><INDENT>if isinstance(sub_check, dict):<EOL><INDENT>for test,value in sub_check.iteritems():<EOL><INDENT>try:<EOL><INDENT>op = operators[test]<EOL><DEDENT>except KeyError:<EOL><INDENT>raise KeyError("<STR_LIT>"%test)<EOL><DEDENT>if isinstance(op, tuple):<EOL><INDENT>op, join = op<EOL><DEDENT>if value is None and op in null_operators:<EOL><INDENT>expr = "<STR_LIT>" % (name, null_operators[op])<EOL><DEDENT>else:<EOL><INDENT>expr = "<STR_LIT>"%(name, op)<EOL>if isinstance(value, (tuple,list)):<EOL><INDENT>if op in null_operators and any([v is None for v in value]):<EOL><INDENT>raise ValueError("<STR_LIT>"%test)<EOL><DEDENT>expr = '<STR_LIT>'%( join.join([expr]*len(value)) )<EOL>args.extend(value)<EOL><DEDENT>else:<EOL><INDENT>args.append(value)<EOL><DEDENT><DEDENT>expressions.append(expr)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if sub_check is None:<EOL><INDENT>expressions.append("<STR_LIT>" % name)<EOL><DEDENT>else:<EOL><INDENT>expressions.append("<STR_LIT>"%name)<EOL>args.append(sub_check)<EOL><DEDENT><DEDENT><DEDENT>expr = "<STR_LIT>".join(expressions)<EOL>return expr, args<EOL>

Turn a mongodb-style search dict into an SQL query.


d = self._defaults()<EOL>d.update(rec)<EOL>d['<STR_LIT>'] = msg_id<EOL>line = self._dict_to_list(d)<EOL>tups = '<STR_LIT>'%('<STR_LIT:U+002C>'.join(['<STR_LIT:?>']*len(line)))<EOL>self._db.execute("<STR_LIT>"%(self.table, tups), line)<EOL>


cursor = self._db.execute("""<STR_LIT>"""%self.table, (msg_id,))<EOL>line = cursor.fetchone()<EOL>if line is None:<EOL><INDENT>raise KeyError("<STR_LIT>"%msg_id)<EOL><DEDENT>return self._list_to_dict(line)<EOL>


query = "<STR_LIT>"%self.table<EOL>sets = []<EOL>keys = sorted(rec.keys())<EOL>values = []<EOL>for key in keys:<EOL><INDENT>sets.append('<STR_LIT>'%key)<EOL>values.append(rec[key])<EOL><DEDENT>query += '<STR_LIT:U+002CU+0020>'.join(sets)<EOL>query += '<STR_LIT>'<EOL>values.append(msg_id)<EOL>self._db.execute(query, values)<EOL>


self._db.execute("""<STR_LIT>"""%self.table, (msg_id,))<EOL>


expr,args = self._render_expression(check)<EOL>query = "<STR_LIT>"%(self.table, expr)<EOL>self._db.execute(query,args)<EOL>


def find_records(self, check, keys=None):

if keys:<EOL><INDENT>bad_keys = [ key for key in keys if key not in self._keys ]<EOL>if bad_keys:<EOL><INDENT>raise KeyError("<STR_LIT>"%bad_keys)<EOL><DEDENT><DEDENT>if keys:<EOL><INDENT>if '<STR_LIT>' in keys:<EOL><INDENT>keys.remove('<STR_LIT>')<EOL><DEDENT>keys.insert(<NUM_LIT:0>, '<STR_LIT>')<EOL>req = '<STR_LIT:U+002CU+0020>'.join(keys)<EOL><DEDENT>else:<EOL><INDENT>req = '<STR_LIT:*>'<EOL><DEDENT>expr,args = self._render_expression(check)<EOL>query = """<STR_LIT>"""%(req, self.table, expr)<EOL>cursor = self._db.execute(query, args)<EOL>matches = cursor.fetchall()<EOL>records = []<EOL>for line in matches:<EOL><INDENT>rec = self._list_to_dict(line, keys)<EOL>records.append(rec)<EOL><DEDENT>return records<EOL>

Find records matching a query dict, optionally extracting subset of keys.

        Returns list of matching records.

        Parameters
        ----------

        check: dict
            mongodb-style query argument
        keys: list of strs [optional]
            if specified, the subset of keys to extract.  msg_id will *always* be
            included.


query = """<STR_LIT>"""%self.table<EOL>cursor = self._db.execute(query)<EOL>return [ tup[<NUM_LIT:0>] for tup in cursor.fetchall()]<EOL>

get all msg_ids, ordered by time submitted.


@decorator<EOL>def save_ids(f, self, *args, **kwargs):

n_previous = len(self.client.history)<EOL>try:<EOL><INDENT>ret = f(self, *args, **kwargs)<EOL><DEDENT>finally:<EOL><INDENT>nmsgs = len(self.client.history) - n_previous<EOL>msg_ids = self.client.history[-nmsgs:]<EOL>self.history.extend(msg_ids)<EOL>list(map(self.outstanding.add, msg_ids))<EOL><DEDENT>return ret<EOL>

Keep our history and outstanding attributes up to date after a method call.


@decorator<EOL>def sync_results(f, self, *args, **kwargs):

ret = f(self, *args, **kwargs)<EOL>delta = self.outstanding.difference(self.client.outstanding)<EOL>completed = self.outstanding.intersection(delta)<EOL>self.outstanding = self.outstanding.difference(completed)<EOL>return ret<EOL>

sync relevant results from self.client to our results attribute.


@decorator<EOL>def spin_after(f, self, *args, **kwargs):

ret = f(self, *args, **kwargs)<EOL>self.spin()<EOL>return ret<EOL>


for name, value in kwargs.items():<EOL><INDENT>if name not in self._flag_names:<EOL><INDENT>raise KeyError("<STR_LIT>"%name)<EOL><DEDENT>else:<EOL><INDENT>setattr(self, name, value)<EOL><DEDENT><DEDENT>

set my attribute flags by keyword.

        Views determine behavior with a few attributes (`block`, `track`, etc.).
        These attributes can be set all at once by name with this method.

        Parameters
        ----------

        block : bool
            whether to wait for results
        track : bool
            whether to create a MessageTracker to allow the user to
            safely edit after arrays and buffers during non-copying
            sends.


<EOL>saved_flags = {}<EOL>for f in self._flag_names:<EOL><INDENT>saved_flags[f] = getattr(self, f)<EOL><DEDENT>self.set_flags(**kwargs)<EOL>try:<EOL><INDENT>yield<EOL><DEDENT>finally:<EOL><INDENT>self.set_flags(**saved_flags)<EOL><DEDENT>

temporarily set flags, for use in `with` statements.

        See set_flags for permanent setting of flags

        Examples
        --------

        >>> view.track=False
        ...
        >>> with view.temp_flags(track=True):
        ...    ar = view.apply(dostuff, my_big_array)
        ...    ar.tracker.wait() # wait for send to finish
        >>> view.track
        False


@sync_results<EOL><INDENT>@save_ids<EOL>def _really_apply(self, f, args, kwargs, block=None, **options):<DEDENT>

raise NotImplementedError("<STR_LIT>")<EOL>


return self._really_apply(f, args, kwargs)<EOL>

calls f(*args, **kwargs) on remote engines, returning the result.

        This method sets all apply flags via this View's attributes.

        if self.block is False:
            returns AsyncResult
        else:
            returns actual result of f(*args, **kwargs)


def apply_async(self, f, *args, **kwargs):

return self._really_apply(f, args, kwargs, block=False)<EOL>

calls f(*args, **kwargs) on remote engines in a nonblocking manner.

        returns AsyncResult


@spin_after<EOL><INDENT>def apply_sync(self, f, *args, **kwargs):<DEDENT>

return self._really_apply(f, args, kwargs, block=True)<EOL>

calls f(*args, **kwargs) on remote engines in a blocking manner,
         returning the result.

        returns: actual result of f(*args, **kwargs)


self.client.spin()<EOL>


@sync_results<EOL><INDENT>def wait(self, jobs=None, timeout=-<NUM_LIT:1>):<DEDENT>

if jobs is None:<EOL><INDENT>jobs = self.history<EOL><DEDENT>return self.client.wait(jobs, timeout)<EOL>

waits on one or more `jobs`, for up to `timeout` seconds.

        Parameters
        ----------

        jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
                ints are indices to self.history
                strs are msg_ids
                default: wait on all outstanding messages
        timeout : float
                a time in seconds, after which to give up.
                default is -1, which means no timeout

        Returns
        -------

        True : when all msg_ids are done
        False : timeout reached, some msg_ids still outstanding


def abort(self, jobs=None, targets=None, block=None):

block = block if block is not None else self.block<EOL>targets = targets if targets is not None else self.targets<EOL>jobs = jobs if jobs is not None else list(self.outstanding)<EOL>return self.client.abort(jobs=jobs, targets=targets, block=block)<EOL>

Abort jobs on my engines.

        Parameters
        ----------

        jobs : None, str, list of strs, optional
            if None: abort all jobs.
            else: abort specific msg_id(s).


def queue_status(self, targets=None, verbose=False):

targets = targets if targets is not None else self.targets<EOL>return self.client.queue_status(targets=targets, verbose=verbose)<EOL>


def purge_results(self, jobs=[], targets=[]):

if targets is None or targets == '<STR_LIT:all>':<EOL><INDENT>targets = self.targets<EOL><DEDENT>return self.client.purge_results(jobs=jobs, targets=targets)<EOL>

Instruct the controller to forget specific results.


def shutdown(self, targets=None, restart=False, hub=False, block=None):

block = self.block if block is None else block<EOL>if targets is None or targets == '<STR_LIT:all>':<EOL><INDENT>targets = self.targets<EOL><DEDENT>return self.client.shutdown(targets=targets, restart=restart, hub=hub, block=block)<EOL>

Terminates one or more engine processes, optionally including the hub.


if indices_or_msg_ids is None:<EOL><INDENT>indices_or_msg_ids = -<NUM_LIT:1><EOL><DEDENT>if isinstance(indices_or_msg_ids, int):<EOL><INDENT>indices_or_msg_ids = self.history[indices_or_msg_ids]<EOL><DEDENT>elif isinstance(indices_or_msg_ids, (list,tuple,set)):<EOL><INDENT>indices_or_msg_ids = list(indices_or_msg_ids)<EOL>for i,index in enumerate(indices_or_msg_ids):<EOL><INDENT>if isinstance(index, int):<EOL><INDENT>indices_or_msg_ids[i] = self.history[index]<EOL><DEDENT><DEDENT><DEDENT>return self.client.get_result(indices_or_msg_ids)<EOL>

return one or more results, specified by history index or msg_id.

        See client.get_result for details.


raise NotImplementedError<EOL>


def map_async(self, f, *sequences, **kwargs):

if '<STR_LIT>' in kwargs:<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>kwargs['<STR_LIT>'] = False<EOL>return self.map(f,*sequences,**kwargs)<EOL>

Parallel version of builtin `map`, using this view's engines.

        This is equivalent to map(...block=False)

        See `self.map` for details.


def map_sync(self, f, *sequences, **kwargs):

if '<STR_LIT>' in kwargs:<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>kwargs['<STR_LIT>'] = True<EOL>return self.map(f,*sequences,**kwargs)<EOL>

Parallel version of builtin `map`, using this view's engines.

        This is equivalent to map(...block=True)

        See `self.map` for details.


def imap(self, f, *sequences, **kwargs):

return iter(self.map_async(f,*sequences, **kwargs))<EOL>

Parallel version of `itertools.imap`.

        See `self.map` for details.


block = self.block if block is None else block<EOL>return remote(self, block=block, **flags)<EOL>


def parallel(self, dist='<STR_LIT:b>', block=None, **flags):

block = self.block if block is None else block<EOL>return parallel(self, dist=dist, block=block, **flags)<EOL>


return self.sync_imports(True)<EOL>

sync_imports(local=True) as a property.

        See sync_imports for details.


@contextmanager<EOL><INDENT>def sync_imports(self, local=True, quiet=False):<DEDENT>

import builtins<EOL>local_import = builtins.__import__<EOL>modules = set()<EOL>results = []<EOL>@util.interactive<EOL>def remote_import(name, fromlist, level):<EOL><INDENT>"""<STR_LIT>"""<EOL>import sys<EOL>user_ns = globals()<EOL>mod = __import__(name, fromlist=fromlist, level=level)<EOL>if fromlist:<EOL><INDENT>for key in fromlist:<EOL><INDENT>user_ns[key] = getattr(mod, key)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>user_ns[name] = sys.modules[name]<EOL><DEDENT><DEDENT>def view_import(name, globals={}, locals={}, fromlist=[], level=-<NUM_LIT:1>):<EOL><INDENT>"""<STR_LIT>"""<EOL>save_import = builtins.__import__<EOL>builtins.__import__ = local_import<EOL>if imp.lock_held():<EOL><INDENT>return local_import(name, globals, locals, fromlist, level)<EOL><DEDENT>imp.acquire_lock()<EOL>if local:<EOL><INDENT>mod = local_import(name, globals, locals, fromlist, level)<EOL><DEDENT>else:<EOL><INDENT>raise NotImplementedError("<STR_LIT>")<EOL><DEDENT>imp.release_lock()<EOL>key = name+'<STR_LIT::>'+'<STR_LIT:U+002C>'.join(fromlist or [])<EOL>if level == -<NUM_LIT:1> and key not in modules:<EOL><INDENT>modules.add(key)<EOL>if not quiet:<EOL><INDENT>if fromlist:<EOL><INDENT>print("<STR_LIT>"%('<STR_LIT:U+002C>'.join(fromlist), name))<EOL><DEDENT>else:<EOL><INDENT>print("<STR_LIT>"%name)<EOL><DEDENT><DEDENT>results.append(self.apply_async(remote_import, name, fromlist, level))<EOL><DEDENT>builtins.__import__ = save_import<EOL>return mod<EOL><DEDENT>builtins.__import__ = view_import<EOL>try:<EOL><INDENT>yield<EOL><DEDENT>except ImportError:<EOL><INDENT>if local:<EOL><INDENT>raise<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>builtins.__import__ = local_import<EOL><DEDENT>for r in results:<EOL><INDENT>r.get()<EOL><DEDENT>

Context Manager for performing simultaneous local and remote imports.

        'import x as y' will *not* work.  The 'as y' part will simply be ignored.

        If `local=True`, then the package will also be imported locally.

        If `quiet=True`, no output will be produced when attempting remote
        imports.

        Note that remote-only (`local=False`) imports have not been implemented.

        >>> with view.sync_imports():
        ...    from numpy import recarray
        importing recarray from numpy on engine(s)


@sync_results<EOL><INDENT>@save_ids<EOL>def _really_apply(self, f, args=None, kwargs=None, targets=None, block=None, track=None):<DEDENT>

args = [] if args is None else args<EOL>kwargs = {} if kwargs is None else kwargs<EOL>block = self.block if block is None else block<EOL>track = self.track if track is None else track<EOL>targets = self.targets if targets is None else targets<EOL>_idents = self.client._build_targets(targets)[<NUM_LIT:0>]<EOL>msg_ids = []<EOL>trackers = []<EOL>for ident in _idents:<EOL><INDENT>msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,<EOL>ident=ident)<EOL>if track:<EOL><INDENT>trackers.append(msg['<STR_LIT>'])<EOL><DEDENT>msg_ids.append(msg['<STR_LIT>']['<STR_LIT>'])<EOL><DEDENT>tracker = None if track is False else zmq.MessageTracker(*trackers)<EOL>ar = AsyncResult(self.client, msg_ids, fname=getname(f), targets=targets, tracker=tracker)<EOL>if block:<EOL><INDENT>try:<EOL><INDENT>return ar.get()<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return ar<EOL>

calls f(*args, **kwargs) on remote engines, returning the result.

        This method sets all of `apply`'s flags via this View's attributes.

        Parameters
        ----------

        f : callable

        args : list [default: empty]

        kwargs : dict [default: empty]

        targets : target list [default: self.targets]
            where to run
        block : bool [default: self.block]
            whether to block
        track : bool [default: self.track]
            whether to ask zmq to track the message, for safe non-copying sends

        Returns
        -------

        if self.block is False:
            returns AsyncResult
        else:
            returns actual result of f(*args, **kwargs) on the engine(s)
            This will be a list of self.targets is also a list (even length 1), or
            the single result if self.targets is an integer engine id


@spin_after<EOL><INDENT>def map(self, f, *sequences, **kwargs):<DEDENT>

block = kwargs.pop('<STR_LIT>', self.block)<EOL>for k in list(kwargs.keys()):<EOL><INDENT>if k not in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>raise TypeError("<STR_LIT>"%k)<EOL><DEDENT><DEDENT>assert len(sequences) > <NUM_LIT:0>, "<STR_LIT>"<EOL>pf = ParallelFunction(self, f, block=block, **kwargs)<EOL>return pf.map(*sequences)<EOL>

view.map(f, *sequences, block=self.block) => list|AsyncMapResult

        Parallel version of builtin `map`, using this View's `targets`.

        There will be one task per target, so work will be chunked
        if the sequences are longer than `targets`.

        Results can be iterated as they are ready, but will become available in chunks.

        Parameters
        ----------

        f : callable
            function to be mapped
        *sequences: one or more sequences of matching length
            the sequences to be distributed and passed to `f`
        block : bool
            whether to wait for the result or not [default self.block]

        Returns
        -------

        if block=False:
            AsyncMapResult
                An object like AsyncResult, but which reassembles the sequence of results
                into a single list. AsyncMapResults can be iterated through before all
                results are complete.
        else:
            list
                the result of map(f,*sequences)


@sync_results<EOL><INDENT>@save_ids<EOL>def execute(self, code, silent=True, targets=None, block=None):<DEDENT>

block = self.block if block is None else block<EOL>targets = self.targets if targets is None else targets<EOL>_idents = self.client._build_targets(targets)[<NUM_LIT:0>]<EOL>msg_ids = []<EOL>trackers = []<EOL>for ident in _idents:<EOL><INDENT>msg = self.client.send_execute_request(self._socket, code, silent=silent, ident=ident)<EOL>msg_ids.append(msg['<STR_LIT>']['<STR_LIT>'])<EOL><DEDENT>ar = AsyncResult(self.client, msg_ids, fname='<STR_LIT>', targets=targets)<EOL>if block:<EOL><INDENT>try:<EOL><INDENT>ar.get()<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return ar<EOL>

Executes `code` on `targets` in blocking or nonblocking manner.

        ``execute`` is always `bound` (affects engine namespace)

        Parameters
        ----------

        code : str
                the code string to be executed
        block : bool
                whether or not to wait until done to return
                default: self.block


def run(self, filename, targets=None, block=None):

with open(filename, '<STR_LIT:r>') as f:<EOL><INDENT>code = f.read()+'<STR_LIT:\n>'<EOL><DEDENT>return self.execute(code, block=block, targets=targets)<EOL>

Execute contents of `filename` on my engine(s).

        This simply reads the contents of the file and calls `execute`.

        Parameters
        ----------

        filename : str
                The path to the file
        targets : int/str/list of ints/strs
                the engines on which to execute
                default : all
        block : bool
                whether or not to wait until done
                default: self.block


return self.push(ns, block=self.block, track=self.track)<EOL>

update remote namespace with dict `ns`

        See `push` for details.


def push(self, ns, targets=None, block=None, track=None):

block = block if block is not None else self.block<EOL>track = track if track is not None else self.track<EOL>targets = targets if targets is not None else self.targets<EOL>if not isinstance(ns, dict):<EOL><INDENT>raise TypeError("<STR_LIT>"%type(ns))<EOL><DEDENT>return self._really_apply(util._push, kwargs=ns, block=block, track=track, targets=targets)<EOL>

update remote namespace with dict `ns`

        Parameters
        ----------

        ns : dict
            dict of keys with which to update engine namespace(s)
        block : bool [default : self.block]
            whether to wait to be notified of engine receipt


<EOL>return self.pull(key_s, block=True)<EOL>

get object(s) by `key_s` from remote namespace

        see `pull` for details.


def pull(self, names, targets=None, block=None):

block = block if block is not None else self.block<EOL>targets = targets if targets is not None else self.targets<EOL>applier = self.apply_sync if block else self.apply_async<EOL>if isinstance(names, str):<EOL><INDENT>pass<EOL><DEDENT>elif isinstance(names, (list,tuple,set)):<EOL><INDENT>for key in names:<EOL><INDENT>if not isinstance(key, str):<EOL><INDENT>raise TypeError("<STR_LIT>"%type(key))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>"%names)<EOL><DEDENT>return self._really_apply(util._pull, (names,), block=block, targets=targets)<EOL>

get object(s) by `name` from remote namespace

        will return one object if it is a key.
        can also take a list of keys, in which case it will return a list of objects.


def scatter(self, key, seq, dist='<STR_LIT:b>', flatten=False, targets=None, block=None, track=None):

block = block if block is not None else self.block<EOL>track = track if track is not None else self.track<EOL>targets = targets if targets is not None else self.targets<EOL>targets = self.client._build_targets(targets)[<NUM_LIT:1>]<EOL>mapObject = Map.dists[dist]()<EOL>nparts = len(targets)<EOL>msg_ids = []<EOL>trackers = []<EOL>for index, engineid in enumerate(targets):<EOL><INDENT>partition = mapObject.getPartition(seq, index, nparts)<EOL>if flatten and len(partition) == <NUM_LIT:1>:<EOL><INDENT>ns = {key: partition[<NUM_LIT:0>]}<EOL><DEDENT>else:<EOL><INDENT>ns = {key: partition}<EOL><DEDENT>r = self.push(ns, block=False, track=track, targets=engineid)<EOL>msg_ids.extend(r.msg_ids)<EOL>if track:<EOL><INDENT>trackers.append(r._tracker)<EOL><DEDENT><DEDENT>if track:<EOL><INDENT>tracker = zmq.MessageTracker(*trackers)<EOL><DEDENT>else:<EOL><INDENT>tracker = None<EOL><DEDENT>r = AsyncResult(self.client, msg_ids, fname='<STR_LIT>', targets=targets, tracker=tracker)<EOL>if block:<EOL><INDENT>r.wait()<EOL><DEDENT>else:<EOL><INDENT>return r<EOL><DEDENT>

Partition a Python sequence and send the partitions to a set of engines.


@sync_results<EOL><INDENT>@save_ids<EOL>def gather(self, key, dist='<STR_LIT:b>', targets=None, block=None):<DEDENT>

block = block if block is not None else self.block<EOL>targets = targets if targets is not None else self.targets<EOL>mapObject = Map.dists[dist]()<EOL>msg_ids = []<EOL>targets = self.client._build_targets(targets)[<NUM_LIT:1>]<EOL>for index, engineid in enumerate(targets):<EOL><INDENT>msg_ids.extend(self.pull(key, block=False, targets=engineid).msg_ids)<EOL><DEDENT>r = AsyncMapResult(self.client, msg_ids, mapObject, fname='<STR_LIT>')<EOL>if block:<EOL><INDENT>try:<EOL><INDENT>return r.get()<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return r<EOL>

Gather a partitioned sequence on a set of engines as a single local seq.


def clear(self, targets=None, block=False):

block = block if block is not None else self.block<EOL>targets = targets if targets is not None else self.targets<EOL>return self.client.clear(targets=targets, block=block)<EOL>

Clear the remote namespaces on my engines.


def kill(self, targets=None, block=True):

block = block if block is not None else self.block<EOL>targets = targets if targets is not None else self.targets<EOL>return self.client.kill(targets=targets, block=block)<EOL>


from IPython.parallel.client.magics import ParallelMagics<EOL>try:<EOL><INDENT>ip = get_ipython()<EOL><DEDENT>except NameError:<EOL><INDENT>print("<STR_LIT>")<EOL>return<EOL><DEDENT>M = ParallelMagics(ip, self, suffix)<EOL>ip.magics_manager.register(M)<EOL>

Activate IPython magics associated with this View

        Defines the magics `%px, %autopx, %pxresult, %%px, %pxconfig`

        Parameters
        ----------

        suffix: str [default: '']
            The suffix, if any, for the magics.  This allows you to have
            multiple views associated with parallel magics at the same time.

            e.g. ``rc[::2].activate(suffix='_even')`` will give you
            the magics ``%px_even``, ``%pxresult_even``, etc. for running magics
            on the even engines.


if dep is None or isinstance(dep, (str, AsyncResult, Dependency)):<EOL><INDENT>return True<EOL><DEDENT>elif isinstance(dep, (list,set, tuple)):<EOL><INDENT>for d in dep:<EOL><INDENT>if not isinstance(d, (str, AsyncResult)):<EOL><INDENT>return False<EOL><DEDENT><DEDENT><DEDENT>elif isinstance(dep, dict):<EOL><INDENT>if set(dep.keys()) != set(Dependency().as_dict().keys()):<EOL><INDENT>return False<EOL><DEDENT>if not isinstance(dep['<STR_LIT>'], list):<EOL><INDENT>return False<EOL><DEDENT>for d in dep['<STR_LIT>']:<EOL><INDENT>if not isinstance(d, str):<EOL><INDENT>return False<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

validate a dependency.

        For use in `set_flags`.


if isinstance(dep, Dependency):<EOL><INDENT>return dep.as_dict()<EOL><DEDENT>elif isinstance(dep, AsyncResult):<EOL><INDENT>return dep.msg_ids<EOL><DEDENT>elif dep is None:<EOL><INDENT>return []<EOL><DEDENT>else:<EOL><INDENT>return list(Dependency(dep))<EOL><DEDENT>

helper for building jsonable dependencies from various input forms.


super(LoadBalancedView, self).set_flags(**kwargs)<EOL>for name in ('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>if name in kwargs:<EOL><INDENT>value = kwargs[name]<EOL>if self._validate_dependency(value):<EOL><INDENT>setattr(self, name, value)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>"%value)<EOL><DEDENT><DEDENT><DEDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>t = kwargs['<STR_LIT>']<EOL>if not isinstance(t, (int, float, type(None))):<EOL><INDENT>raise TypeError("<STR_LIT>"%type(t))<EOL><DEDENT>if t is not None:<EOL><INDENT>if t < <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>"%t)<EOL><DEDENT><DEDENT>self.timeout = t<EOL><DEDENT>

set my attribute flags by keyword.

        A View is a wrapper for the Client's apply method, but with attributes
        that specify keyword arguments, those attributes can be set by keyword
        argument with this method.

        Parameters
        ----------

        block : bool
            whether to wait for results
        track : bool
            whether to create a MessageTracker to allow the user to
            safely edit after arrays and buffers during non-copying
            sends.

        after : Dependency or collection of msg_ids
            Only for load-balanced execution (targets=None)
            Specify a list of msg_ids as a time-based dependency.
            This job will only be run *after* the dependencies
            have been met.

        follow : Dependency or collection of msg_ids
            Only for load-balanced execution (targets=None)
            Specify a list of msg_ids as a location-based dependency.
            This job will only be run on an engine where this dependency
            is met.

        timeout : float/int or None
            Only for load-balanced execution (targets=None)
            Specify an amount of time (in seconds) for the scheduler to
            wait for dependencies to be met before failing with a
            DependencyTimeout.

        retries : int
            Number of times a task will be retried on failure.


@sync_results<EOL><INDENT>@save_ids<EOL>def _really_apply(self, f, args=None, kwargs=None, block=None, track=None,<EOL>after=None, follow=None, timeout=None,<EOL>targets=None, retries=None):<DEDENT>

<EOL>if self._socket.closed:<EOL><INDENT>msg = "<STR_LIT>"<EOL>if self._task_scheme == '<STR_LIT>':<EOL><INDENT>msg += "<STR_LIT>"<EOL>msg += "<STR_LIT>"<EOL><DEDENT>raise RuntimeError(msg)<EOL><DEDENT>if self._task_scheme == '<STR_LIT>':<EOL><INDENT>msg = "<STR_LIT>"<EOL>"<STR_LIT>"<EOL>if (follow or after or retries or targets or timeout):<EOL><INDENT>raise RuntimeError(msg)<EOL><DEDENT>if isinstance(f, dependent):<EOL><INDENT>warnings.warn(msg, RuntimeWarning)<EOL><DEDENT><DEDENT>args = [] if args is None else args<EOL>kwargs = {} if kwargs is None else kwargs<EOL>block = self.block if block is None else block<EOL>track = self.track if track is None else track<EOL>after = self.after if after is None else after<EOL>retries = self.retries if retries is None else retries<EOL>follow = self.follow if follow is None else follow<EOL>timeout = self.timeout if timeout is None else timeout<EOL>targets = self.targets if targets is None else targets<EOL>if not isinstance(retries, int):<EOL><INDENT>raise TypeError('<STR_LIT>'%type(retries))<EOL><DEDENT>if targets is None:<EOL><INDENT>idents = []<EOL><DEDENT>else:<EOL><INDENT>idents = self.client._build_targets(targets)[<NUM_LIT:0>]<EOL>idents = [ ident.decode() for ident in idents ]<EOL><DEDENT>after = self._render_dependency(after)<EOL>follow = self._render_dependency(follow)<EOL>subheader = dict(after=after, follow=follow, timeout=timeout, targets=idents, retries=retries)<EOL>msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,<EOL>subheader=subheader)<EOL>tracker = None if track is False else msg['<STR_LIT>']<EOL>ar = AsyncResult(self.client, msg['<STR_LIT>']['<STR_LIT>'], fname=getname(f), targets=None, tracker=tracker)<EOL>if block:<EOL><INDENT>try:<EOL><INDENT>return ar.get()<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return ar<EOL>

calls f(*args, **kwargs) on a remote engine, returning the result.

        This method temporarily sets all of `apply`'s flags for a single call.

        Parameters
        ----------

        f : callable

        args : list [default: empty]

        kwargs : dict [default: empty]

        block : bool [default: self.block]
            whether to block
        track : bool [default: self.track]
            whether to ask zmq to track the message, for safe non-copying sends

        !!!!!! TODO: THE REST HERE  !!!!

        Returns
        -------

        if self.block is False:
            returns AsyncResult
        else:
            returns actual result of f(*args, **kwargs) on the engine(s)
            This will be a list of self.targets is also a list (even length 1), or
            the single result if self.targets is an integer engine id


@spin_after<EOL><INDENT>@save_ids<EOL>def map(self, f, *sequences, **kwargs):<DEDENT>

<EOL>block = kwargs.get('<STR_LIT>', self.block)<EOL>chunksize = kwargs.get('<STR_LIT>', <NUM_LIT:1>)<EOL>ordered = kwargs.get('<STR_LIT>', True)<EOL>keyset = set(kwargs.keys())<EOL>extra_keys = keyset.difference_update(set(['<STR_LIT>', '<STR_LIT>']))<EOL>if extra_keys:<EOL><INDENT>raise TypeError("<STR_LIT>"%list(extra_keys))<EOL><DEDENT>assert len(sequences) > <NUM_LIT:0>, "<STR_LIT>"<EOL>pf = ParallelFunction(self, f, block=block, chunksize=chunksize, ordered=ordered)<EOL>return pf.map(*sequences)<EOL>

view.map(f, *sequences, block=self.block, chunksize=1, ordered=True) => list|AsyncMapResult

        Parallel version of builtin `map`, load-balanced by this View.

        `block`, and `chunksize` can be specified by keyword only.

        Each `chunksize` elements will be a separate task, and will be
        load-balanced. This lets individual elements be available for iteration
        as soon as they arrive.

        Parameters
        ----------

        f : callable
            function to be mapped
        *sequences: one or more sequences of matching length
            the sequences to be distributed and passed to `f`
        block : bool [default self.block]
            whether to wait for the result or not
        track : bool
            whether to create a MessageTracker to allow the user to
            safely edit after arrays and buffers during non-copying
            sends.
        chunksize : int [default 1]
            how many elements should be in each task.
        ordered : bool [default True]
            Whether the results should be gathered as they arrive, or enforce
            the order of submission.

            Only applies when iterating through AsyncMapResult as results arrive.
            Has no effect when block=True.

        Returns
        -------

        if block=False:
            AsyncMapResult
                An object like AsyncResult, but which reassembles the sequence of results
                into a single list. AsyncMapResults can be iterated through before all
                results are complete.
            else:
                the result of map(f,*sequences)


try:<EOL><INDENT>return td.total_seconds()<EOL><DEDENT>except AttributeError:<EOL><INDENT>return <NUM_LIT> * (td.microseconds + (td.seconds + td.days * <NUM_LIT> * <NUM_LIT>) * <NUM_LIT:10>**<NUM_LIT:6>)<EOL><DEDENT>

timedelta.total_seconds was added in 2.7


@decorator<EOL>def check_ready(f, self, *args, **kwargs):

self.wait(<NUM_LIT:0>)<EOL>if not self._ready:<EOL><INDENT>raise error.TimeoutError("<STR_LIT>")<EOL><DEDENT>return f(self, *args, **kwargs)<EOL>

Call spin() to sync state prior to calling the method.


if self._single_result:<EOL><INDENT>return res[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>return res<EOL><DEDENT>

Reconstruct our result from actual result list (always a list)

        Override me in subclasses for turning a list of results
        into the expected form.


if not self.ready():<EOL><INDENT>self.wait(timeout)<EOL><DEDENT>if self._ready:<EOL><INDENT>if self._success:<EOL><INDENT>return self._result<EOL><DEDENT>else:<EOL><INDENT>raise self._exception<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise error.TimeoutError("<STR_LIT>")<EOL><DEDENT>

Return the result when it arrives.

        If `timeout` is not ``None`` and the result does not arrive within
        `timeout` seconds then ``TimeoutError`` is raised. If the
        remote call raised an exception then that exception will be reraised
        by get() inside a `RemoteError`.


if not self._ready:<EOL><INDENT>self.wait(<NUM_LIT:0>)<EOL><DEDENT>return self._ready<EOL>


if self._ready:<EOL><INDENT>return<EOL><DEDENT>self._ready = self._client.wait(self.msg_ids, timeout)<EOL>if self._ready:<EOL><INDENT>try:<EOL><INDENT>results = list(map(self._client.results.get, self.msg_ids))<EOL>self._result = results<EOL>if self._single_result:<EOL><INDENT>r = results[<NUM_LIT:0>]<EOL>if isinstance(r, Exception):<EOL><INDENT>raise r<EOL><DEDENT><DEDENT>else:<EOL><INDENT>results = error.collect_exceptions(results, self._fname)<EOL><DEDENT>self._result = self._reconstruct_result(results)<EOL><DEDENT>except Exception as e:<EOL><INDENT>self._exception = e<EOL>self._success = False<EOL><DEDENT>else:<EOL><INDENT>self._success = True<EOL><DEDENT>finally:<EOL><INDENT>self._metadata = list(map(self._client.metadata.get, self.msg_ids))<EOL>self._wait_for_outputs(<NUM_LIT:10>)<EOL><DEDENT><DEDENT>

Wait until the result is available or until `timeout` seconds pass.

        This method always returns None.


assert self.ready()<EOL>return self._success<EOL>

Return whether the call completed without raising an exception.

        Will raise ``AssertionError`` if the result is not ready.


results = self.get(timeout)<EOL>engine_ids = [ md['<STR_LIT>'] for md in self._metadata ]<EOL>bycount = sorted(engine_ids, key=lambda k: engine_ids.count(k))<EOL>maxcount = bycount.count(bycount[-<NUM_LIT:1>])<EOL>if maxcount > <NUM_LIT:1>:<EOL><INDENT>raise ValueError("<STR_LIT>"%(<EOL>maxcount, bycount[-<NUM_LIT:1>]))<EOL><DEDENT>return dict(list(zip(engine_ids,results)))<EOL>

Get the results as a dict, keyed by engine_id.

        timeout behavior is described in `get()`.


return self.get()<EOL>

result property wrapper for `get(timeout=0)`.


if self._single_result:<EOL><INDENT>return self._metadata[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>return self._metadata<EOL><DEDENT>

property for accessing execution metadata.


return self.get_dict()<EOL>


assert not self.ready(), "<STR_LIT>"<EOL>return self._client.abort(self.msg_ids, targets=self._targets, block=True)<EOL>


return self._tracker.done<EOL>

check whether my messages have been sent.


return self._tracker.wait(timeout)<EOL>

wait for pyzmq send to complete.

        This is necessary when sending arrays that you intend to edit in-place.
        `timeout` is in seconds, and will raise TimeoutError if it is reached
        before the send completes.


if isinstance(key, int):<EOL><INDENT>return error.collect_exceptions([self._result[key]], self._fname)[<NUM_LIT:0>]<EOL><DEDENT>elif isinstance(key, slice):<EOL><INDENT>return error.collect_exceptions(self._result[key], self._fname)<EOL><DEDENT>elif isinstance(key, str):<EOL><INDENT>values = [ md[key] for md in self._metadata ]<EOL>if self._single_result:<EOL><INDENT>return values[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>return values<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>"%type(key))<EOL><DEDENT>

getitem returns result value(s) if keyed by int/slice, or metadata if key is str.


try:<EOL><INDENT>return self.__getitem__(key)<EOL><DEDENT>except (error.TimeoutError, KeyError):<EOL><INDENT>raise AttributeError("<STR_LIT>"%(<EOL>self.__class__.__name__, key))<EOL><DEDENT>

getattr maps to getitem for convenient attr access to metadata.


def timedelta(self, start, end, start_key=min, end_key=max):

if not isinstance(start, datetime):<EOL><INDENT>start = start_key(start)<EOL><DEDENT>if not isinstance(end, datetime):<EOL><INDENT>end = end_key(end)<EOL><DEDENT>return _total_seconds(end - start)<EOL>

compute the difference between two sets of timestamps

        The default behavior is to use the earliest of the first
        and the latest of the second list, but this can be changed
        by passing a different

        Parameters
        ----------

        start : one or more datetime objects (e.g. ar.submitted)
        end : one or more datetime objects (e.g. ar.received)
        start_key : callable
            Function to call on `start` to extract the relevant
            entry [defalt: min]
        end_key : callable
            Function to call on `end` to extract the relevant
            entry [default: max]

        Returns
        -------

        dt : float
            The time elapsed (in seconds) between the two selected timestamps.


self.wait(<NUM_LIT:0>)<EOL>return len(self) - len(set(self.msg_ids).intersection(self._client.outstanding))<EOL>

the number of tasks which have been completed at this point.

        Fractional progress would be given by 1.0 * ar.progress / len(ar)


if self.ready():<EOL><INDENT>return self.wall_time<EOL><DEDENT>now = submitted = datetime.now()<EOL>for msg_id in self.msg_ids:<EOL><INDENT>if msg_id in self._client.metadata:<EOL><INDENT>stamp = self._client.metadata[msg_id]['<STR_LIT>']<EOL>if stamp and stamp < submitted:<EOL><INDENT>submitted = stamp<EOL><DEDENT><DEDENT><DEDENT>return _total_seconds(now-submitted)<EOL>


t = <NUM_LIT:0><EOL>for md in self._metadata:<EOL><INDENT>t += _total_seconds(md['<STR_LIT>'] - md['<STR_LIT>'])<EOL><DEDENT>return t<EOL>

serial computation time of a parallel calculation

        Computed as the sum of (completed-started) of each task


return self.timedelta(self.submitted, self.received)<EOL>

actual computation time of a parallel calculation

        Computed as the time between the latest `received` stamp
        and the earliest `submitted`.

        Only reliable if Client was spinning/waiting when the task finished, because
        the `received` timestamp is created when a result is pulled off of the zmq queue,
        which happens as a result of `client.spin()`.

        For similar comparison of other timestamp pairs, check out AsyncResult.timedelta.


def wait_interactive(self, interval=<NUM_LIT:1.>, timeout=None):

N = len(self)<EOL>tic = time.time()<EOL>while not self.ready() and (timeout is None or time.time() - tic <= timeout):<EOL><INDENT>self.wait(interval)<EOL>clear_output()<EOL>print("<STR_LIT>" % (self.progress, N, self.elapsed), end="<STR_LIT>")<EOL>sys.stdout.flush()<EOL><DEDENT>print()<EOL>print("<STR_LIT>")<EOL>

interactive wait, printing progress at regular intervals


def _republish_displaypub(self, content, eid):

try:<EOL><INDENT>ip = get_ipython()<EOL><DEDENT>except NameError:<EOL><INDENT>return<EOL><DEDENT>md = content['<STR_LIT>'] or {}<EOL>md['<STR_LIT>'] = eid<EOL>ip.display_pub.publish(content['<STR_LIT:source>'], content['<STR_LIT:data>'], md)<EOL>

republish individual displaypub content dicts


if not self._success:<EOL><INDENT>return<EOL><DEDENT>tic = time.time()<EOL>while not all(md['<STR_LIT>'] for md in self._metadata):<EOL><INDENT>time.sleep(<NUM_LIT>)<EOL>self._client._flush_iopub(self._client._iopub_socket)<EOL>if timeout >= <NUM_LIT:0> and time.time() > tic + timeout:<EOL><INDENT>break<EOL><DEDENT><DEDENT>

wait for the 'status=idle' message that indicates we have all outputs


if self._single_result:<EOL><INDENT>self._display_single_result()<EOL>return<EOL><DEDENT>stdouts = self.stdout<EOL>stderrs = self.stderr<EOL>pyouts  = self.pyout<EOL>output_lists = self.outputs<EOL>results = self.get()<EOL>targets = self.engine_id<EOL>if groupby == "<STR_LIT>":<EOL><INDENT>for eid,stdout,stderr,outputs,r,pyout in zip(<EOL>targets, stdouts, stderrs, output_lists, results, pyouts<EOL>):<EOL><INDENT>self._display_stream(stdout, '<STR_LIT>' % eid)<EOL>self._display_stream(stderr, '<STR_LIT>' % eid, file=sys.stderr)<EOL>try:<EOL><INDENT>get_ipython()<EOL><DEDENT>except NameError:<EOL><INDENT>return <EOL><DEDENT>if outputs or pyout is not None:<EOL><INDENT>_raw_text('<STR_LIT>' % eid)<EOL><DEDENT>for output in outputs:<EOL><INDENT>self._republish_displaypub(output, eid)<EOL><DEDENT>if pyout is not None:<EOL><INDENT>display(r)<EOL><DEDENT><DEDENT><DEDENT>elif groupby in ('<STR_LIT:type>', '<STR_LIT>'):<EOL><INDENT>for eid,stdout in zip(targets, stdouts):<EOL><INDENT>self._display_stream(stdout, '<STR_LIT>' % eid)<EOL><DEDENT>for eid,stderr in zip(targets, stderrs):<EOL><INDENT>self._display_stream(stderr, '<STR_LIT>' % eid, file=sys.stderr)<EOL><DEDENT>try:<EOL><INDENT>get_ipython()<EOL><DEDENT>except NameError:<EOL><INDENT>return<EOL><DEDENT>if groupby == '<STR_LIT>':<EOL><INDENT>output_dict = dict((eid, outputs) for eid,outputs in zip(targets, output_lists))<EOL>N = max(len(outputs) for outputs in output_lists)<EOL>for i in range(N):<EOL><INDENT>for eid in targets:<EOL><INDENT>outputs = output_dict[eid]<EOL>if len(outputs) >= N:<EOL><INDENT>_raw_text('<STR_LIT>' % eid)<EOL>self._republish_displaypub(outputs[i], eid)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for eid,outputs in zip(targets, output_lists):<EOL><INDENT>if outputs:<EOL><INDENT>_raw_text('<STR_LIT>' % eid)<EOL><DEDENT>for output in outputs:<EOL><INDENT>self._republish_displaypub(output, eid)<EOL><DEDENT><DEDENT><DEDENT>for eid,r,pyout in zip(targets, results, pyouts):<EOL><INDENT>if pyout is not None:<EOL><INDENT>display(r)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % groupby)<EOL><DEDENT>

republish the outputs of the computation

        Parameters
        ----------

        groupby : str [default: type]
            if 'type':
                Group outputs by type (show all stdout, then all stderr, etc.):

                [stdout:1] foo
                [stdout:2] foo
                [stderr:1] bar
                [stderr:2] bar
            if 'engine':
                Display outputs for each engine before moving on to the next:

                [stdout:1] foo
                [stderr:1] bar
                [stdout:2] foo
                [stderr:2] bar

            if 'order':
                Like 'type', but further collate individual displaypub
                outputs.  This is meant for cases of each command producing
                several plots, and you would like to see all of the first
                plots together, then all of the second plots, and so on.


return self._mapObject.joinPartitions(res)<EOL>

Perform the gather on the actual results.


try:<EOL><INDENT>rlist = self.get(<NUM_LIT:0>)<EOL><DEDENT>except error.TimeoutError:<EOL><INDENT>for msg_id in self.msg_ids:<EOL><INDENT>ar = AsyncResult(self._client, msg_id, self._fname)<EOL>rlist = ar.get()<EOL>try:<EOL><INDENT>for r in rlist:<EOL><INDENT>yield r<EOL><DEDENT><DEDENT>except TypeError:<EOL><INDENT>yield rlist<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for r in rlist:<EOL><INDENT>yield r<EOL><DEDENT><DEDENT>

iterator for results *as they arrive*, preserving submission order.


try:<EOL><INDENT>rlist = self.get(<NUM_LIT:0>)<EOL><DEDENT>except error.TimeoutError:<EOL><INDENT>pending = set(self.msg_ids)<EOL>while pending:<EOL><INDENT>try:<EOL><INDENT>self._client.wait(pending, <NUM_LIT>)<EOL><DEDENT>except error.TimeoutError:<EOL><INDENT>pass<EOL><DEDENT>ready = pending.difference(self._client.outstanding)<EOL>pending = pending.difference(ready)<EOL>while ready:<EOL><INDENT>msg_id = ready.pop()<EOL>ar = AsyncResult(self._client, msg_id, self._fname)<EOL>rlist = ar.get()<EOL>try:<EOL><INDENT>for r in rlist:<EOL><INDENT>yield r<EOL><DEDENT><DEDENT>except TypeError:<EOL><INDENT>yield rlist<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for r in rlist:<EOL><INDENT>yield r<EOL><DEDENT><DEDENT>

iterator for results *as they arrive*, on FCFS basis, ignoring submission order.


return<EOL>

no-op, because HubResults are never incomplete


start = time.time()<EOL>if self._ready:<EOL><INDENT>return<EOL><DEDENT>local_ids = [msg_id for msg_id in self.msg_ids if msg_id in self._client.outstanding]<EOL>local_ready = self._client.wait(local_ids, timeout)<EOL>if local_ready:<EOL><INDENT>remote_ids = [msg_id for msg_id in self.msg_ids if msg_id not in self._client.results]<EOL>if not remote_ids:<EOL><INDENT>self._ready = True<EOL><DEDENT>else:<EOL><INDENT>rdict = self._client.result_status(remote_ids, status_only=False)<EOL>pending = rdict['<STR_LIT>']<EOL>while pending and (timeout < <NUM_LIT:0> or time.time() < start+timeout):<EOL><INDENT>rdict = self._client.result_status(remote_ids, status_only=False)<EOL>pending = rdict['<STR_LIT>']<EOL>if pending:<EOL><INDENT>time.sleep(<NUM_LIT:0.1>)<EOL><DEDENT><DEDENT>if not pending:<EOL><INDENT>self._ready = True<EOL><DEDENT><DEDENT><DEDENT>if self._ready:<EOL><INDENT>try:<EOL><INDENT>results = list(map(self._client.results.get, self.msg_ids))<EOL>self._result = results<EOL>if self._single_result:<EOL><INDENT>r = results[<NUM_LIT:0>]<EOL>if isinstance(r, Exception):<EOL><INDENT>raise r<EOL><DEDENT><DEDENT>else:<EOL><INDENT>results = error.collect_exceptions(results, self._fname)<EOL><DEDENT>self._result = self._reconstruct_result(results)<EOL><DEDENT>except Exception as e:<EOL><INDENT>self._exception = e<EOL>self._success = False<EOL><DEDENT>else:<EOL><INDENT>self._success = True<EOL><DEDENT>finally:<EOL><INDENT>self._metadata = list(map(self._client.metadata.get, self.msg_ids))<EOL><DEDENT><DEDENT>


@decorator<EOL>def spin_first(f, self, *args, **kwargs):

self.spin()<EOL>return f(self, *args, **kwargs)<EOL>

Call spin() to sync state prior to calling the method.


if key in iter(self.keys()):<EOL><INDENT>return self[key]<EOL><DEDENT>else:<EOL><INDENT>raise AttributeError(key)<EOL><DEDENT>


if key in iter(self.keys()):<EOL><INDENT>self[key] = value<EOL><DEDENT>else:<EOL><INDENT>raise AttributeError(key)<EOL><DEDENT>


if key in iter(self.keys()):<EOL><INDENT>dict.__setitem__(self, key, value)<EOL><DEDENT>else:<EOL><INDENT>raise KeyError(key)<EOL><DEDENT>


self.close()<EOL>


for k,v in engines.items():<EOL><INDENT>eid = int(k)<EOL>self._engines[eid] = v<EOL>self._ids.append(eid)<EOL><DEDENT>self._ids = sorted(self._ids)<EOL>if sorted(self._engines.keys()) != list(range(len(self._engines))) andself._task_scheme == '<STR_LIT>' and self._task_socket:<EOL><INDENT>self._stop_scheduling_tasks()<EOL><DEDENT>

Update our engines dict and _ids from a dict of the form: {id:uuid}.


self._task_socket.close()<EOL>self._task_socket = None<EOL>msg = "<STR_LIT>" +"<STR_LIT>"<EOL>if self.outstanding:<EOL><INDENT>msg += "<STR_LIT>" +"<STR_LIT>"<EOL><DEDENT>warnings.warn(msg, RuntimeWarning)<EOL>

Stop scheduling tasks because an engine has been unregistered
        from a pure ZMQ scheduler.


if not self._ids:<EOL><INDENT>if not self.ids:<EOL><INDENT>raise error.NoEnginesRegistered("<STR_LIT>")<EOL><DEDENT><DEDENT>if targets is None:<EOL><INDENT>targets = self._ids<EOL><DEDENT>elif isinstance(targets, str):<EOL><INDENT>if targets.lower() == '<STR_LIT:all>':<EOL><INDENT>targets = self._ids<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>"%(targets))<EOL><DEDENT><DEDENT>elif isinstance(targets, int):<EOL><INDENT>if targets < <NUM_LIT:0>:<EOL><INDENT>targets = self.ids[targets]<EOL><DEDENT>if targets not in self._ids:<EOL><INDENT>raise IndexError("<STR_LIT>"%targets)<EOL><DEDENT>targets = [targets]<EOL><DEDENT>if isinstance(targets, slice):<EOL><INDENT>indices = list(range(len(self._ids)))[targets]<EOL>ids = self.ids<EOL>targets = [ ids[i] for i in indices ]<EOL><DEDENT>if not isinstance(targets, (tuple, list, xrange)):<EOL><INDENT>raise TypeError("<STR_LIT>"%(type(targets)))<EOL><DEDENT>return [cast_bytes(self._engines[t]) for t in targets], list(targets)<EOL>

Turn valid target IDs or 'all' into two lists:
        (int_ids, uuids).


def _connect(self, sshserver, ssh_kwargs, timeout):

<EOL>if self._connected:<EOL><INDENT>return<EOL><DEDENT>self._connected=True<EOL>def connect_socket(s, url):<EOL><INDENT>url = util.disambiguate_url(url, self._config['<STR_LIT:location>'])<EOL>if self._ssh:<EOL><INDENT>return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs)<EOL><DEDENT>else:<EOL><INDENT>return s.connect(url)<EOL><DEDENT><DEDENT>self.session.send(self._query_socket, '<STR_LIT>')<EOL>poller = zmq.Poller()<EOL>poller.register(self._query_socket, zmq.POLLIN)<EOL>evts = poller.poll(timeout*<NUM_LIT:1000>)<EOL>if not evts:<EOL><INDENT>raise error.TimeoutError("<STR_LIT>")<EOL><DEDENT>idents,msg = self.session.recv(self._query_socket,mode=<NUM_LIT:0>)<EOL>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>msg = Message(msg)<EOL>content = msg.content<EOL>self._config['<STR_LIT>'] = dict(content)<EOL>if content.status == '<STR_LIT>':<EOL><INDENT>ident = self.session.bsession<EOL>if content.mux:<EOL><INDENT>self._mux_socket = self._context.socket(zmq.DEALER)<EOL>self._mux_socket.setsockopt(zmq.IDENTITY, ident)<EOL>connect_socket(self._mux_socket, content.mux)<EOL><DEDENT>if content.task:<EOL><INDENT>self._task_scheme, task_addr = content.task<EOL>self._task_socket = self._context.socket(zmq.DEALER)<EOL>self._task_socket.setsockopt(zmq.IDENTITY, ident)<EOL>connect_socket(self._task_socket, task_addr)<EOL><DEDENT>if content.notification:<EOL><INDENT>self._notification_socket = self._context.socket(zmq.SUB)<EOL>connect_socket(self._notification_socket, content.notification)<EOL>self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'<STR_LIT>')<EOL><DEDENT>if content.control:<EOL><INDENT>self._control_socket = self._context.socket(zmq.DEALER)<EOL>self._control_socket.setsockopt(zmq.IDENTITY, ident)<EOL>connect_socket(self._control_socket, content.control)<EOL><DEDENT>if content.iopub:<EOL><INDENT>self._iopub_socket = self._context.socket(zmq.SUB)<EOL>self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'<STR_LIT>')<EOL>self._iopub_socket.setsockopt(zmq.IDENTITY, ident)<EOL>connect_socket(self._iopub_socket, content.iopub)<EOL><DEDENT>self._update_engines(dict(content.engines))<EOL><DEDENT>else:<EOL><INDENT>self._connected = False<EOL>raise Exception("<STR_LIT>")<EOL><DEDENT>

setup all our socket connections to the cluster. This is called from
        __init__.


e = error.unwrap_exception(content)<EOL>if e.engine_info:<EOL><INDENT>e_uuid = e.engine_info['<STR_LIT>']<EOL>eid = self._engines[e_uuid]<EOL>e.engine_info['<STR_LIT>'] = eid<EOL><DEDENT>return e<EOL>

unwrap exception, and remap engine_id to int.


content = msg['<STR_LIT:content>']<EOL>eid = content['<STR_LIT:id>']<EOL>d = {eid : content['<STR_LIT>']}<EOL>self._update_engines(d)<EOL>

Register a new engine, and update our connection info.


content = msg['<STR_LIT:content>']<EOL>eid = int(content['<STR_LIT:id>'])<EOL>if eid in self._ids:<EOL><INDENT>self._ids.remove(eid)<EOL>uuid = self._engines.pop(eid)<EOL>self._handle_stranded_msgs(eid, uuid)<EOL><DEDENT>if self._task_socket and self._task_scheme == '<STR_LIT>':<EOL><INDENT>self._stop_scheduling_tasks()<EOL><DEDENT>


def _handle_stranded_msgs(self, eid, uuid):

outstanding = self._outstanding_dict[uuid]<EOL>for msg_id in list(outstanding):<EOL><INDENT>if msg_id in self.results:<EOL><INDENT>continue<EOL><DEDENT>try:<EOL><INDENT>raise error.EngineError("<STR_LIT>"%(eid, msg_id))<EOL><DEDENT>except:<EOL><INDENT>content = error.wrap_exception()<EOL><DEDENT>parent = {}<EOL>header = {}<EOL>parent['<STR_LIT>'] = msg_id<EOL>header['<STR_LIT>'] = uuid<EOL>header['<STR_LIT:date>'] = datetime.now()<EOL>msg = dict(parent_header=parent, header=header, content=content)<EOL>self._handle_apply_reply(msg)<EOL><DEDENT>

Handle messages known to be on an engine when the engine unregisters.

        It is possible that this will fire prematurely - that is, an engine will
        go down after completing a result, and the client will be notified
        of the unregistration and later receive the successful result.


parent = msg['<STR_LIT>']<EOL>msg_id = parent['<STR_LIT>']<EOL>if msg_id not in self.outstanding:<EOL><INDENT>if msg_id in self.history:<EOL><INDENT>print(("<STR_LIT>"%msg_id))<EOL><DEDENT>else:<EOL><INDENT>print(("<STR_LIT>"%msg_id))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.outstanding.remove(msg_id)<EOL><DEDENT>content = msg['<STR_LIT:content>']<EOL>header = msg['<STR_LIT>']<EOL>md = self.metadata[msg_id]<EOL>md.update(self._extract_metadata(header, parent, content))<EOL>self.metadata[msg_id] = md<EOL>e_outstanding = self._outstanding_dict[md['<STR_LIT>']]<EOL>if msg_id in e_outstanding:<EOL><INDENT>e_outstanding.remove(msg_id)<EOL><DEDENT>if content['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>self.results[msg_id] = ExecuteReply(msg_id, content, md)<EOL><DEDENT>elif content['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>self.results[msg_id] = error.TaskAborted(msg_id)<EOL><DEDENT>elif content['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>self.results[msg_id] = self._unwrap_exception(content)<EOL><DEDENT>

Save the reply to an execute_request into our results.

        execute messages are never actually used. apply is used instead.


parent = msg['<STR_LIT>']<EOL>msg_id = parent['<STR_LIT>']<EOL>if msg_id not in self.outstanding:<EOL><INDENT>if msg_id in self.history:<EOL><INDENT>print(("<STR_LIT>"%msg_id))<EOL>print(self.results[msg_id])<EOL>print(msg)<EOL><DEDENT>else:<EOL><INDENT>print(("<STR_LIT>"%msg_id))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.outstanding.remove(msg_id)<EOL><DEDENT>content = msg['<STR_LIT:content>']<EOL>header = msg['<STR_LIT>']<EOL>md = self.metadata[msg_id]<EOL>md.update(self._extract_metadata(header, parent, content))<EOL>self.metadata[msg_id] = md<EOL>e_outstanding = self._outstanding_dict[md['<STR_LIT>']]<EOL>if msg_id in e_outstanding:<EOL><INDENT>e_outstanding.remove(msg_id)<EOL><DEDENT>if content['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>self.results[msg_id] = util.unserialize_object(msg['<STR_LIT>'])[<NUM_LIT:0>]<EOL><DEDENT>elif content['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>self.results[msg_id] = error.TaskAborted(msg_id)<EOL><DEDENT>elif content['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>self.results[msg_id] = self._unwrap_exception(content)<EOL><DEDENT>

Save the reply to an apply_request into our results.


idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)<EOL>while msg is not None:<EOL><INDENT>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>msg_type = msg['<STR_LIT>']['<STR_LIT>']<EOL>handler = self._notification_handlers.get(msg_type, None)<EOL>if handler is None:<EOL><INDENT>raise Exception("<STR_LIT>"%msg.msg_type)<EOL><DEDENT>else:<EOL><INDENT>handler(msg)<EOL><DEDENT>idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)<EOL><DEDENT>

Flush notifications of engine registrations waiting
        in ZMQ queue.


idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)<EOL>while msg is not None:<EOL><INDENT>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>msg_type = msg['<STR_LIT>']['<STR_LIT>']<EOL>handler = self._queue_handlers.get(msg_type, None)<EOL>if handler is None:<EOL><INDENT>raise Exception("<STR_LIT>"%msg.msg_type)<EOL><DEDENT>else:<EOL><INDENT>handler(msg)<EOL><DEDENT>idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)<EOL><DEDENT>

Flush task or queue results waiting in ZMQ queue.


if self._ignored_control_replies <= <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)<EOL>while msg is not None:<EOL><INDENT>self._ignored_control_replies -= <NUM_LIT:1><EOL>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)<EOL><DEDENT>

Flush replies from the control channel waiting
        in the ZMQ queue.

        Currently: ignore them.


while self._ignored_control_replies > <NUM_LIT:0>:<EOL><INDENT>self.session.recv(self._control_socket)<EOL>self._ignored_control_replies -= <NUM_LIT:1><EOL><DEDENT>


idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)<EOL>while msg is not None:<EOL><INDENT>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>parent = msg['<STR_LIT>']<EOL>if not parent:<EOL><INDENT>continue<EOL><DEDENT>msg_id = parent['<STR_LIT>']<EOL>content = msg['<STR_LIT:content>']<EOL>header = msg['<STR_LIT>']<EOL>msg_type = msg['<STR_LIT>']['<STR_LIT>']<EOL>md = self.metadata[msg_id]<EOL>if msg_type == '<STR_LIT>':<EOL><INDENT>name = content['<STR_LIT:name>']<EOL>s = md[name] or '<STR_LIT>'<EOL>md[name] = s + content['<STR_LIT:data>']<EOL><DEDENT>elif msg_type == '<STR_LIT>':<EOL><INDENT>md.update({'<STR_LIT>' : self._unwrap_exception(content)})<EOL><DEDENT>elif msg_type == '<STR_LIT>':<EOL><INDENT>md.update({'<STR_LIT>' : content['<STR_LIT:code>']})<EOL><DEDENT>elif msg_type == '<STR_LIT>':<EOL><INDENT>md['<STR_LIT>'].append(content)<EOL><DEDENT>elif msg_type == '<STR_LIT>':<EOL><INDENT>md['<STR_LIT>'] = content<EOL><DEDENT>elif msg_type == '<STR_LIT:status>':<EOL><INDENT>if content['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>md['<STR_LIT>'] = True<EOL><DEDENT><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT>self.metadata[msg_id] = md<EOL>idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)<EOL><DEDENT>

Flush replies from the iopub channel waiting
        in the ZMQ queue.


return len(self.ids)<EOL>


if not isinstance(key, (int, slice, tuple, list, xrange)):<EOL><INDENT>raise TypeError("<STR_LIT>"%(type(key)))<EOL><DEDENT>else:<EOL><INDENT>return self.direct_view(key)<EOL><DEDENT>

index access returns DirectView multiplexer objects

        Must be int, slice, or list/tuple/xrange of ints


self._flush_notifications()<EOL>return list(self._ids)<EOL>


def activate(self, targets='<STR_LIT:all>', suffix='<STR_LIT>'):

view = self.direct_view(targets)<EOL>view.block = True<EOL>view.activate(suffix)<EOL>return view<EOL>

Create a DirectView and register it with IPython magics

        Defines the magics `%px, %autopx, %pxresult, %%px`

        Parameters
        ----------

        targets: int, list of ints, or 'all'
            The engines on which the view's magics will run
        suffix: str [default: '']
            The suffix, if any, for the magics.  This allows you to have
            multiple views associated with parallel magics at the same time.

            e.g. ``rc.activate(targets=0, suffix='0')`` will give you
            the magics ``%px0``, ``%pxresult0``, etc. for running magics just
            on engine 0.


while True:<EOL><INDENT>if self._stop_spinning.is_set():<EOL><INDENT>return<EOL><DEDENT>time.sleep(interval)<EOL>self.spin()<EOL><DEDENT>


if self._spin_thread is not None:<EOL><INDENT>self.stop_spin_thread()<EOL><DEDENT>self._stop_spinning.clear()<EOL>self._spin_thread = Thread(target=self._spin_every, args=(interval,))<EOL>self._spin_thread.daemon = True<EOL>self._spin_thread.start()<EOL>

call Client.spin() in a background thread on some regular interval

        This helps ensure that messages don't pile up too much in the zmq queue
        while you are working on other things, or just leaving an idle terminal.

        It also helps limit potential padding of the `received` timestamp
        on AsyncResult objects, used for timings.

        Parameters
        ----------

        interval : float, optional
            The interval on which to spin the client in the background thread
            (simply passed to time.sleep).

        Notes
        -----

        For precision timing, you may want to use this method to put a bound
        on the jitter (in seconds) in `received` timestamps used
        in AsyncResult.wall_time.


if self._spin_thread is not None:<EOL><INDENT>self._stop_spinning.set()<EOL>self._spin_thread.join()<EOL>self._spin_thread = None<EOL><DEDENT>


Flush any registration notifications and execution results
        waiting in the ZMQ queue.


def wait(self, jobs=None, timeout=-<NUM_LIT:1>):

tic = time.time()<EOL>if jobs is None:<EOL><INDENT>theids = self.outstanding<EOL><DEDENT>else:<EOL><INDENT>if isinstance(jobs, (int, str, AsyncResult)):<EOL><INDENT>jobs = [jobs]<EOL><DEDENT>theids = set()<EOL>for job in jobs:<EOL><INDENT>if isinstance(job, int):<EOL><INDENT>job = self.history[job]<EOL><DEDENT>elif isinstance(job, AsyncResult):<EOL><INDENT>list(map(theids.add, job.msg_ids))<EOL>continue<EOL><DEDENT>theids.add(job)<EOL><DEDENT><DEDENT>if not theids.intersection(self.outstanding):<EOL><INDENT>return True<EOL><DEDENT>self.spin()<EOL>while theids.intersection(self.outstanding):<EOL><INDENT>if timeout >= <NUM_LIT:0> and ( time.time()-tic ) > timeout:<EOL><INDENT>break<EOL><DEDENT>time.sleep(<NUM_LIT>)<EOL>self.spin()<EOL><DEDENT>return len(theids.intersection(self.outstanding)) == <NUM_LIT:0><EOL>

waits on one or more `jobs`, for up to `timeout` seconds.

        Parameters
        ----------

        jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
                ints are indices to self.history
                strs are msg_ids
                default: wait on all outstanding messages
        timeout : float
                a time in seconds, after which to give up.
                default is -1, which means no timeout

        Returns
        -------

        True : when all msg_ids are done
        False : timeout reached, some msg_ids still outstanding


@spin_first<EOL><INDENT>def clear(self, targets=None, block=None):<DEDENT>

block = self.block if block is None else block<EOL>targets = self._build_targets(targets)[<NUM_LIT:0>]<EOL>for t in targets:<EOL><INDENT>self.session.send(self._control_socket, '<STR_LIT>', content={}, ident=t)<EOL><DEDENT>error = False<EOL>if block:<EOL><INDENT>self._flush_ignored_control()<EOL>for i in range(len(targets)):<EOL><INDENT>idents,msg = self.session.recv(self._control_socket,<NUM_LIT:0>)<EOL>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>if msg['<STR_LIT:content>']['<STR_LIT:status>'] != '<STR_LIT>':<EOL><INDENT>error = self._unwrap_exception(msg['<STR_LIT:content>'])<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>self._ignored_control_replies += len(targets)<EOL><DEDENT>if error:<EOL><INDENT>raise error<EOL><DEDENT>


@spin_first<EOL><INDENT>def abort(self, jobs=None, targets=None, block=None):<DEDENT>

block = self.block if block is None else block<EOL>jobs = jobs if jobs is not None else list(self.outstanding)<EOL>targets = self._build_targets(targets)[<NUM_LIT:0>]<EOL>msg_ids = []<EOL>if isinstance(jobs, (str,AsyncResult)):<EOL><INDENT>jobs = [jobs]<EOL><DEDENT>bad_ids = [obj for obj in jobs if not isinstance(obj, (str, AsyncResult))]<EOL>if bad_ids:<EOL><INDENT>raise TypeError("<STR_LIT>"%bad_ids[<NUM_LIT:0>])<EOL><DEDENT>for j in jobs:<EOL><INDENT>if isinstance(j, AsyncResult):<EOL><INDENT>msg_ids.extend(j.msg_ids)<EOL><DEDENT>else:<EOL><INDENT>msg_ids.append(j)<EOL><DEDENT><DEDENT>content = dict(msg_ids=msg_ids)<EOL>for t in targets:<EOL><INDENT>self.session.send(self._control_socket, '<STR_LIT>',<EOL>content=content, ident=t)<EOL><DEDENT>error = False<EOL>if block:<EOL><INDENT>self._flush_ignored_control()<EOL>for i in range(len(targets)):<EOL><INDENT>idents,msg = self.session.recv(self._control_socket,<NUM_LIT:0>)<EOL>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>if msg['<STR_LIT:content>']['<STR_LIT:status>'] != '<STR_LIT>':<EOL><INDENT>error = self._unwrap_exception(msg['<STR_LIT:content>'])<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>self._ignored_control_replies += len(targets)<EOL><DEDENT>if error:<EOL><INDENT>raise error<EOL><DEDENT>

Abort specific jobs from the execution queues of target(s).

        This is a mechanism to prevent jobs that have already been submitted
        from executing.

        Parameters
        ----------

        jobs : msg_id, list of msg_ids, or AsyncResult
            The jobs to be aborted

            If unspecified/None: abort all outstanding jobs.


@spin_first<EOL><INDENT>def shutdown(self, targets='<STR_LIT:all>', restart=False, hub=False, block=None):<DEDENT>

if restart:<EOL><INDENT>raise NotImplementedError("<STR_LIT>")<EOL><DEDENT>block = self.block if block is None else block<EOL>if hub:<EOL><INDENT>targets = '<STR_LIT:all>'<EOL><DEDENT>targets = self._build_targets(targets)[<NUM_LIT:0>]<EOL>for t in targets:<EOL><INDENT>self.session.send(self._control_socket, '<STR_LIT>',<EOL>content={'<STR_LIT>':restart},ident=t)<EOL><DEDENT>error = False<EOL>if block or hub:<EOL><INDENT>self._flush_ignored_control()<EOL>for i in range(len(targets)):<EOL><INDENT>idents,msg = self.session.recv(self._control_socket, <NUM_LIT:0>)<EOL>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>if msg['<STR_LIT:content>']['<STR_LIT:status>'] != '<STR_LIT>':<EOL><INDENT>error = self._unwrap_exception(msg['<STR_LIT:content>'])<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>self._ignored_control_replies += len(targets)<EOL><DEDENT>if hub:<EOL><INDENT>time.sleep(<NUM_LIT>)<EOL>self.session.send(self._query_socket, '<STR_LIT>')<EOL>idents,msg = self.session.recv(self._query_socket, <NUM_LIT:0>)<EOL>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>if msg['<STR_LIT:content>']['<STR_LIT:status>'] != '<STR_LIT>':<EOL><INDENT>error = self._unwrap_exception(msg['<STR_LIT:content>'])<EOL><DEDENT><DEDENT>if error:<EOL><INDENT>raise error<EOL><DEDENT>

Terminates one or more engine processes, optionally including the hub.

        Parameters
        ----------

        targets: list of ints or 'all' [default: all]
            Which engines to shutdown.
        hub: bool [default: False]
            Whether to include the Hub.  hub=True implies targets='all'.
        block: bool [default: self.block]
            Whether to wait for clean shutdown replies or not.
        restart: bool [default: False]
            NOT IMPLEMENTED
            whether to restart engines after shutting them down.


if isinstance(result, error.RemoteError):<EOL><INDENT>raise result<EOL><DEDENT>return result<EOL>

wrapper for maybe raising an exception if apply failed.


def send_apply_request(self, socket, f, args=None, kwargs=None, subheader=None, track=False,<EOL>ident=None):

if self._closed:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>args = args if args is not None else []<EOL>kwargs = kwargs if kwargs is not None else {}<EOL>subheader = subheader if subheader is not None else {}<EOL>if not callable(f) and not isinstance(f, Reference):<EOL><INDENT>raise TypeError("<STR_LIT>"%type(f))<EOL><DEDENT>if not isinstance(args, (tuple, list)):<EOL><INDENT>raise TypeError("<STR_LIT>"%type(args))<EOL><DEDENT>if not isinstance(kwargs, dict):<EOL><INDENT>raise TypeError("<STR_LIT>"%type(kwargs))<EOL><DEDENT>if not isinstance(subheader, dict):<EOL><INDENT>raise TypeError("<STR_LIT>"%type(subheader))<EOL><DEDENT>bufs = util.pack_apply_message(f,args,kwargs)<EOL>msg = self.session.send(socket, "<STR_LIT>", buffers=bufs, ident=ident,<EOL>subheader=subheader, track=track)<EOL>msg_id = msg['<STR_LIT>']['<STR_LIT>']<EOL>self.outstanding.add(msg_id)<EOL>if ident:<EOL><INDENT>if isinstance(ident, list):<EOL><INDENT>ident = ident[-<NUM_LIT:1>]<EOL><DEDENT>if ident in list(self._engines.values()):<EOL><INDENT>self._outstanding_dict[ident].add(msg_id)<EOL><DEDENT><DEDENT>self.history.append(msg_id)<EOL>self.metadata[msg_id]['<STR_LIT>'] = datetime.now()<EOL>return msg<EOL>

construct and send an apply message via a socket.

        This is the principal method with which all engine execution is performed by views.


def send_execute_request(self, socket, code, silent=True, subheader=None, ident=None):

if self._closed:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>subheader = subheader if subheader is not None else {}<EOL>if not isinstance(code, str):<EOL><INDENT>raise TypeError("<STR_LIT>" % type(code))<EOL><DEDENT>if not isinstance(subheader, dict):<EOL><INDENT>raise TypeError("<STR_LIT>" % type(subheader))<EOL><DEDENT>content = dict(code=code, silent=bool(silent), user_variables=[], user_expressions={})<EOL>msg = self.session.send(socket, "<STR_LIT>", content=content, ident=ident,<EOL>subheader=subheader)<EOL>msg_id = msg['<STR_LIT>']['<STR_LIT>']<EOL>self.outstanding.add(msg_id)<EOL>if ident:<EOL><INDENT>if isinstance(ident, list):<EOL><INDENT>ident = ident[-<NUM_LIT:1>]<EOL><DEDENT>if ident in list(self._engines.values()):<EOL><INDENT>self._outstanding_dict[ident].add(msg_id)<EOL><DEDENT><DEDENT>self.history.append(msg_id)<EOL>self.metadata[msg_id]['<STR_LIT>'] = datetime.now()<EOL>return msg<EOL>

construct and send an execute request via a socket.


if targets == '<STR_LIT:all>':<EOL><INDENT>targets = None<EOL><DEDENT>if targets is not None:<EOL><INDENT>targets = self._build_targets(targets)[<NUM_LIT:1>]<EOL><DEDENT>return LoadBalancedView(client=self, socket=self._task_socket, targets=targets)<EOL>

construct a DirectView object.

        If no arguments are specified, create a LoadBalancedView
        using all engines.

        Parameters
        ----------

        targets: list,slice,int,etc. [default: use all engines]
            The subset of engines across which to load-balance


single = isinstance(targets, int)<EOL>if targets != '<STR_LIT:all>':<EOL><INDENT>targets = self._build_targets(targets)[<NUM_LIT:1>]<EOL><DEDENT>if single:<EOL><INDENT>targets = targets[<NUM_LIT:0>]<EOL><DEDENT>return DirectView(client=self, socket=self._mux_socket, targets=targets)<EOL>

construct a DirectView object.

        If no targets are specified, create a DirectView using all engines.

        rc.direct_view('all') is distinguished from rc[:] in that 'all' will
        evaluate the target engines at each execution, whereas rc[:] will connect to
        all *current* engines, and that list will not change.

        That is, 'all' will always use all engines, whereas rc[:] will not use
        engines added after the DirectView is constructed.

        Parameters
        ----------

        targets: list,slice,int,etc. [default: use all engines]
            The engines to use for the View


@spin_first<EOL><INDENT>def get_result(self, indices_or_msg_ids=None, block=None):<DEDENT>

block = self.block if block is None else block<EOL>if indices_or_msg_ids is None:<EOL><INDENT>indices_or_msg_ids = -<NUM_LIT:1><EOL><DEDENT>if not isinstance(indices_or_msg_ids, (list,tuple)):<EOL><INDENT>indices_or_msg_ids = [indices_or_msg_ids]<EOL><DEDENT>theids = []<EOL>for id in indices_or_msg_ids:<EOL><INDENT>if isinstance(id, int):<EOL><INDENT>id = self.history[id]<EOL><DEDENT>if not isinstance(id, str):<EOL><INDENT>raise TypeError("<STR_LIT>"%id)<EOL><DEDENT>theids.append(id)<EOL><DEDENT>local_ids = [msg_id for msg_id in theids if msg_id in self.history or msg_id in self.results]<EOL>remote_ids = [msg_id for msg_id in theids if msg_id not in local_ids]<EOL>if remote_ids:<EOL><INDENT>ar = AsyncHubResult(self, msg_ids=theids)<EOL><DEDENT>else:<EOL><INDENT>ar = AsyncResult(self, msg_ids=theids)<EOL><DEDENT>if block:<EOL><INDENT>ar.wait()<EOL><DEDENT>return ar<EOL>

Retrieve a result by msg_id or history index, wrapped in an AsyncResult object.

        If the client already has the results, no request to the Hub will be made.

        This is a convenient way to construct AsyncResult objects, which are wrappers
        that include metadata about execution, and allow for awaiting results that
        were not submitted by this Client.

        It can also be a convenient way to retrieve the metadata associated with
        blocking execution, since it always retrieves

        Examples
        --------
        ::

            In [10]: r = client.apply()

        Parameters
        ----------

        indices_or_msg_ids : integer history index, str msg_id, or list of either
            The indices or msg_ids of indices to be retrieved

        block : bool
            Whether to wait for the result to be done

        Returns
        -------

        AsyncResult
            A single AsyncResult object will always be returned.

        AsyncHubResult
            A subclass of AsyncResult that retrieves results from the Hub


@spin_first<EOL><INDENT>def resubmit(self, indices_or_msg_ids=None, subheader=None, block=None):<DEDENT>

block = self.block if block is None else block<EOL>if indices_or_msg_ids is None:<EOL><INDENT>indices_or_msg_ids = -<NUM_LIT:1><EOL><DEDENT>if not isinstance(indices_or_msg_ids, (list,tuple)):<EOL><INDENT>indices_or_msg_ids = [indices_or_msg_ids]<EOL><DEDENT>theids = []<EOL>for id in indices_or_msg_ids:<EOL><INDENT>if isinstance(id, int):<EOL><INDENT>id = self.history[id]<EOL><DEDENT>if not isinstance(id, str):<EOL><INDENT>raise TypeError("<STR_LIT>"%id)<EOL><DEDENT>theids.append(id)<EOL><DEDENT>content = dict(msg_ids = theids)<EOL>self.session.send(self._query_socket, '<STR_LIT>', content)<EOL>zmq.select([self._query_socket], [], [])<EOL>idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)<EOL>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>content = msg['<STR_LIT:content>']<EOL>if content['<STR_LIT:status>'] != '<STR_LIT>':<EOL><INDENT>raise self._unwrap_exception(content)<EOL><DEDENT>mapping = content['<STR_LIT>']<EOL>new_ids = [ mapping[msg_id] for msg_id in theids ]<EOL>ar = AsyncHubResult(self, msg_ids=new_ids)<EOL>if block:<EOL><INDENT>ar.wait()<EOL><DEDENT>return ar<EOL>

Resubmit one or more tasks.

        in-flight tasks may not be resubmitted.

        Parameters
        ----------

        indices_or_msg_ids : integer history index, str msg_id, or list of either
            The indices or msg_ids of indices to be retrieved

        block : bool
            Whether to wait for the result to be done

        Returns
        -------

        AsyncHubResult
            A subclass of AsyncResult that retrieves results from the Hub


@spin_first<EOL><INDENT>def result_status(self, msg_ids, status_only=True):<DEDENT>

if not isinstance(msg_ids, (list,tuple)):<EOL><INDENT>msg_ids = [msg_ids]<EOL><DEDENT>theids = []<EOL>for msg_id in msg_ids:<EOL><INDENT>if isinstance(msg_id, int):<EOL><INDENT>msg_id = self.history[msg_id]<EOL><DEDENT>if not isinstance(msg_id, str):<EOL><INDENT>raise TypeError("<STR_LIT>"%msg_id)<EOL><DEDENT>theids.append(msg_id)<EOL><DEDENT>completed = []<EOL>local_results = {}<EOL>for msg_id in theids:<EOL><INDENT>if msg_id in self.results:<EOL><INDENT>completed.append(msg_id)<EOL>local_results[msg_id] = self.results[msg_id]<EOL>theids.remove(msg_id)<EOL><DEDENT><DEDENT>if theids: <EOL><INDENT>content = dict(msg_ids=theids, status_only=status_only)<EOL>msg = self.session.send(self._query_socket, "<STR_LIT>", content=content)<EOL>zmq.select([self._query_socket], [], [])<EOL>idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)<EOL>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>content = msg['<STR_LIT:content>']<EOL>if content['<STR_LIT:status>'] != '<STR_LIT>':<EOL><INDENT>raise self._unwrap_exception(content)<EOL><DEDENT>buffers = msg['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>content = dict(completed=[],pending=[])<EOL><DEDENT>content['<STR_LIT>'].extend(completed)<EOL>if status_only:<EOL><INDENT>return content<EOL><DEDENT>failures = []<EOL>content.update(local_results)<EOL>for msg_id in sorted(theids):<EOL><INDENT>if msg_id in content['<STR_LIT>']:<EOL><INDENT>rec = content[msg_id]<EOL>parent = rec['<STR_LIT>']<EOL>header = rec['<STR_LIT>']<EOL>rcontent = rec['<STR_LIT>']<EOL>iodict = rec['<STR_LIT>']<EOL>if isinstance(rcontent, str):<EOL><INDENT>rcontent = self.session.unpack(rcontent)<EOL><DEDENT>md = self.metadata[msg_id]<EOL>md.update(self._extract_metadata(header, parent, rcontent))<EOL>if rec.get('<STR_LIT>'):<EOL><INDENT>md['<STR_LIT>'] = rec['<STR_LIT>']<EOL><DEDENT>md.update(iodict)<EOL>if rcontent['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>if header['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>res,buffers = util.unserialize_object(buffers)<EOL><DEDENT>elif header['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>res = ExecuteReply(msg_id, rcontent, md)<EOL><DEDENT>else:<EOL><INDENT>raise KeyError("<STR_LIT>" % header[msg_type])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>res = self._unwrap_exception(rcontent)<EOL>failures.append(res)<EOL><DEDENT>self.results[msg_id] = res<EOL>content[msg_id] = res<EOL><DEDENT><DEDENT>if len(theids) == <NUM_LIT:1> and failures:<EOL><INDENT>raise failures[<NUM_LIT:0>]<EOL><DEDENT>error.collect_exceptions(failures, "<STR_LIT>")<EOL>return content<EOL>

Check on the status of the result(s) of the apply request with `msg_ids`.

        If status_only is False, then the actual results will be retrieved, else
        only the status of the results will be checked.

        Parameters
        ----------

        msg_ids : list of msg_ids
            if int:
                Passed as index to self.history for convenience.
        status_only : bool (default: True)
            if False:
                Retrieve the actual results of completed tasks.

        Returns
        -------

        results : dict
            There will always be the keys 'pending' and 'completed', which will
            be lists of msg_ids that are incomplete or complete. If `status_only`
            is False, then completed results will be keyed by their `msg_id`.


@spin_first<EOL><INDENT>def queue_status(self, targets='<STR_LIT:all>', verbose=False):<DEDENT>

if targets == '<STR_LIT:all>':<EOL><INDENT>engine_ids = None<EOL><DEDENT>else:<EOL><INDENT>engine_ids = self._build_targets(targets)[<NUM_LIT:1>]<EOL><DEDENT>content = dict(targets=engine_ids, verbose=verbose)<EOL>self.session.send(self._query_socket, "<STR_LIT>", content=content)<EOL>idents,msg = self.session.recv(self._query_socket, <NUM_LIT:0>)<EOL>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>content = msg['<STR_LIT:content>']<EOL>status = content.pop('<STR_LIT:status>')<EOL>if status != '<STR_LIT>':<EOL><INDENT>raise self._unwrap_exception(content)<EOL><DEDENT>content = rekey(content)<EOL>if isinstance(targets, int):<EOL><INDENT>return content[targets]<EOL><DEDENT>else:<EOL><INDENT>return content<EOL><DEDENT>

Fetch the status of engine queues.

        Parameters
        ----------

        targets : int/str/list of ints/strs
                the engines whose states are to be queried.
                default : all
        verbose : bool
                Whether to return lengths only, or lists of ids for each element


@spin_first<EOL><INDENT>def purge_results(self, jobs=[], targets=[]):<DEDENT>

if not targets and not jobs:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if targets:<EOL><INDENT>targets = self._build_targets(targets)[<NUM_LIT:1>]<EOL><DEDENT>if jobs == '<STR_LIT:all>':<EOL><INDENT>msg_ids = jobs<EOL><DEDENT>else:<EOL><INDENT>msg_ids = []<EOL>if isinstance(jobs, (str,AsyncResult)):<EOL><INDENT>jobs = [jobs]<EOL><DEDENT>bad_ids = [obj for obj in jobs if not isinstance(obj, (str, AsyncResult))]<EOL>if bad_ids:<EOL><INDENT>raise TypeError("<STR_LIT>"%bad_ids[<NUM_LIT:0>])<EOL><DEDENT>for j in jobs:<EOL><INDENT>if isinstance(j, AsyncResult):<EOL><INDENT>msg_ids.extend(j.msg_ids)<EOL><DEDENT>else:<EOL><INDENT>msg_ids.append(j)<EOL><DEDENT><DEDENT><DEDENT>content = dict(engine_ids=targets, msg_ids=msg_ids)<EOL>self.session.send(self._query_socket, "<STR_LIT>", content=content)<EOL>idents, msg = self.session.recv(self._query_socket, <NUM_LIT:0>)<EOL>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>content = msg['<STR_LIT:content>']<EOL>if content['<STR_LIT:status>'] != '<STR_LIT>':<EOL><INDENT>raise self._unwrap_exception(content)<EOL><DEDENT>

Tell the Hub to forget results.

        Individual results can be purged by msg_id, or the entire
        history of specific targets can be purged.

        Use `purge_results('all')` to scrub everything from the Hub's db.

        Parameters
        ----------

        jobs : str or list of str or AsyncResult objects
                the msg_ids whose results should be forgotten.
        targets : int/str/list of ints/strs
                The targets, by int_id, whose entire history is to be purged.

                default : None


self.session.send(self._query_socket, "<STR_LIT>", content={})<EOL>idents, msg = self.session.recv(self._query_socket, <NUM_LIT:0>)<EOL>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>content = msg['<STR_LIT:content>']<EOL>if content['<STR_LIT:status>'] != '<STR_LIT>':<EOL><INDENT>raise self._unwrap_exception(content)<EOL><DEDENT>else:<EOL><INDENT>return content['<STR_LIT>']<EOL><DEDENT>

Get the Hub's history

        Just like the Client, the Hub has a history, which is a list of msg_ids.
        This will contain the history of all clients, and, depending on configuration,
        may contain history across multiple cluster sessions.

        Any msg_id returned here is a valid argument to `get_result`.

        Returns
        -------

        msg_ids : list of strs
                list of all msg_ids, ordered by task submission time.


@spin_first<EOL><INDENT>def db_query(self, query, keys=None):<DEDENT>

if isinstance(keys, str):<EOL><INDENT>keys = [keys]<EOL><DEDENT>content = dict(query=query, keys=keys)<EOL>self.session.send(self._query_socket, "<STR_LIT>", content=content)<EOL>idents, msg = self.session.recv(self._query_socket, <NUM_LIT:0>)<EOL>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>content = msg['<STR_LIT:content>']<EOL>if content['<STR_LIT:status>'] != '<STR_LIT>':<EOL><INDENT>raise self._unwrap_exception(content)<EOL><DEDENT>records = content['<STR_LIT>']<EOL>buffer_lens = content['<STR_LIT>']<EOL>result_buffer_lens = content['<STR_LIT>']<EOL>buffers = msg['<STR_LIT>']<EOL>has_bufs = buffer_lens is not None<EOL>has_rbufs = result_buffer_lens is not None<EOL>for i,rec in enumerate(records):<EOL><INDENT>if has_bufs:<EOL><INDENT>blen = buffer_lens[i]<EOL>rec['<STR_LIT>'], buffers = buffers[:blen],buffers[blen:]<EOL><DEDENT>if has_rbufs:<EOL><INDENT>blen = result_buffer_lens[i]<EOL>rec['<STR_LIT>'], buffers = buffers[:blen],buffers[blen:]<EOL><DEDENT><DEDENT>return records<EOL>

Query the Hub's TaskRecord database

        This will return a list of task record dicts that match `query`

        Parameters
        ----------

        query : mongodb query dict
            The search dict. See mongodb query docs for details.
        keys : list of strs [optional]
            The subset of keys to be returned.  The default is to fetch everything but buffers.
            'msg_id' will *always* be included.


args = [<EOL>magic_arguments.argument('<STR_LIT>', '<STR_LIT>', action="<STR_LIT>",<EOL>const=True, dest='<STR_LIT>',<EOL>help="<STR_LIT>",<EOL>),<EOL>magic_arguments.argument('<STR_LIT>', '<STR_LIT>', action="<STR_LIT>",<EOL>const=False, dest='<STR_LIT>',<EOL>help="<STR_LIT>",<EOL>),<EOL>magic_arguments.argument('<STR_LIT>', '<STR_LIT>', type=str,<EOL>help="<STR_LIT>",<EOL>),<EOL>magic_arguments.argument('<STR_LIT>', action="<STR_LIT>",<EOL>const=True, dest="<STR_LIT>",<EOL>help="<STR_LIT>",<EOL>),<EOL>magic_arguments.argument('<STR_LIT>', action="<STR_LIT>",<EOL>const=False, dest="<STR_LIT>",<EOL>help="<STR_LIT>",<EOL>),<EOL>]<EOL>for a in args:<EOL><INDENT>f = a(f)<EOL><DEDENT>return f<EOL>

decorator for adding block/targets args for execution

    applied to %pxconfig and %%px


args = [<EOL>magic_arguments.argument('<STR_LIT>', action="<STR_LIT>", dest='<STR_LIT>',<EOL>const='<STR_LIT>',<EOL>help="<STR_LIT>"<EOL>),<EOL>magic_arguments.argument('<STR_LIT>', action="<STR_LIT>", dest='<STR_LIT>',<EOL>const='<STR_LIT>',<EOL>help="<STR_LIT>"<EOL>),<EOL>magic_arguments.argument('<STR_LIT>', dest='<STR_LIT>', type=str,<EOL>choices=['<STR_LIT>', '<STR_LIT>', '<STR_LIT:type>'], default='<STR_LIT:type>',<EOL>help="""<STR_LIT>"""<EOL>),<EOL>magic_arguments.argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT>', type=str,<EOL>help="""<STR_LIT>"""<EOL>),<EOL>]<EOL>for a in args:<EOL><INDENT>f = a(f)<EOL><DEDENT>return f<EOL>

decorator for output-formatting args

    applied to %pxresult and %%px


args = magic_arguments.parse_argstring(self.pxconfig, line)<EOL>if args.targets:<EOL><INDENT>self.view.targets = self._eval_target_str(args.targets)<EOL><DEDENT>if args.block is not None:<EOL><INDENT>self.view.block = args.block<EOL><DEDENT>if args.set_verbose is not None:<EOL><INDENT>self.verbose = args.set_verbose<EOL><DEDENT>

configure default targets/blocking for %px magics


def parallel_execute(self, cell, block=None, groupby='<STR_LIT:type>', save_name=None):

<EOL>block = self.view.block if block is None else block<EOL>base = "<STR_LIT>" if block else "<STR_LIT>"<EOL>targets = self.view.targets<EOL>if isinstance(targets, list) and len(targets) > <NUM_LIT:10>:<EOL><INDENT>str_targets = str(targets[:<NUM_LIT:4>])[:-<NUM_LIT:1>] + '<STR_LIT>' + str(targets[-<NUM_LIT:4>:])[<NUM_LIT:1>:]<EOL><DEDENT>else:<EOL><INDENT>str_targets = str(targets)<EOL><DEDENT>if self.verbose:<EOL><INDENT>print(base + "<STR_LIT>" % str_targets)<EOL><DEDENT>result = self.view.execute(cell, silent=False, block=False)<EOL>self.last_result = result<EOL>if save_name:<EOL><INDENT>self.shell.user_ns[save_name] = result<EOL><DEDENT>if block:<EOL><INDENT>result.get()<EOL>result.display_outputs(groupby)<EOL><DEDENT>else:<EOL><INDENT>return result<EOL><DEDENT>

implementation used by %px and %%parallel


<EOL>self._original_run_cell = self.shell.run_cell<EOL>self.shell.run_cell = self.pxrun_cell<EOL>self._autopx = True<EOL>print("<STR_LIT>")<EOL>

Enable %autopx mode by saving the original run_cell and installing
        pxrun_cell.


if self._autopx:<EOL><INDENT>self.shell.run_cell = self._original_run_cell<EOL>self._autopx = False<EOL>print("<STR_LIT>")<EOL><DEDENT>

Disable %autopx by restoring the original InteractiveShell.run_cell.


def pxrun_cell(self, raw_cell, store_history=False, silent=False):

if (not raw_cell) or raw_cell.isspace():<EOL><INDENT>return<EOL><DEDENT>ipself = self.shell<EOL>with ipself.builtin_trap:<EOL><INDENT>cell = ipself.prefilter_manager.prefilter_lines(raw_cell)<EOL>if store_history:<EOL><INDENT>ipself.history_manager.store_inputs(ipself.execution_count,<EOL>cell, raw_cell)<EOL><DEDENT>cell_name = ipself.compile.cache(cell, ipself.execution_count)<EOL>try:<EOL><INDENT>ast.parse(cell, filename=cell_name)<EOL><DEDENT>except (OverflowError, SyntaxError, ValueError, TypeError,<EOL>MemoryError):<EOL><INDENT>ipself.showsyntaxerror()<EOL>ipself.execution_count += <NUM_LIT:1><EOL>return None<EOL><DEDENT>except NameError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>if store_history:<EOL><INDENT>ipself.history_manager.store_output(ipself.execution_count)<EOL>ipself.execution_count += <NUM_LIT:1><EOL><DEDENT>if re.search(r'<STR_LIT>', cell):<EOL><INDENT>self._disable_autopx()<EOL>return False<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>result = self.view.execute(cell, silent=False, block=False)<EOL><DEDENT>except:<EOL><INDENT>ipself.showtraceback()<EOL>return True<EOL><DEDENT>else:<EOL><INDENT>if self.view.block:<EOL><INDENT>try:<EOL><INDENT>result.get()<EOL><DEDENT>except:<EOL><INDENT>self.shell.showtraceback()<EOL>return True<EOL><DEDENT>else:<EOL><INDENT>with ipself.builtin_trap:<EOL><INDENT>result.display_outputs()<EOL><DEDENT><DEDENT><DEDENT>return False<EOL><DEDENT><DEDENT>

drop-in replacement for InteractiveShell.run_cell.

        This executes code remotely, instead of in the local namespace.

        See InteractiveShell.run_cell for details.


if isinstance(obj, (tuple,list)):<EOL><INDENT>return True<EOL><DEDENT>for m in arrayModules:<EOL><INDENT>if isinstance(obj,m['<STR_LIT:type>']):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

return whether an object is mappable or not.


<EOL>if p<<NUM_LIT:0> or p>=q:<EOL><INDENT>print("<STR_LIT>")<EOL>return<EOL><DEDENT>remainder = len(seq)%q<EOL>basesize = len(seq)//q<EOL>hi = []<EOL>lo = []<EOL>for n in range(q):<EOL><INDENT>if n < remainder:<EOL><INDENT>lo.append(n * (basesize + <NUM_LIT:1>))<EOL>hi.append(lo[-<NUM_LIT:1>] + basesize + <NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>lo.append(n*basesize + remainder)<EOL>hi.append(lo[-<NUM_LIT:1>] + basesize)<EOL><DEDENT><DEDENT>try:<EOL><INDENT>result = seq[lo[p]:hi[p]]<EOL><DEDENT>except TypeError:<EOL><INDENT>result = list(islice(seq, lo[p], hi[p]))<EOL><DEDENT>return result<EOL>

Returns the pth partition of q partitions of seq.


try:<EOL><INDENT>return f.__name__<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT>try:<EOL><INDENT>return f.name<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT>return str(f)<EOL>

Get the name of an object.

    For use in case of callables that are not functions, and
    thus may not have __name__ defined.

    Order: f.__name__ >  f.name > str(f)


<EOL>self._map = True<EOL>try:<EOL><INDENT>ret = self.__call__(*sequences)<EOL><DEDENT>finally:<EOL><INDENT>del self._map<EOL><DEDENT>return ret<EOL>

call a function on each element of a sequence remotely.
        This should behave very much like the builtin map, but return an AsyncMapResult
        if self.block is False.


from IPython.zmq.ipkernel import IPKernelApp<EOL>from IPython.parallel.apps.ipengineapp import IPEngineApp<EOL>if IPKernelApp.initialized() and isinstance(IPKernelApp._instance, IPKernelApp):<EOL><INDENT>return<EOL><DEDENT>if IPEngineApp.initialized():<EOL><INDENT>try:<EOL><INDENT>app = IPEngineApp.instance()<EOL><DEDENT>except MultipleInstanceError:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>return app.bind_kernel(**kwargs)<EOL><DEDENT><DEDENT>raise RuntimeError("<STR_LIT>")<EOL>

Bind an Engine's Kernel to be used as a full IPython kernel.

    This allows a running Engine to be used simultaneously as a full IPython kernel
    with the QtConsole or other frontends.

    This function returns immediately.


elist = []<EOL>if isinstance(rdict_or_list, dict):<EOL><INDENT>rlist = rdict_or_list.values()<EOL><DEDENT>else:<EOL><INDENT>rlist = rdict_or_list<EOL><DEDENT>for r in rlist:<EOL><INDENT>if isinstance(r, RemoteError):<EOL><INDENT>en, ev, etb, ei = r.ename, r.evalue, r.traceback, r.engine_info<EOL>if en=='<STR_LIT>':<EOL><INDENT>for e in ev.elist:<EOL><INDENT>elist.append(e)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>elist.append((en, ev, etb, ei))<EOL><DEDENT><DEDENT><DEDENT>if len(elist)==<NUM_LIT:0>:<EOL><INDENT>return rdict_or_list<EOL><DEDENT>else:<EOL><INDENT>msg = "<STR_LIT>" % (method)<EOL>try:<EOL><INDENT>raise CompositeError(msg, elist)<EOL><DEDENT>except CompositeError as e:<EOL><INDENT>raise e<EOL><DEDENT><DEDENT>

check a result dict for errors, and raise CompositeError if any exist.
    Passthrough otherwise.


return (self.traceback or "<STR_LIT>").splitlines()<EOL>


return self.render_traceback()<EOL>

Special method for custom tracebacks within IPython.

        This will be called by IPython instead of displaying the local traceback.

        It should return a traceback rendered as a list of lines.


print('<STR_LIT:\n>'.join(self.render_traceback()))<EOL>


lines = []<EOL>if excid is None:<EOL><INDENT>for (en,ev,etb,ei) in self.elist:<EOL><INDENT>lines.append(self._get_engine_str(ei))<EOL>lines.extend((etb or '<STR_LIT>').splitlines())<EOL>lines.append('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>en,ev,etb,ei = self.elist[excid]<EOL><DEDENT>except:<EOL><INDENT>raise IndexError("<STR_LIT>"%excid)<EOL><DEDENT>else:<EOL><INDENT>lines.append(self._get_engine_str(ei))<EOL>lines.extend((etb or '<STR_LIT>').splitlines())<EOL><DEDENT><DEDENT>return lines<EOL>

render one or all of my tracebacks to a list of lines


if self.url:<EOL><INDENT>iface = self.url.split('<STR_LIT>',<NUM_LIT:1>)<EOL>if len(iface) == <NUM_LIT:2>:<EOL><INDENT>self.transport,iface = iface<EOL><DEDENT>iface = iface.split('<STR_LIT::>')<EOL>self.ip = iface[<NUM_LIT:0>]<EOL>if iface[<NUM_LIT:1>]:<EOL><INDENT>self.regport = int(iface[<NUM_LIT:1>])<EOL><DEDENT><DEDENT>

Ensure self.url contains full transport://interface:port


warnings.warn("<STR_LIT>"<EOL>"<STR_LIT>",<EOL>DeprecationWarning, stacklevel=<NUM_LIT:2>)<EOL>return base[:<NUM_LIT:1>] == "<STR_LIT:_>" and not base[:<NUM_LIT:2>] == "<STR_LIT>" == base[-<NUM_LIT:2>:]<EOL>

prefix, base -> true iff name prefix + "." + base is "private".

    Prefix may be an empty string, and base does not contain a period.
    Prefix is ignored (although functions you write conforming to this
    protocol may make use of it).
    Return true iff base begins with an (at least one) underscore, but
    does not both begin and end with (at least) two underscores.

    >>> is_private("a.b", "my_func")
    False
    >>> is_private("____", "_my_func")
    True
    >>> is_private("someclass", "__init__")
    False
    >>> is_private("sometypo", "__init_")
    True
    >>> is_private("x.y.z", "_")
    True
    >>> is_private("_x.y.z", "__")
    False
    >>> is_private("", "")  # senseless but consistent
    False


flags = <NUM_LIT:0><EOL>for fname in __future__.all_feature_names:<EOL><INDENT>feature = globs.get(fname, None)<EOL>if feature is getattr(__future__, fname):<EOL><INDENT>flags |= feature.compiler_flag<EOL><DEDENT><DEDENT>return flags<EOL>

Return the compiler-flags associated with the future features that
have been imported into the given namespace (globs).


if inspect.ismodule(module):<EOL><INDENT>return module<EOL><DEDENT>elif isinstance(module, str):<EOL><INDENT>return __import__(module, globals(), locals(), ["<STR_LIT:*>"])<EOL><DEDENT>elif module is None:<EOL><INDENT>return sys.modules[sys._getframe(depth).f_globals['<STR_LIT>']]<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>

Return the module specified by `module`.  In particular:
  - If `module` is a module, then return module.
  - If `module` is a string, then import and return the
    module with that name.
  - If `module` is None, then return the calling module.
    The calling module is assumed to be the module of
    the stack frame at the given depth in the call stack.


<EOL>return re.sub('<STR_LIT>', indent*'<STR_LIT:U+0020>', s)<EOL>

Add the given number of space characters to the beginning every
non-blank line in `s`, and return the result.


<EOL>excout = StringIO()<EOL>exc_type, exc_val, exc_tb = exc_info<EOL>traceback.print_exception(exc_type, exc_val, exc_tb, file=excout)<EOL>return excout.getvalue()<EOL>

Return a string containing a traceback message for the given
exc_info tuple (as returned by sys.exc_info()).


if want.find(ELLIPSIS_MARKER)==-<NUM_LIT:1>:<EOL><INDENT>return want == got<EOL><DEDENT>ws = want.split(ELLIPSIS_MARKER)<EOL>assert len(ws) >= <NUM_LIT:2><EOL>startpos, endpos = <NUM_LIT:0>, len(got)<EOL>w = ws[<NUM_LIT:0>]<EOL>if w:   <EOL><INDENT>if got.startswith(w):<EOL><INDENT>startpos = len(w)<EOL>del ws[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>w = ws[-<NUM_LIT:1>]<EOL>if w:   <EOL><INDENT>if got.endswith(w):<EOL><INDENT>endpos -= len(w)<EOL>del ws[-<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>if startpos > endpos:<EOL><INDENT>return False<EOL><DEDENT>for w in ws:<EOL><INDENT>startpos = got.find(w, startpos, endpos)<EOL>if startpos < <NUM_LIT:0>:<EOL><INDENT>return False<EOL><DEDENT>startpos += len(w)<EOL><DEDENT>return True<EOL>

Essentially the only subtle case:
>>> _ellipsis_match('aa...aa', 'aaa')
False


line = line.rstrip()<EOL>if line:<EOL><INDENT>return '<STR_LIT>'+line<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT:#>'<EOL><DEDENT>

Return a commented form of the given line