self.log.debug("<STR_LIT>", msg.get('<STR_LIT:content>', '<STR_LIT>'))<EOL>if not self._hidden and not self._is_from_this_session(msg):<EOL><INDENT>if self._local_kernel:<EOL><INDENT>if not msg['<STR_LIT:content>']['<STR_LIT>']:<EOL><INDENT>self.exit_requested.emit(self)<EOL><DEDENT>else:<EOL><INDENT>time.sleep(<NUM_LIT>) <EOL>self.reset()<EOL><DEDENT><DEDENT>else: <EOL><INDENT>title = self.window().windowTitle()<EOL>if not msg['<STR_LIT:content>']['<STR_LIT>']:<EOL><INDENT>reply = QtGui.QMessageBox.question(self, title,<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>QtGui.QMessageBox.Yes,QtGui.QMessageBox.No)<EOL>if reply == QtGui.QMessageBox.Yes:<EOL><INDENT>self.exit_requested.emit(self)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>reply = QtGui.QMessageBox.question(self, title,<EOL>"<STR_LIT>",<EOL>QtGui.QMessageBox.Yes,QtGui.QMessageBox.No)<EOL>if reply == QtGui.QMessageBox.Yes:<EOL><INDENT>time.sleep(<NUM_LIT>) <EOL>self.reset()<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

Handle shutdown signal, only if from other console.


self.reset(clear=True)<EOL>

Called when the KernelManager channels have started listening or
            when the frontend is assigned an already listening KernelManager.


self.log.debug("<STR_LIT>", msg.get('<STR_LIT:content>', '<STR_LIT>'))<EOL>if not self._hidden and self._is_from_this_session(msg):<EOL><INDENT>text = msg['<STR_LIT:content>']['<STR_LIT:data>']<EOL>self._append_plain_text(text + '<STR_LIT:\n>', before_prompt=True)<EOL><DEDENT>


self._append_plain_text("<STR_LIT>")<EOL>

Process a reply for an aborted execution request.


def _is_complete(self, source, interactive):

complete = self._input_splitter.push(source)<EOL>if interactive:<EOL><INDENT>complete = not self._input_splitter.push_accepts_more()<EOL><DEDENT>return complete<EOL>

Returns whether 'source' can be completely processed and a new
            prompt created. When triggered by an Enter/Return key press,
            'interactive' is True; otherwise, it is False.


Copy the currently selected text to the clipboard without attempting
            to remove prompts or otherwise alter the text.


payload = msg['<STR_LIT:content>']['<STR_LIT>']<EOL>for item in payload:<EOL><INDENT>if not self._process_execute_payload(item):<EOL><INDENT>warning = '<STR_LIT>'<EOL>print(warning % repr(item['<STR_LIT:source>']))<EOL><DEDENT><DEDENT>

Process a reply for a successful execution request.


def _document_contents_change(self, position, removed, added):

<EOL>position += added<EOL>document = self._control.document()<EOL>if position == self._get_cursor().position():<EOL><INDENT>self._call_tip()<EOL><DEDENT>

Called whenever the document's content changes. Display a call tip
            if appropriate.


banner = '<STR_LIT>''<STR_LIT>'<EOL>return banner % (sys.version, sys.platform)<EOL>


if self._executing:<EOL><INDENT>self._executing = False<EOL>self._request_info['<STR_LIT>'] = {}<EOL><DEDENT>self._reading = False<EOL>self._highlighter.highlighting_on = False<EOL>if self.clear_on_kernel_restart or clear:<EOL><INDENT>self._control.clear()<EOL>self._append_plain_text(self.banner)<EOL><DEDENT>else:<EOL><INDENT>self._append_plain_text("<STR_LIT>")<EOL>self._append_html("<STR_LIT>")<EOL><DEDENT>self._append_before_prompt_pos = self._get_cursor().position()<EOL>self._show_interpreter_prompt()<EOL>

Resets the widget to its initial state if ``clear`` parameter or
        ``clear_on_kernel_restart`` configuration setting is True, otherwise
        prints a visual indication of the fact that the kernel restarted, but
        does not clear the traces from previous usage of the kernel before it
        was restarted.  With ``clear=True``, it is similar to ``%clear``, but
        also re-writes the banner and aborts execution if necessary.


<EOL>return False<EOL>

Process a single payload item from the list of payload items in an
            execution reply. Returns whether the payload was handled.


if not items :<EOL><INDENT>return<EOL><DEDENT>self.cancel_completion()<EOL>strng = text.columnize(items)<EOL>self._console_widget._fill_temporary_buffer(cursor, strng, html=False)<EOL>

Shows the completion widget with 'items' at the position specified
            by 'cursor'.


signal.signal(signal.SIGINT, lambda sig, frame: self.exit(-<NUM_LIT:2>))<EOL>timer = QtCore.QTimer()<EOL>timer.timeout.connect(lambda: None)<EOL>timer.start(<NUM_LIT:200>)<EOL>self._sigint_timer = timer<EOL>


text_edit = self._text_edit<EOL>point = text_edit.cursorRect(cursor).bottomRight()<EOL>point = text_edit.mapToGlobal(point)<EOL>height = self.sizeHint().height()<EOL>screen_rect = QtGui.QApplication.desktop().availableGeometry(self)<EOL>if screen_rect.size().height() - point.y() - height < <NUM_LIT:0>:<EOL><INDENT>point = text_edit.mapToGlobal(text_edit.cursorRect().topRight())<EOL>point.setY(point.y() - height)<EOL><DEDENT>self.move(point)<EOL>self._start_position = cursor.position()<EOL>self.clear()<EOL>self.addItems(items)<EOL>self.setCurrentRow(<NUM_LIT:0>)<EOL>self.show()<EOL>


super(CompletionWidget, self).hideEvent(event)<EOL>self._text_edit.cursorPositionChanged.disconnect(self._update_current)<EOL>self._text_edit.removeEventFilter(self)<EOL>

Reimplemented to disconnect signal handlers and event filter.


self.lexer = lexer<EOL>

Create a CompletionLexer using the specified Pygments lexer.


prev_data = self.currentBlock().previous().userData()<EOL>if prev_data is not None:<EOL><INDENT>self._lexer._saved_state_stack = prev_data.syntax_stack<EOL><DEDENT>elif hasattr(self._lexer, '<STR_LIT>'):<EOL><INDENT>del self._lexer._saved_state_stack<EOL><DEDENT>index = <NUM_LIT:0><EOL>for token, text in self._lexer.get_tokens(string):<EOL><INDENT>length = len(text)<EOL>self.setFormat(index, length, self._get_format(token))<EOL>index += length<EOL><DEDENT>if hasattr(self._lexer, '<STR_LIT>'):<EOL><INDENT>data = PygmentsBlockUserData(<EOL>syntax_stack=self._lexer._saved_state_stack)<EOL>self.currentBlock().setUserData(data)<EOL>del self._lexer._saved_state_stack<EOL><DEDENT>


def default_image_tag(match, path = None, format = "<STR_LIT>"):

return '<STR_LIT>'<EOL>

Return (X)HTML mark-up for the image-tag given by match.

    This default implementation merely removes the image, and exists mostly
    for documentation purposes. More information than is present in the Qt
    HTML is required to supply the images.

    Parameters
    ----------
    match : re.SRE_Match
        A match to an HTML image tag as exported by Qt, with match.group("Name")
        containing the matched image ID.

    path : string|None, optional [default None]
        If not None, specifies a path to which supporting files may be written
        (e.g., for linked images).  If None, all images are to be included
        inline.

    format : "png"|"svg", optional [default "png"]
        Format for returned or referenced images.


if py3compat.PY3:<EOL><INDENT>return image_tag<EOL><DEDENT>def utf8_image_tag(*args, **kwargs):<EOL><INDENT>s = image_tag(*args, **kwargs)<EOL>if isinstance(s, str):<EOL><INDENT>s = s.encode('<STR_LIT:utf8>')<EOL><DEDENT>return s<EOL><DEDENT>return utf8_image_tag<EOL>

wrapper for ensuring image_tag returns utf8-encoded str on Python 2


def export_xhtml(html, filename, image_tag=None):

if image_tag is None:<EOL><INDENT>image_tag = default_image_tag<EOL><DEDENT>else:<EOL><INDENT>image_tag = ensure_utf8(image_tag)<EOL><DEDENT>with open(filename, '<STR_LIT:w>') as f:<EOL><INDENT>offset = html.find("<STR_LIT>")<EOL>assert offset > -<NUM_LIT:1>, '<STR_LIT>'<EOL>html = ('<STR_LIT>'+<EOL>html[offset+<NUM_LIT:6>:])<EOL>html = fix_html(html)<EOL>f.write(IMG_RE.sub(lambda x: image_tag(x, path = None, format = "<STR_LIT>"),<EOL>html))<EOL><DEDENT>

Export the contents of the ConsoleWidget as XHTML with inline SVGs.

    Parameters:
    -----------
    html : str,
        A utf-8 encoded Python string containing the Qt HTML to export.

    filename : str
        The file to be saved.

    image_tag : callable, optional (default None)
        Used to convert images. See ``default_image_tag()`` for information.


msg_type = msg['<STR_LIT>']['<STR_LIT>']<EOL>handler = getattr(self, '<STR_LIT>' + msg_type, None)<EOL>if handler:<EOL><INDENT>handler(msg)<EOL><DEDENT>

Calls the frontend handler associated with the message type of the
            given message.


Called when the KernelManager starts (or restarts) the kernel subprocess.
        Channels may or may not be running at this point.


session = self._kernel_manager.session.session<EOL>parent = msg['<STR_LIT>']<EOL>if not parent:<EOL><INDENT>return True<EOL><DEDENT>else:<EOL><INDENT>return parent.get('<STR_LIT>') == session<EOL><DEDENT>

Returns whether a reply from the kernel originated from a request
            from this frontend.


if not isinstance(url, str):<EOL><INDENT>raise TypeError("<STR_LIT>"%type(url))<EOL><DEDENT>url = url.lower()<EOL>proto_addr = url.split('<STR_LIT>')<EOL>assert len(proto_addr) == <NUM_LIT:2>, '<STR_LIT>'%url<EOL>proto, addr = proto_addr<EOL>assert proto in ['<STR_LIT>','<STR_LIT>','<STR_LIT>','<STR_LIT>','<STR_LIT>'], "<STR_LIT>"%proto<EOL>pat = re.compile(r'<STR_LIT>')<EOL>if proto == '<STR_LIT>':<EOL><INDENT>lis = addr.split('<STR_LIT::>')<EOL>assert len(lis) == <NUM_LIT:2>, '<STR_LIT>'%url<EOL>addr,s_port = lis<EOL>try:<EOL><INDENT>port = int(s_port)<EOL><DEDENT>except ValueError:<EOL><INDENT>raise AssertionError("<STR_LIT>"%(port, url))<EOL><DEDENT>assert addr == '<STR_LIT:*>' or pat.match(addr) is not None, '<STR_LIT>'%url<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT>return True<EOL>


if hasattr(dict, key):<EOL><INDENT>raise KeyError("<STR_LIT>"%key)<EOL><DEDENT>self[key] = value<EOL>


exec(code, globals())<EOL>

helper method for implementing `client.execute` via `client.apply`


user_ns = globals()<EOL>if isinstance(keys, (list,tuple, set)):<EOL><INDENT>for key in keys:<EOL><INDENT>if key not in user_ns:<EOL><INDENT>raise NameError("<STR_LIT>"%key)<EOL><DEDENT><DEDENT>return list(map(user_ns.get, keys))<EOL><DEDENT>else:<EOL><INDENT>if keys not in user_ns:<EOL><INDENT>raise NameError("<STR_LIT>"%keys)<EOL><DEDENT>return user_ns.get(keys)<EOL><DEDENT>

helper method for implementing `client.pull` via `client.apply`


ports = []<EOL>for i in range(n):<EOL><INDENT>sock = socket.socket()<EOL>sock.bind(('<STR_LIT>', <NUM_LIT:0>))<EOL>while sock.getsockname()[<NUM_LIT:1>] in _random_ports:<EOL><INDENT>sock.close()<EOL>sock = socket.socket()<EOL>sock.bind(('<STR_LIT>', <NUM_LIT:0>))<EOL><DEDENT>ports.append(sock)<EOL><DEDENT>for i, sock in enumerate(ports):<EOL><INDENT>port = sock.getsockname()[<NUM_LIT:1>]<EOL>sock.close()<EOL>ports[i] = port<EOL>_random_ports.add(port)<EOL><DEDENT>return ports<EOL>

Selects and return n random ports that are available.


pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'<STR_LIT>')<EOL>if os.path.isfile(pid_file):<EOL><INDENT>pid = self.get_pid_from_file()<EOL>if not overwrite:<EOL><INDENT>raise PIDFileError(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (pid_file, pid)<EOL>)<EOL><DEDENT><DEDENT>with open(pid_file, '<STR_LIT:w>') as f:<EOL><INDENT>self.log.info("<STR_LIT>" % pid_file)<EOL>f.write(repr(os.getpid())+'<STR_LIT:\n>')<EOL><DEDENT>

Create a .pid file in the pid_dir with my pid.

        This must be called after pre_construct, which sets `self.pid_dir`.
        This raises :exc:`PIDFileError` if the pid file exists already.


return u"<STR_LIT>"<EOL>

override default log format to include time


if self.kernel_app is not None:<EOL><INDENT>return<EOL><DEDENT>self.log.info("<STR_LIT>")<EOL>kernel = self.kernel<EOL>kwargs.setdefault('<STR_LIT>', self.config)<EOL>kwargs.setdefault('<STR_LIT>', self.log)<EOL>kwargs.setdefault('<STR_LIT>', self.profile_dir)<EOL>kwargs.setdefault('<STR_LIT>', self.engine.session)<EOL>app = self.kernel_app = IPKernelApp(**kwargs)<EOL>IPKernelApp._instance = app<EOL>app.init_connection_file()<EOL>app.shell_port = app._bind_socket(kernel.shell_streams[<NUM_LIT:0>], app.shell_port)<EOL>app.log.debug("<STR_LIT>", app.shell_port)<EOL>app.iopub_port = app._bind_socket(kernel.iopub_socket, app.iopub_port)<EOL>app.log.debug("<STR_LIT>", app.iopub_port)<EOL>kernel.stdin_socket = self.engine.context.socket(zmq.ROUTER)<EOL>app.stdin_port = app._bind_socket(kernel.stdin_socket, app.stdin_port)<EOL>app.log.debug("<STR_LIT>", app.stdin_port)<EOL>app.init_heartbeat()<EOL>app.log_connection_info()<EOL>app.write_connection_file()<EOL>

Promote engine to listening kernel, accessible to frontends.


self.tasks.append(task)<EOL>

Add a task to the job.

        Parameters
        ----------
        task : :class:`WinHPCTask`
            The task object to add.


txt = self.tostring()<EOL>with open(filename, '<STR_LIT:w>') as f:<EOL><INDENT>f.write(txt)<EOL><DEDENT>

Write the XML job description to a file.


try:<EOL><INDENT>pid = self.get_pid_from_file()<EOL><DEDENT>except PIDFileError:<EOL><INDENT>self.log.critical(<EOL>'<STR_LIT>'<EOL>)<EOL>self.remove_pid_file()<EOL>self.exit(ALREADY_STOPPED)<EOL><DEDENT>if not self.check_pid(pid):<EOL><INDENT>self.log.critical(<EOL>'<STR_LIT>' % pid<EOL>)<EOL>self.remove_pid_file()<EOL>self.exit(ALREADY_STOPPED)<EOL><DEDENT>elif os.name=='<STR_LIT>':<EOL><INDENT>sig = self.signal<EOL>self.log.info(<EOL>"<STR_LIT>" % (pid, sig)<EOL>)<EOL>try:<EOL><INDENT>os.kill(pid, sig)<EOL><DEDENT>except OSError:<EOL><INDENT>self.log.error("<STR_LIT>",<EOL>exc_info=True)<EOL>self.remove_pid_file()<EOL><DEDENT><DEDENT>elif os.name=='<STR_LIT>':<EOL><INDENT>try:<EOL><INDENT>p = check_call(['<STR_LIT>', '<STR_LIT>', str(pid), '<STR_LIT>', '<STR_LIT>'], stdout=PIPE,stderr=PIPE)<EOL><DEDENT>except (CalledProcessError, OSError):<EOL><INDENT>self.log.error("<STR_LIT>",<EOL>exc_info=True)<EOL><DEDENT>self.remove_pid_file()<EOL><DEDENT>


app = IPClusterApp.instance()<EOL>app.initialize()<EOL>app.start()<EOL>


self.log.info("<STR_LIT>")<EOL>self.log.info(<EOL>'<STR_LIT>' % self.daemonize<EOL>)<EOL>if self.daemonize:<EOL><INDENT>if os.name=='<STR_LIT>':<EOL><INDENT>daemonize()<EOL><DEDENT><DEDENT>dc = ioloop.DelayedCallback(self.start_engines, <NUM_LIT:0>, self.loop)<EOL>dc.start()<EOL>try:<EOL><INDENT>self.loop.start()<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>pass<EOL><DEDENT>except zmq.ZMQError as e:<EOL><INDENT>if e.errno == errno.EINTR:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>

Start the app for the engines subcommand.


<EOL>self.write_batch_script(n)<EOL>piped_cmd = self.args[<NUM_LIT:0>]+'<STR_LIT>'+self.args[<NUM_LIT:1>]+'<STR_LIT>'<EOL>self.log.debug("<STR_LIT>", self.__class__.__name__, piped_cmd)<EOL>p = Popen(piped_cmd, shell=True,env=os.environ,stdout=PIPE)<EOL>output,err = p.communicate()<EOL>job_id = self.parse_job_id(output)<EOL>self.notify_start(job_id)<EOL>return job_id<EOL>

Start n copies of the process using LSF batch system.
        This cant inherit from the base class because bsub expects
        to be piped a shell script in order to honor the #BSUB directives :
        bsub < script


return self.mpi_cmd + ['<STR_LIT>', str(self.n)] + self.mpi_args +self.program + self.program_args<EOL>


return super(WindowsHPCEngineSetLauncher, self).start(n)<EOL>


self.n = n<EOL>if self.batch_template_file and not self.batch_template:<EOL><INDENT>with open(self.batch_template_file) as f:<EOL><INDENT>self.batch_template = f.read()<EOL><DEDENT><DEDENT>if not self.batch_template:<EOL><INDENT>self.batch_template = self.default_template<EOL>if not self.job_array_regexp.search(self.batch_template):<EOL><INDENT>self.log.debug("<STR_LIT>")<EOL>firstline, rest = self.batch_template.split('<STR_LIT:\n>',<NUM_LIT:1>)<EOL>self.batch_template = u'<STR_LIT:\n>'.join([firstline, self.job_array_template, rest])<EOL><DEDENT>if self.queue and not self.queue_regexp.search(self.batch_template):<EOL><INDENT>self.log.debug("<STR_LIT>")<EOL>firstline, rest = self.batch_template.split('<STR_LIT:\n>',<NUM_LIT:1>)<EOL>self.batch_template = u'<STR_LIT:\n>'.join([firstline, self.queue_template, rest])<EOL><DEDENT><DEDENT>script_as_string = self.formatter.format(self.batch_template, **self.context)<EOL>self.log.debug('<STR_LIT>', self.batch_file)<EOL>with open(self.batch_file, '<STR_LIT:w>') as f:<EOL><INDENT>f.write(script_as_string)<EOL><DEDENT>os.chmod(self.batch_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)<EOL>

Instantiate and write the batch script to the work_dir.


return super(LSFControllerLauncher, self).start(<NUM_LIT:1>)<EOL>

Start the controller by profile or profile_dir.


if self.state == '<STR_LIT>':<EOL><INDENT>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>


return '<STR_LIT:U+0020>'.join(self.args)<EOL>

The string form of the program arguments.


def save_connection_dict(self, fname, cdict):

c = self.config<EOL>url = cdict['<STR_LIT:url>']<EOL>location = cdict['<STR_LIT:location>']<EOL>if not location:<EOL><INDENT>try:<EOL><INDENT>proto,ip,port = split_url(url)<EOL><DEDENT>except AssertionError:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>location = socket.gethostbyname_ex(socket.gethostname())[<NUM_LIT:2>][-<NUM_LIT:1>]<EOL><DEDENT>except (socket.gaierror, IndexError):<EOL><INDENT>self.log.warn("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL>location = '<STR_LIT:127.0.0.1>'<EOL><DEDENT><DEDENT>cdict['<STR_LIT:location>'] = location<EOL><DEDENT>fname = os.path.join(self.profile_dir.security_dir, fname)<EOL>self.log.info("<STR_LIT>", fname)<EOL>with open(fname, '<STR_LIT:w>') as f:<EOL><INDENT>f.write(json.dumps(cdict, indent=<NUM_LIT:2>))<EOL><DEDENT>os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)<EOL>


c = Client(profile='<STR_LIT>', context=self.context)<EOL>for name in [n for n in dir(c) if n.endswith('<STR_LIT>')]:<EOL><INDENT>s = getattr(c, name)<EOL>s.setsockopt(zmq.LINGER, <NUM_LIT:0>)<EOL>self.sockets.append(s)<EOL><DEDENT>return c<EOL>

connect a client with my Context, and track its sockets for cleanup


import ctypes<EOL>ctypes.memset(-<NUM_LIT:1>,<NUM_LIT:0>,<NUM_LIT:1>)<EOL>


def minimum_engines(self, n=<NUM_LIT:1>, block=True):

self.engines.extend(add_engines(n, total=True))<EOL>if block:<EOL><INDENT>self.wait_on_engines()<EOL><DEDENT>

add engines until there are at least n connected


msg_id = job.msg_id<EOL>self.log.debug("<STR_LIT>", msg_id)<EOL>if not self.targets:<EOL><INDENT>return False<EOL><DEDENT>if job.follow or job.targets or job.blacklist or self.hwm:<EOL><INDENT>def can_run(idx):<EOL><INDENT>if self.hwm and self.loads[idx] == self.hwm:<EOL><INDENT>return False<EOL><DEDENT>target = self.targets[idx]<EOL>if target in job.blacklist:<EOL><INDENT>return False<EOL><DEDENT>if job.targets and target not in job.targets:<EOL><INDENT>return False<EOL><DEDENT>return job.follow.check(self.completed[target], self.failed[target])<EOL><DEDENT>indices = filter(can_run, range(len(self.targets)))<EOL>if not indices:<EOL><INDENT>if job.follow.all:<EOL><INDENT>dests = set()<EOL>relevant = set()<EOL>if job.follow.success:<EOL><INDENT>relevant = self.all_completed<EOL><DEDENT>if job.follow.failure:<EOL><INDENT>relevant = relevant.union(self.all_failed)<EOL><DEDENT>for m in job.follow.intersection(relevant):<EOL><INDENT>dests.add(self.destinations[m])<EOL><DEDENT>if len(dests) > <NUM_LIT:1>:<EOL><INDENT>self.depending[msg_id] = job<EOL>self.fail_unreachable(msg_id)<EOL>return False<EOL><DEDENT><DEDENT>if job.targets:<EOL><INDENT>job.targets.difference_update(job.blacklist)<EOL>if not job.targets or not job.targets.intersection(self.targets):<EOL><INDENT>self.depending[msg_id] = job<EOL>self.fail_unreachable(msg_id)<EOL>return False<EOL><DEDENT><DEDENT>return False<EOL><DEDENT><DEDENT>else:<EOL><INDENT>indices = None<EOL><DEDENT>self.submit_task(job, indices)<EOL>return True<EOL>

check location dependencies, and run if they are met.


msg_id = job.msg_id<EOL>self.depending[msg_id] = job<EOL>for dep_id in job.after.union(job.follow).difference(self.all_done):<EOL><INDENT>if dep_id not in self.graph:<EOL><INDENT>self.graph[dep_id] = set()<EOL><DEDENT>self.graph[dep_id].add(msg_id)<EOL><DEDENT>

Save a message for later submission when its dependencies are met.


def update_graph(self, dep_id=None, success=True):

<EOL>jobs = self.graph.pop(dep_id, [])<EOL>if dep_id is None or self.hwm and any( [ load==self.hwm-<NUM_LIT:1> for load in self.loads ]):<EOL><INDENT>jobs = self.depending.keys()<EOL><DEDENT>for msg_id in sorted(jobs, key=lambda msg_id: self.depending[msg_id].timestamp):<EOL><INDENT>job = self.depending[msg_id]<EOL>if job.after.unreachable(self.all_completed, self.all_failed)or job.follow.unreachable(self.all_completed, self.all_failed):<EOL><INDENT>self.fail_unreachable(msg_id)<EOL><DEDENT>elif job.after.check(self.all_completed, self.all_failed): <EOL><INDENT>if self.maybe_run(job):<EOL><INDENT>self.depending.pop(msg_id)<EOL>for mid in job.dependents:<EOL><INDENT>if mid in self.graph:<EOL><INDENT>self.graph[mid].remove(msg_id)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>

dep_id just finished. Update our dependency
        graph and submit any jobs that just became runable.

        Called with dep_id=None to update entire graph for hwm, but without finishing
        a task.


now = datetime.now()<EOL>for msg_id in self.depending.keys():<EOL><INDENT>if msg_id in self.depending:<EOL><INDENT>job = self.depending[msg_id]<EOL>if job.timeout and job.timeout < now:<EOL><INDENT>self.fail_unreachable(msg_id, error.TaskTimeout)<EOL><DEDENT><DEDENT><DEDENT>

Audit all waiting tasks for expired timeouts.


self.client_stream.on_recv(self.dispatch_submission, copy=False)<EOL>


try:<EOL><INDENT>idents,msg = self.session.feed_identities(msg)<EOL><DEDENT>except ValueError:<EOL><INDENT>self.log.warn("<STR_LIT>",msg)<EOL>return<EOL><DEDENT>try:<EOL><INDENT>msg = self.session.unserialize(msg)<EOL><DEDENT>except ValueError:<EOL><INDENT>self.log.warn("<STR_LIT>"%idents)<EOL>return<EOL><DEDENT>msg_type = msg['<STR_LIT>']['<STR_LIT>']<EOL>handler = self._notification_handlers.get(msg_type, None)<EOL>if handler is None:<EOL><INDENT>self.log.error("<STR_LIT>"%msg_type)<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>handler(cast_bytes(msg['<STR_LIT:content>']['<STR_LIT>']))<EOL><DEDENT>except Exception:<EOL><INDENT>self.log.error("<STR_LIT>", msg, exc_info=True)<EOL><DEDENT><DEDENT>


def handle_result(self, idents, parent, raw_msg, success=True):

<EOL>engine = idents[<NUM_LIT:0>]<EOL>client = idents[<NUM_LIT:1>]<EOL>raw_msg[:<NUM_LIT:2>] = [client,engine]<EOL>self.client_stream.send_multipart(raw_msg, copy=False)<EOL>msg_id = parent['<STR_LIT>']<EOL>self.pending[engine].pop(msg_id)<EOL>if success:<EOL><INDENT>self.completed[engine].add(msg_id)<EOL>self.all_completed.add(msg_id)<EOL><DEDENT>else:<EOL><INDENT>self.failed[engine].add(msg_id)<EOL>self.all_failed.add(msg_id)<EOL><DEDENT>self.all_done.add(msg_id)<EOL>self.destinations[msg_id] = engine<EOL>self.update_graph(msg_id, success)<EOL>

handle a real task result, either success or failure


return loads.index(min(loads))<EOL>

Always choose the lowest load.

    If the lowest load occurs more than once, the first
    occurance will be used.  If loads has LRU ordering, this means
    the LRU of those with the lowest load is chosen.


if len(self.targets) == <NUM_LIT:1>:<EOL><INDENT>pass<EOL><DEDENT>self.engine_stream.flush()<EOL>idx = self.targets.index(uid)<EOL>self.targets.pop(idx)<EOL>self.loads.pop(idx)<EOL>if self.pending[uid]:<EOL><INDENT>dc = ioloop.DelayedCallback(lambda : self.handle_stranded_tasks(uid), <NUM_LIT>, self.loop)<EOL>dc.start()<EOL><DEDENT>else:<EOL><INDENT>self.completed.pop(uid)<EOL>self.failed.pop(uid)<EOL><DEDENT>

Existing engine with ident `uid` became unavailable.


self.loads[idx] -= <NUM_LIT:1><EOL>

Called after self.targets[idx] just finished a job.
        Override with subclasses.


self.log.debug("<STR_LIT>", handler)<EOL>self._new_handlers.add(handler)<EOL>


self.log.debug("<STR_LIT>", handler)<EOL>self._failure_handlers.add(handler)<EOL>


r = self._records.find_one({'<STR_LIT>': msg_id})<EOL>if not r:<EOL><INDENT>raise KeyError(msg_id)<EOL><DEDENT>return r<EOL>


matches = self._match(check)<EOL>for m in matches:<EOL><INDENT>del self._records[m['<STR_LIT>']]<EOL><DEDENT>


del self._records[msg_id]<EOL>


matches = []<EOL>tests = {}<EOL>for k,v in check.iteritems():<EOL><INDENT>if isinstance(v, dict):<EOL><INDENT>tests[k] = CompositeFilter(v)<EOL><DEDENT>else:<EOL><INDENT>tests[k] = lambda o: o==v<EOL><DEDENT><DEDENT>for rec in self._records.itervalues():<EOL><INDENT>if self._match_one(rec, tests):<EOL><INDENT>matches.append(copy(rec))<EOL><DEDENT><DEDENT>return matches<EOL>


newid = self._idcounter<EOL>self._idcounter += <NUM_LIT:1><EOL>return newid<EOL>

gemerate a new ID.

        No longer reuse old ids, just count from 0.


def save_iopub_message(self, topics, msg):

<EOL>try:<EOL><INDENT>msg = self.session.unserialize(msg, content=True)<EOL><DEDENT>except Exception:<EOL><INDENT>self.log.error("<STR_LIT>", exc_info=True)<EOL>return<EOL><DEDENT>parent = msg['<STR_LIT>']<EOL>if not parent:<EOL><INDENT>self.log.warn("<STR_LIT>", msg)<EOL>return<EOL><DEDENT>msg_id = parent['<STR_LIT>']<EOL>msg_type = msg['<STR_LIT>']['<STR_LIT>']<EOL>content = msg['<STR_LIT:content>']<EOL>try:<EOL><INDENT>rec = self.db.get_record(msg_id)<EOL><DEDENT>except KeyError:<EOL><INDENT>rec = empty_record()<EOL>rec['<STR_LIT>'] = msg_id<EOL>self.db.add_record(msg_id, rec)<EOL><DEDENT>d = {}<EOL>if msg_type == '<STR_LIT>':<EOL><INDENT>name = content['<STR_LIT:name>']<EOL>s = rec[name] or '<STR_LIT>'<EOL>d[name] = s + content['<STR_LIT:data>']<EOL><DEDENT>elif msg_type == '<STR_LIT>':<EOL><INDENT>d['<STR_LIT>'] = content<EOL><DEDENT>elif msg_type == '<STR_LIT>':<EOL><INDENT>d['<STR_LIT>'] = content['<STR_LIT:code>']<EOL><DEDENT>elif msg_type in ('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>d[msg_type] = content<EOL><DEDENT>elif msg_type == '<STR_LIT:status>':<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>self.log.warn("<STR_LIT>", msg_type)<EOL><DEDENT>if not d:<EOL><INDENT>return<EOL><DEDENT>try:<EOL><INDENT>self.db.update_record(msg_id, d)<EOL><DEDENT>except Exception:<EOL><INDENT>self.log.error("<STR_LIT>", msg_id, exc_info=True)<EOL><DEDENT>


def save_task_result(self, idents, msg):

client_id = idents[<NUM_LIT:0>]<EOL>try:<EOL><INDENT>msg = self.session.unserialize(msg)<EOL><DEDENT>except Exception:<EOL><INDENT>self.log.error("<STR_LIT>",<EOL>client_id, msg, exc_info=True)<EOL>return<EOL><DEDENT>parent = msg['<STR_LIT>']<EOL>if not parent:<EOL><INDENT>self.log.warn("<STR_LIT>", msg)<EOL>return<EOL><DEDENT>msg_id = parent['<STR_LIT>']<EOL>if msg_id in self.unassigned:<EOL><INDENT>self.unassigned.remove(msg_id)<EOL><DEDENT>header = msg['<STR_LIT>']<EOL>engine_uuid = header.get('<STR_LIT>', u'<STR_LIT>')<EOL>eid = self.by_ident.get(cast_bytes(engine_uuid), None)<EOL>status = header.get('<STR_LIT:status>', None)<EOL>if msg_id in self.pending:<EOL><INDENT>self.log.info("<STR_LIT>", msg_id, eid)<EOL>self.pending.remove(msg_id)<EOL>self.all_completed.add(msg_id)<EOL>if eid is not None:<EOL><INDENT>if status != '<STR_LIT>':<EOL><INDENT>self.completed[eid].append(msg_id)<EOL><DEDENT>if msg_id in self.tasks[eid]:<EOL><INDENT>self.tasks[eid].remove(msg_id)<EOL><DEDENT><DEDENT>completed = header['<STR_LIT:date>']<EOL>started = header.get('<STR_LIT>', None)<EOL>result = {<EOL>'<STR_LIT>' : header,<EOL>'<STR_LIT>': msg['<STR_LIT:content>'],<EOL>'<STR_LIT>' : started,<EOL>'<STR_LIT>' : completed,<EOL>'<STR_LIT>' : datetime.now(),<EOL>'<STR_LIT>': engine_uuid,<EOL>}<EOL>result['<STR_LIT>'] = msg['<STR_LIT>']<EOL>try:<EOL><INDENT>self.db.update_record(msg_id, result)<EOL><DEDENT>except Exception:<EOL><INDENT>self.log.error("<STR_LIT>", msg_id, exc_info=True)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.log.debug("<STR_LIT>", msg_id)<EOL><DEDENT>


content = msg['<STR_LIT:content>']<EOL>query = content.get('<STR_LIT>', {})<EOL>keys = content.get('<STR_LIT>', None)<EOL>buffers = []<EOL>empty = list()<EOL>try:<EOL><INDENT>records = self.db.find_records(query, keys)<EOL><DEDENT>except Exception as e:<EOL><INDENT>content = error.wrap_exception()<EOL><DEDENT>else:<EOL><INDENT>if keys is not None:<EOL><INDENT>buffer_lens = [] if '<STR_LIT>' in keys else None<EOL>result_buffer_lens = [] if '<STR_LIT>' in keys else None<EOL><DEDENT>else:<EOL><INDENT>buffer_lens = None<EOL>result_buffer_lens = None<EOL><DEDENT>for rec in records:<EOL><INDENT>b = rec.pop('<STR_LIT>', empty) or empty<EOL>if buffer_lens is not None:<EOL><INDENT>buffer_lens.append(len(b))<EOL>buffers.extend(b)<EOL><DEDENT>rb = rec.pop('<STR_LIT>', empty) or empty<EOL>if result_buffer_lens is not None:<EOL><INDENT>result_buffer_lens.append(len(rb))<EOL>buffers.extend(rb)<EOL><DEDENT><DEDENT>content = dict(status='<STR_LIT>', records=records, buffer_lens=buffer_lens,<EOL>result_buffer_lens=result_buffer_lens)<EOL><DEDENT>self.session.send(self.query, "<STR_LIT>", content=content,<EOL>parent=msg, ident=client_id,<EOL>buffers=buffers)<EOL>

Perform a raw query on the task record database.


return {<EOL>'<STR_LIT>' : None,<EOL>'<STR_LIT>' : None,<EOL>'<STR_LIT:content>': None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>' : None,<EOL>'<STR_LIT>' : None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>' : None,<EOL>'<STR_LIT>' : None,<EOL>'<STR_LIT>' : None,<EOL>'<STR_LIT>' : None,<EOL>'<STR_LIT>' : None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>}<EOL>

Return an empty dict with all record keys.


content = msg['<STR_LIT:content>']<EOL>msg_ids = sorted(set(content['<STR_LIT>']))<EOL>statusonly = content.get('<STR_LIT>', False)<EOL>pending = []<EOL>completed = []<EOL>content = dict(status='<STR_LIT>')<EOL>content['<STR_LIT>'] = pending<EOL>content['<STR_LIT>'] = completed<EOL>buffers = []<EOL>if not statusonly:<EOL><INDENT>try:<EOL><INDENT>matches = self.db.find_records(dict(msg_id={'<STR_LIT>':msg_ids}))<EOL>records = {}<EOL>for rec in matches:<EOL><INDENT>records[rec['<STR_LIT>']] = rec<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>content = error.wrap_exception()<EOL>self.session.send(self.query, "<STR_LIT>", content=content,<EOL>parent=msg, ident=client_id)<EOL>return<EOL><DEDENT><DEDENT>else:<EOL><INDENT>records = {}<EOL><DEDENT>for msg_id in msg_ids:<EOL><INDENT>if msg_id in self.pending:<EOL><INDENT>pending.append(msg_id)<EOL><DEDENT>elif msg_id in self.all_completed:<EOL><INDENT>completed.append(msg_id)<EOL>if not statusonly:<EOL><INDENT>c,bufs = self._extract_record(records[msg_id])<EOL>content[msg_id] = c<EOL>buffers.extend(bufs)<EOL><DEDENT><DEDENT>elif msg_id in records:<EOL><INDENT>if rec['<STR_LIT>']:<EOL><INDENT>completed.append(msg_id)<EOL>c,bufs = self._extract_record(records[msg_id])<EOL>content[msg_id] = c<EOL>buffers.extend(bufs)<EOL><DEDENT>else:<EOL><INDENT>pending.append(msg_id)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>raise KeyError('<STR_LIT>'+msg_id)<EOL><DEDENT>except:<EOL><INDENT>content = error.wrap_exception()<EOL><DEDENT>break<EOL><DEDENT><DEDENT>self.session.send(self.query, "<STR_LIT>", content=content,<EOL>parent=msg, ident=client_id,<EOL>buffers=buffers)<EOL>


def _handle_stranded_msgs(self, eid, uuid):

outstanding = self.queues[eid]<EOL>for msg_id in outstanding:<EOL><INDENT>self.pending.remove(msg_id)<EOL>self.all_completed.add(msg_id)<EOL>try:<EOL><INDENT>raise error.EngineError("<STR_LIT>" % (eid, msg_id))<EOL><DEDENT>except:<EOL><INDENT>content = error.wrap_exception()<EOL><DEDENT>header = {}<EOL>header['<STR_LIT>'] = uuid<EOL>header['<STR_LIT:date>'] = datetime.now()<EOL>rec = dict(result_content=content, result_header=header, result_buffers=[])<EOL>rec['<STR_LIT>'] = header['<STR_LIT:date>']<EOL>rec['<STR_LIT>'] = uuid<EOL>try:<EOL><INDENT>self.db.update_record(msg_id, rec)<EOL><DEDENT>except Exception:<EOL><INDENT>self.log.error("<STR_LIT>", msg_id, exc_info=True)<EOL><DEDENT><DEDENT>

Handle messages known to be on an engine when the engine unregisters.

        It is possible that this will fire prematurely - that is, an engine will
        go down after completing a result, and the client will be notified
        that the result failed and later receive the actual result.


def purge_results(self, client_id, msg):

content = msg['<STR_LIT:content>']<EOL>self.log.info("<STR_LIT>", content)<EOL>msg_ids = content.get('<STR_LIT>', [])<EOL>reply = dict(status='<STR_LIT>')<EOL>if msg_ids == '<STR_LIT:all>':<EOL><INDENT>try:<EOL><INDENT>self.db.drop_matching_records(dict(completed={'<STR_LIT>':None}))<EOL><DEDENT>except Exception:<EOL><INDENT>reply = error.wrap_exception()<EOL><DEDENT><DEDENT>else:<EOL><INDENT>pending = filter(lambda m: m in self.pending, msg_ids)<EOL>if pending:<EOL><INDENT>try:<EOL><INDENT>raise IndexError("<STR_LIT>" % pending[<NUM_LIT:0>])<EOL><DEDENT>except:<EOL><INDENT>reply = error.wrap_exception()<EOL><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>self.db.drop_matching_records(dict(msg_id={'<STR_LIT>':msg_ids}))<EOL><DEDENT>except Exception:<EOL><INDENT>reply = error.wrap_exception()<EOL><DEDENT><DEDENT>if reply['<STR_LIT:status>'] == '<STR_LIT>':<EOL><INDENT>eids = content.get('<STR_LIT>', [])<EOL>for eid in eids:<EOL><INDENT>if eid not in self.engines:<EOL><INDENT>try:<EOL><INDENT>raise IndexError("<STR_LIT>" % eid)<EOL><DEDENT>except:<EOL><INDENT>reply = error.wrap_exception()<EOL><DEDENT>break<EOL><DEDENT>uid = self.engines[eid].queue<EOL>try:<EOL><INDENT>self.db.drop_matching_records(dict(engine_uuid=uid, completed={'<STR_LIT>':None}))<EOL><DEDENT>except Exception:<EOL><INDENT>reply = error.wrap_exception()<EOL>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT>self.session.send(self.query, '<STR_LIT>', content=reply, ident=client_id)<EOL>

Purge results from memory. This method is more valuable before we move
        to a DB based message storage mechanism.


query = "<STR_LIT>"%self.table<EOL>sets = []<EOL>keys = sorted(rec.keys())<EOL>values = []<EOL>for key in keys:<EOL><INDENT>sets.append('<STR_LIT>'%key)<EOL>values.append(rec[key])<EOL><DEDENT>query += '<STR_LIT:U+002CU+0020>'.join(sets)<EOL>query += '<STR_LIT>'<EOL>values.append(msg_id)<EOL>self._db.execute(query, values)<EOL>


expressions = []<EOL>args = []<EOL>skeys = set(check.keys())<EOL>skeys.difference_update(set(self._keys))<EOL>skeys.difference_update(set(['<STR_LIT>', '<STR_LIT>']))<EOL>if skeys:<EOL><INDENT>raise KeyError("<STR_LIT>"%skeys)<EOL><DEDENT>for name,sub_check in check.iteritems():<EOL><INDENT>if isinstance(sub_check, dict):<EOL><INDENT>for test,value in sub_check.iteritems():<EOL><INDENT>try:<EOL><INDENT>op = operators[test]<EOL><DEDENT>except KeyError:<EOL><INDENT>raise KeyError("<STR_LIT>"%test)<EOL><DEDENT>if isinstance(op, tuple):<EOL><INDENT>op, join = op<EOL><DEDENT>if value is None and op in null_operators:<EOL><INDENT>expr = "<STR_LIT>" % (name, null_operators[op])<EOL><DEDENT>else:<EOL><INDENT>expr = "<STR_LIT>"%(name, op)<EOL>if isinstance(value, (tuple,list)):<EOL><INDENT>if op in null_operators and any([v is None for v in value]):<EOL><INDENT>raise ValueError("<STR_LIT>"%test)<EOL><DEDENT>expr = '<STR_LIT>'%( join.join([expr]*len(value)) )<EOL>args.extend(value)<EOL><DEDENT>else:<EOL><INDENT>args.append(value)<EOL><DEDENT><DEDENT>expressions.append(expr)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if sub_check is None:<EOL><INDENT>expressions.append("<STR_LIT>" % name)<EOL><DEDENT>else:<EOL><INDENT>expressions.append("<STR_LIT>"%name)<EOL>args.append(sub_check)<EOL><DEDENT><DEDENT><DEDENT>expr = "<STR_LIT>".join(expressions)<EOL>return expr, args<EOL>

Turn a mongodb-style search dict into an SQL query.


self._db.execute("""<STR_LIT>"""%self.table, (msg_id,))<EOL>


cursor = self._db.execute("""<STR_LIT>"""%self.table, (msg_id,))<EOL>line = cursor.fetchone()<EOL>if line is None:<EOL><INDENT>raise KeyError("<STR_LIT>"%msg_id)<EOL><DEDENT>return self._list_to_dict(line)<EOL>


def shutdown(self, targets=None, restart=False, hub=False, block=None):

block = self.block if block is None else block<EOL>if targets is None or targets == '<STR_LIT:all>':<EOL><INDENT>targets = self.targets<EOL><DEDENT>return self.client.shutdown(targets=targets, restart=restart, hub=hub, block=block)<EOL>

Terminates one or more engine processes, optionally including the hub.


def apply_async(self, f, *args, **kwargs):

return self._really_apply(f, args, kwargs, block=False)<EOL>

calls f(*args, **kwargs) on remote engines in a nonblocking manner.

        returns AsyncResult


block = self.block if block is None else block<EOL>return remote(self, block=block, **flags)<EOL>


super(LoadBalancedView, self).set_flags(**kwargs)<EOL>for name in ('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>if name in kwargs:<EOL><INDENT>value = kwargs[name]<EOL>if self._validate_dependency(value):<EOL><INDENT>setattr(self, name, value)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>"%value)<EOL><DEDENT><DEDENT><DEDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>t = kwargs['<STR_LIT>']<EOL>if not isinstance(t, (int, float, type(None))):<EOL><INDENT>raise TypeError("<STR_LIT>"%type(t))<EOL><DEDENT>if t is not None:<EOL><INDENT>if t < <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>"%t)<EOL><DEDENT><DEDENT>self.timeout = t<EOL><DEDENT>

set my attribute flags by keyword.

        A View is a wrapper for the Client's apply method, but with attributes
        that specify keyword arguments, those attributes can be set by keyword
        argument with this method.

        Parameters
        ----------

        block : bool
            whether to wait for results
        track : bool
            whether to create a MessageTracker to allow the user to
            safely edit after arrays and buffers during non-copying
            sends.

        after : Dependency or collection of msg_ids
            Only for load-balanced execution (targets=None)
            Specify a list of msg_ids as a time-based dependency.
            This job will only be run *after* the dependencies
            have been met.

        follow : Dependency or collection of msg_ids
            Only for load-balanced execution (targets=None)
            Specify a list of msg_ids as a location-based dependency.
            This job will only be run on an engine where this dependency
            is met.

        timeout : float/int or None
            Only for load-balanced execution (targets=None)
            Specify an amount of time (in seconds) for the scheduler to
            wait for dependencies to be met before failing with a
            DependencyTimeout.

        retries : int
            Number of times a task will be retried on failure.


@spin_after<EOL><INDENT>def map(self, f, *sequences, **kwargs):<DEDENT>

block = kwargs.pop('<STR_LIT>', self.block)<EOL>for k in list(kwargs.keys()):<EOL><INDENT>if k not in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>raise TypeError("<STR_LIT>"%k)<EOL><DEDENT><DEDENT>assert len(sequences) > <NUM_LIT:0>, "<STR_LIT>"<EOL>pf = ParallelFunction(self, f, block=block, **kwargs)<EOL>return pf.map(*sequences)<EOL>

view.map(f, *sequences, block=self.block) => list|AsyncMapResult

        Parallel version of builtin `map`, using this View's `targets`.

        There will be one task per target, so work will be chunked
        if the sequences are longer than `targets`.

        Results can be iterated as they are ready, but will become available in chunks.

        Parameters
        ----------

        f : callable
            function to be mapped
        *sequences: one or more sequences of matching length
            the sequences to be distributed and passed to `f`
        block : bool
            whether to wait for the result or not [default self.block]

        Returns
        -------

        if block=False:
            AsyncMapResult
                An object like AsyncResult, but which reassembles the sequence of results
                into a single list. AsyncMapResults can be iterated through before all
                results are complete.
        else:
            list
                the result of map(f,*sequences)


if indices_or_msg_ids is None:<EOL><INDENT>indices_or_msg_ids = -<NUM_LIT:1><EOL><DEDENT>if isinstance(indices_or_msg_ids, int):<EOL><INDENT>indices_or_msg_ids = self.history[indices_or_msg_ids]<EOL><DEDENT>elif isinstance(indices_or_msg_ids, (list,tuple,set)):<EOL><INDENT>indices_or_msg_ids = list(indices_or_msg_ids)<EOL>for i,index in enumerate(indices_or_msg_ids):<EOL><INDENT>if isinstance(index, int):<EOL><INDENT>indices_or_msg_ids[i] = self.history[index]<EOL><DEDENT><DEDENT><DEDENT>return self.client.get_result(indices_or_msg_ids)<EOL>

return one or more results, specified by history index or msg_id.

        See client.get_result for details.


raise NotImplementedError<EOL>


from IPython.parallel.client.magics import ParallelMagics<EOL>try:<EOL><INDENT>ip = get_ipython()<EOL><DEDENT>except NameError:<EOL><INDENT>print("<STR_LIT>")<EOL>return<EOL><DEDENT>M = ParallelMagics(ip, self, suffix)<EOL>ip.magics_manager.register(M)<EOL>

Activate IPython magics associated with this View

        Defines the magics `%px, %autopx, %pxresult, %%px, %pxconfig`

        Parameters
        ----------

        suffix: str [default: '']
            The suffix, if any, for the magics.  This allows you to have
            multiple views associated with parallel magics at the same time.

            e.g. ``rc[::2].activate(suffix='_even')`` will give you
            the magics ``%px_even``, ``%pxresult_even``, etc. for running magics
            on the even engines.


<EOL>return self.pull(key_s, block=True)<EOL>

get object(s) by `key_s` from remote namespace

        see `pull` for details.


self.client.spin()<EOL>


<EOL>saved_flags = {}<EOL>for f in self._flag_names:<EOL><INDENT>saved_flags[f] = getattr(self, f)<EOL><DEDENT>self.set_flags(**kwargs)<EOL>try:<EOL><INDENT>yield<EOL><DEDENT>finally:<EOL><INDENT>self.set_flags(**saved_flags)<EOL><DEDENT>

temporarily set flags, for use in `with` statements.

        See set_flags for permanent setting of flags

        Examples
        --------

        >>> view.track=False
        ...
        >>> with view.temp_flags(track=True):
        ...    ar = view.apply(dostuff, my_big_array)
        ...    ar.tracker.wait() # wait for send to finish
        >>> view.track
        False


@sync_results<EOL><INDENT>@save_ids<EOL>def gather(self, key, dist='<STR_LIT:b>', targets=None, block=None):<DEDENT>

block = block if block is not None else self.block<EOL>targets = targets if targets is not None else self.targets<EOL>mapObject = Map.dists[dist]()<EOL>msg_ids = []<EOL>targets = self.client._build_targets(targets)[<NUM_LIT:1>]<EOL>for index, engineid in enumerate(targets):<EOL><INDENT>msg_ids.extend(self.pull(key, block=False, targets=engineid).msg_ids)<EOL><DEDENT>r = AsyncMapResult(self.client, msg_ids, mapObject, fname='<STR_LIT>')<EOL>if block:<EOL><INDENT>try:<EOL><INDENT>return r.get()<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return r<EOL>

Gather a partitioned sequence on a set of engines as a single local seq.


def map_sync(self, f, *sequences, **kwargs):

if '<STR_LIT>' in kwargs:<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>kwargs['<STR_LIT>'] = True<EOL>return self.map(f,*sequences,**kwargs)<EOL>

Parallel version of builtin `map`, using this view's engines.

        This is equivalent to map(...block=True)

        See `self.map` for details.


return self._really_apply(f, args, kwargs)<EOL>

calls f(*args, **kwargs) on remote engines, returning the result.

        This method sets all apply flags via this View's attributes.

        if self.block is False:
            returns AsyncResult
        else:
            returns actual result of f(*args, **kwargs)


if self._single_result:<EOL><INDENT>return res[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>return res<EOL><DEDENT>

Reconstruct our result from actual result list (always a list)

        Override me in subclasses for turning a list of results
        into the expected form.


if self._single_result:<EOL><INDENT>return self._metadata[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>return self._metadata<EOL><DEDENT>

property for accessing execution metadata.


if self._single_result:<EOL><INDENT>self._display_single_result()<EOL>return<EOL><DEDENT>stdouts = self.stdout<EOL>stderrs = self.stderr<EOL>pyouts  = self.pyout<EOL>output_lists = self.outputs<EOL>results = self.get()<EOL>targets = self.engine_id<EOL>if groupby == "<STR_LIT>":<EOL><INDENT>for eid,stdout,stderr,outputs,r,pyout in zip(<EOL>targets, stdouts, stderrs, output_lists, results, pyouts<EOL>):<EOL><INDENT>self._display_stream(stdout, '<STR_LIT>' % eid)<EOL>self._display_stream(stderr, '<STR_LIT>' % eid, file=sys.stderr)<EOL>try:<EOL><INDENT>get_ipython()<EOL><DEDENT>except NameError:<EOL><INDENT>return <EOL><DEDENT>if outputs or pyout is not None:<EOL><INDENT>_raw_text('<STR_LIT>' % eid)<EOL><DEDENT>for output in outputs:<EOL><INDENT>self._republish_displaypub(output, eid)<EOL><DEDENT>if pyout is not None:<EOL><INDENT>display(r)<EOL><DEDENT><DEDENT><DEDENT>elif groupby in ('<STR_LIT:type>', '<STR_LIT>'):<EOL><INDENT>for eid,stdout in zip(targets, stdouts):<EOL><INDENT>self._display_stream(stdout, '<STR_LIT>' % eid)<EOL><DEDENT>for eid,stderr in zip(targets, stderrs):<EOL><INDENT>self._display_stream(stderr, '<STR_LIT>' % eid, file=sys.stderr)<EOL><DEDENT>try:<EOL><INDENT>get_ipython()<EOL><DEDENT>except NameError:<EOL><INDENT>return<EOL><DEDENT>if groupby == '<STR_LIT>':<EOL><INDENT>output_dict = dict((eid, outputs) for eid,outputs in zip(targets, output_lists))<EOL>N = max(len(outputs) for outputs in output_lists)<EOL>for i in range(N):<EOL><INDENT>for eid in targets:<EOL><INDENT>outputs = output_dict[eid]<EOL>if len(outputs) >= N:<EOL><INDENT>_raw_text('<STR_LIT>' % eid)<EOL>self._republish_displaypub(outputs[i], eid)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for eid,outputs in zip(targets, output_lists):<EOL><INDENT>if outputs:<EOL><INDENT>_raw_text('<STR_LIT>' % eid)<EOL><DEDENT>for output in outputs:<EOL><INDENT>self._republish_displaypub(output, eid)<EOL><DEDENT><DEDENT><DEDENT>for eid,r,pyout in zip(targets, results, pyouts):<EOL><INDENT>if pyout is not None:<EOL><INDENT>display(r)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % groupby)<EOL><DEDENT>

republish the outputs of the computation

        Parameters
        ----------

        groupby : str [default: type]
            if 'type':
                Group outputs by type (show all stdout, then all stderr, etc.):

                [stdout:1] foo
                [stdout:2] foo
                [stderr:1] bar
                [stderr:2] bar
            if 'engine':
                Display outputs for each engine before moving on to the next:

                [stdout:1] foo
                [stderr:1] bar
                [stdout:2] foo
                [stderr:2] bar

            if 'order':
                Like 'type', but further collate individual displaypub
                outputs.  This is meant for cases of each command producing
                several plots, and you would like to see all of the first
                plots together, then all of the second plots, and so on.


try:<EOL><INDENT>rlist = self.get(<NUM_LIT:0>)<EOL><DEDENT>except error.TimeoutError:<EOL><INDENT>for msg_id in self.msg_ids:<EOL><INDENT>ar = AsyncResult(self._client, msg_id, self._fname)<EOL>rlist = ar.get()<EOL>try:<EOL><INDENT>for r in rlist:<EOL><INDENT>yield r<EOL><DEDENT><DEDENT>except TypeError:<EOL><INDENT>yield rlist<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for r in rlist:<EOL><INDENT>yield r<EOL><DEDENT><DEDENT>

iterator for results *as they arrive*, preserving submission order.


@decorator<EOL>def check_ready(f, self, *args, **kwargs):

self.wait(<NUM_LIT:0>)<EOL>if not self._ready:<EOL><INDENT>raise error.TimeoutError("<STR_LIT>")<EOL><DEDENT>return f(self, *args, **kwargs)<EOL>

Call spin() to sync state prior to calling the method.


e = error.unwrap_exception(content)<EOL>if e.engine_info:<EOL><INDENT>e_uuid = e.engine_info['<STR_LIT>']<EOL>eid = self._engines[e_uuid]<EOL>e.engine_info['<STR_LIT>'] = eid<EOL><DEDENT>return e<EOL>

unwrap exception, and remap engine_id to int.


idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)<EOL>while msg is not None:<EOL><INDENT>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>parent = msg['<STR_LIT>']<EOL>if not parent:<EOL><INDENT>continue<EOL><DEDENT>msg_id = parent['<STR_LIT>']<EOL>content = msg['<STR_LIT:content>']<EOL>header = msg['<STR_LIT>']<EOL>msg_type = msg['<STR_LIT>']['<STR_LIT>']<EOL>md = self.metadata[msg_id]<EOL>if msg_type == '<STR_LIT>':<EOL><INDENT>name = content['<STR_LIT:name>']<EOL>s = md[name] or '<STR_LIT>'<EOL>md[name] = s + content['<STR_LIT:data>']<EOL><DEDENT>elif msg_type == '<STR_LIT>':<EOL><INDENT>md.update({'<STR_LIT>' : self._unwrap_exception(content)})<EOL><DEDENT>elif msg_type == '<STR_LIT>':<EOL><INDENT>md.update({'<STR_LIT>' : content['<STR_LIT:code>']})<EOL><DEDENT>elif msg_type == '<STR_LIT>':<EOL><INDENT>md['<STR_LIT>'].append(content)<EOL><DEDENT>elif msg_type == '<STR_LIT>':<EOL><INDENT>md['<STR_LIT>'] = content<EOL><DEDENT>elif msg_type == '<STR_LIT:status>':<EOL><INDENT>if content['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>md['<STR_LIT>'] = True<EOL><DEDENT><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT>self.metadata[msg_id] = md<EOL>idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)<EOL><DEDENT>

Flush replies from the iopub channel waiting
        in the ZMQ queue.


self._task_socket.close()<EOL>self._task_socket = None<EOL>msg = "<STR_LIT>" +"<STR_LIT>"<EOL>if self.outstanding:<EOL><INDENT>msg += "<STR_LIT>" +"<STR_LIT>"<EOL><DEDENT>warnings.warn(msg, RuntimeWarning)<EOL>

Stop scheduling tasks because an engine has been unregistered
        from a pure ZMQ scheduler.


def send_execute_request(self, socket, code, silent=True, subheader=None, ident=None):

if self._closed:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>subheader = subheader if subheader is not None else {}<EOL>if not isinstance(code, str):<EOL><INDENT>raise TypeError("<STR_LIT>" % type(code))<EOL><DEDENT>if not isinstance(subheader, dict):<EOL><INDENT>raise TypeError("<STR_LIT>" % type(subheader))<EOL><DEDENT>content = dict(code=code, silent=bool(silent), user_variables=[], user_expressions={})<EOL>msg = self.session.send(socket, "<STR_LIT>", content=content, ident=ident,<EOL>subheader=subheader)<EOL>msg_id = msg['<STR_LIT>']['<STR_LIT>']<EOL>self.outstanding.add(msg_id)<EOL>if ident:<EOL><INDENT>if isinstance(ident, list):<EOL><INDENT>ident = ident[-<NUM_LIT:1>]<EOL><DEDENT>if ident in list(self._engines.values()):<EOL><INDENT>self._outstanding_dict[ident].add(msg_id)<EOL><DEDENT><DEDENT>self.history.append(msg_id)<EOL>self.metadata[msg_id]['<STR_LIT>'] = datetime.now()<EOL>return msg<EOL>

construct and send an execute request via a socket.


if self._spin_thread is not None:<EOL><INDENT>self.stop_spin_thread()<EOL><DEDENT>self._stop_spinning.clear()<EOL>self._spin_thread = Thread(target=self._spin_every, args=(interval,))<EOL>self._spin_thread.daemon = True<EOL>self._spin_thread.start()<EOL>

call Client.spin() in a background thread on some regular interval

        This helps ensure that messages don't pile up too much in the zmq queue
        while you are working on other things, or just leaving an idle terminal.

        It also helps limit potential padding of the `received` timestamp
        on AsyncResult objects, used for timings.

        Parameters
        ----------

        interval : float, optional
            The interval on which to spin the client in the background thread
            (simply passed to time.sleep).

        Notes
        -----

        For precision timing, you may want to use this method to put a bound
        on the jitter (in seconds) in `received` timestamps used
        in AsyncResult.wall_time.


self.close()<EOL>


@spin_first<EOL><INDENT>def shutdown(self, targets='<STR_LIT:all>', restart=False, hub=False, block=None):<DEDENT>

if restart:<EOL><INDENT>raise NotImplementedError("<STR_LIT>")<EOL><DEDENT>block = self.block if block is None else block<EOL>if hub:<EOL><INDENT>targets = '<STR_LIT:all>'<EOL><DEDENT>targets = self._build_targets(targets)[<NUM_LIT:0>]<EOL>for t in targets:<EOL><INDENT>self.session.send(self._control_socket, '<STR_LIT>',<EOL>content={'<STR_LIT>':restart},ident=t)<EOL><DEDENT>error = False<EOL>if block or hub:<EOL><INDENT>self._flush_ignored_control()<EOL>for i in range(len(targets)):<EOL><INDENT>idents,msg = self.session.recv(self._control_socket, <NUM_LIT:0>)<EOL>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>if msg['<STR_LIT:content>']['<STR_LIT:status>'] != '<STR_LIT>':<EOL><INDENT>error = self._unwrap_exception(msg['<STR_LIT:content>'])<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>self._ignored_control_replies += len(targets)<EOL><DEDENT>if hub:<EOL><INDENT>time.sleep(<NUM_LIT>)<EOL>self.session.send(self._query_socket, '<STR_LIT>')<EOL>idents,msg = self.session.recv(self._query_socket, <NUM_LIT:0>)<EOL>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>if msg['<STR_LIT:content>']['<STR_LIT:status>'] != '<STR_LIT>':<EOL><INDENT>error = self._unwrap_exception(msg['<STR_LIT:content>'])<EOL><DEDENT><DEDENT>if error:<EOL><INDENT>raise error<EOL><DEDENT>

Terminates one or more engine processes, optionally including the hub.

        Parameters
        ----------

        targets: list of ints or 'all' [default: all]
            Which engines to shutdown.
        hub: bool [default: False]
            Whether to include the Hub.  hub=True implies targets='all'.
        block: bool [default: self.block]
            Whether to wait for clean shutdown replies or not.
        restart: bool [default: False]
            NOT IMPLEMENTED
            whether to restart engines after shutting them down.


return len(self.ids)<EOL>


@spin_first<EOL><INDENT>def resubmit(self, indices_or_msg_ids=None, subheader=None, block=None):<DEDENT>

block = self.block if block is None else block<EOL>if indices_or_msg_ids is None:<EOL><INDENT>indices_or_msg_ids = -<NUM_LIT:1><EOL><DEDENT>if not isinstance(indices_or_msg_ids, (list,tuple)):<EOL><INDENT>indices_or_msg_ids = [indices_or_msg_ids]<EOL><DEDENT>theids = []<EOL>for id in indices_or_msg_ids:<EOL><INDENT>if isinstance(id, int):<EOL><INDENT>id = self.history[id]<EOL><DEDENT>if not isinstance(id, str):<EOL><INDENT>raise TypeError("<STR_LIT>"%id)<EOL><DEDENT>theids.append(id)<EOL><DEDENT>content = dict(msg_ids = theids)<EOL>self.session.send(self._query_socket, '<STR_LIT>', content)<EOL>zmq.select([self._query_socket], [], [])<EOL>idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)<EOL>if self.debug:<EOL><INDENT>pprint(msg)<EOL><DEDENT>content = msg['<STR_LIT:content>']<EOL>if content['<STR_LIT:status>'] != '<STR_LIT>':<EOL><INDENT>raise self._unwrap_exception(content)<EOL><DEDENT>mapping = content['<STR_LIT>']<EOL>new_ids = [ mapping[msg_id] for msg_id in theids ]<EOL>ar = AsyncHubResult(self, msg_ids=new_ids)<EOL>if block:<EOL><INDENT>ar.wait()<EOL><DEDENT>return ar<EOL>

Resubmit one or more tasks.

        in-flight tasks may not be resubmitted.

        Parameters
        ----------

        indices_or_msg_ids : integer history index, str msg_id, or list of either
            The indices or msg_ids of indices to be retrieved

        block : bool
            Whether to wait for the result to be done

        Returns
        -------

        AsyncHubResult
            A subclass of AsyncResult that retrieves results from the Hub


args = magic_arguments.parse_argstring(self.pxconfig, line)<EOL>if args.targets:<EOL><INDENT>self.view.targets = self._eval_target_str(args.targets)<EOL><DEDENT>if args.block is not None:<EOL><INDENT>self.view.block = args.block<EOL><DEDENT>if args.set_verbose is not None:<EOL><INDENT>self.verbose = args.set_verbose<EOL><DEDENT>

configure default targets/blocking for %px magics


def parallel_execute(self, cell, block=None, groupby='<STR_LIT:type>', save_name=None):

<EOL>block = self.view.block if block is None else block<EOL>base = "<STR_LIT>" if block else "<STR_LIT>"<EOL>targets = self.view.targets<EOL>if isinstance(targets, list) and len(targets) > <NUM_LIT:10>:<EOL><INDENT>str_targets = str(targets[:<NUM_LIT:4>])[:-<NUM_LIT:1>] + '<STR_LIT>' + str(targets[-<NUM_LIT:4>:])[<NUM_LIT:1>:]<EOL><DEDENT>else:<EOL><INDENT>str_targets = str(targets)<EOL><DEDENT>if self.verbose:<EOL><INDENT>print(base + "<STR_LIT>" % str_targets)<EOL><DEDENT>result = self.view.execute(cell, silent=False, block=False)<EOL>self.last_result = result<EOL>if save_name:<EOL><INDENT>self.shell.user_ns[save_name] = result<EOL><DEDENT>if block:<EOL><INDENT>result.get()<EOL>result.display_outputs(groupby)<EOL><DEDENT>else:<EOL><INDENT>return result<EOL><DEDENT>

implementation used by %px and %%parallel


<EOL>self._original_run_cell = self.shell.run_cell<EOL>self.shell.run_cell = self.pxrun_cell<EOL>self._autopx = True<EOL>print("<STR_LIT>")<EOL>

Enable %autopx mode by saving the original run_cell and installing
        pxrun_cell.


if isinstance(obj, (tuple,list)):<EOL><INDENT>return True<EOL><DEDENT>for m in arrayModules:<EOL><INDENT>if isinstance(obj,m['<STR_LIT:type>']):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

return whether an object is mappable or not.


<EOL>if p<<NUM_LIT:0> or p>=q:<EOL><INDENT>print("<STR_LIT>")<EOL>return<EOL><DEDENT>remainder = len(seq)%q<EOL>basesize = len(seq)//q<EOL>hi = []<EOL>lo = []<EOL>for n in range(q):<EOL><INDENT>if n < remainder:<EOL><INDENT>lo.append(n * (basesize + <NUM_LIT:1>))<EOL>hi.append(lo[-<NUM_LIT:1>] + basesize + <NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>lo.append(n*basesize + remainder)<EOL>hi.append(lo[-<NUM_LIT:1>] + basesize)<EOL><DEDENT><DEDENT>try:<EOL><INDENT>result = seq[lo[p]:hi[p]]<EOL><DEDENT>except TypeError:<EOL><INDENT>result = list(islice(seq, lo[p], hi[p]))<EOL><DEDENT>return result<EOL>

Returns the pth partition of q partitions of seq.


try:<EOL><INDENT>return f.__name__<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT>try:<EOL><INDENT>return f.name<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT>return str(f)<EOL>

Get the name of an object.

    For use in case of callables that are not functions, and
    thus may not have __name__ defined.

    Order: f.__name__ >  f.name > str(f)


from IPython.zmq.ipkernel import IPKernelApp<EOL>from IPython.parallel.apps.ipengineapp import IPEngineApp<EOL>if IPKernelApp.initialized() and isinstance(IPKernelApp._instance, IPKernelApp):<EOL><INDENT>return<EOL><DEDENT>if IPEngineApp.initialized():<EOL><INDENT>try:<EOL><INDENT>app = IPEngineApp.instance()<EOL><DEDENT>except MultipleInstanceError:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>return app.bind_kernel(**kwargs)<EOL><DEDENT><DEDENT>raise RuntimeError("<STR_LIT>")<EOL>

Bind an Engine's Kernel to be used as a full IPython kernel.

    This allows a running Engine to be used simultaneously as a full IPython kernel
    with the QtConsole or other frontends.

    This function returns immediately.


lines = []<EOL>if excid is None:<EOL><INDENT>for (en,ev,etb,ei) in self.elist:<EOL><INDENT>lines.append(self._get_engine_str(ei))<EOL>lines.extend((etb or '<STR_LIT>').splitlines())<EOL>lines.append('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>en,ev,etb,ei = self.elist[excid]<EOL><DEDENT>except:<EOL><INDENT>raise IndexError("<STR_LIT>"%excid)<EOL><DEDENT>else:<EOL><INDENT>lines.append(self._get_engine_str(ei))<EOL>lines.extend((etb or '<STR_LIT>').splitlines())<EOL><DEDENT><DEDENT>return lines<EOL>

render one or all of my tracebacks to a list of lines


print('<STR_LIT:\n>'.join(self.render_traceback()))<EOL>


if module is None:<EOL><INDENT>return True<EOL><DEDENT>elif inspect.isfunction(object):<EOL><INDENT>return module.__dict__ is object.__globals__<EOL><DEDENT>elif inspect.isclass(object):<EOL><INDENT>return module.__name__ == object.__module__<EOL><DEDENT>elif inspect.getmodule(object) is not None:<EOL><INDENT>return module is inspect.getmodule(object)<EOL><DEDENT>elif hasattr(object, '<STR_LIT>'):<EOL><INDENT>return module.__name__ == object.__module__<EOL><DEDENT>elif isinstance(object, property):<EOL><INDENT>return True <EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>

Return true if the given object is defined in the given
module.


if inspect.ismodule(module):<EOL><INDENT>return module<EOL><DEDENT>elif isinstance(module, str):<EOL><INDENT>return __import__(module, globals(), locals(), ["<STR_LIT:*>"])<EOL><DEDENT>elif module is None:<EOL><INDENT>return sys.modules[sys._getframe(depth).f_globals['<STR_LIT>']]<EOL><DEDENT>else:<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>

Return the module specified by `module`.  In particular:
  - If `module` is a module, then return module.
  - If `module` is a string, then import and return the
    module with that name.
  - If `module` is None, then return the calling module.
    The calling module is assumed to be the module of
    the stack frame at the given depth in the call stack.


def __init__(self, verbose=False, parser=DocTestParser(),<EOL>recurse=True, _namefilter=None, exclude_empty=True):

self._parser = parser<EOL>self._verbose = verbose<EOL>self._recurse = recurse<EOL>self._exclude_empty = exclude_empty<EOL>self._namefilter = _namefilter<EOL>

Create a new doctest finder.

The optional argument `parser` specifies a class or
function that should be used to create new DocTest objects (or
objects that implement the same interface as DocTest).  The
signature for this factory function should match the signature
of the DocTest constructor.

If the optional argument `recurse` is false, then `find` will
only examine the given object, and not any contained objects.

If the optional argument `exclude_empty` is false, then `find`
will include tests for objects with empty docstrings.


warnings.warn("<STR_LIT>"<EOL>"<STR_LIT>",<EOL>DeprecationWarning, stacklevel=<NUM_LIT:2>)<EOL>return base[:<NUM_LIT:1>] == "<STR_LIT:_>" and not base[:<NUM_LIT:2>] == "<STR_LIT>" == base[-<NUM_LIT:2>:]<EOL>

prefix, base -> true iff name prefix + "." + base is "private".

    Prefix may be an empty string, and base does not contain a period.
    Prefix is ignored (although functions you write conforming to this
    protocol may make use of it).
    Return true iff base begins with an (at least one) underscore, but
    does not both begin and end with (at least) two underscores.

    >>> is_private("a.b", "my_func")
    False
    >>> is_private("____", "_my_func")
    True
    >>> is_private("someclass", "__init__")
    False
    >>> is_private("sometypo", "__init_")
    True
    >>> is_private("x.y.z", "_")
    True
    >>> is_private("_x.y.z", "__")
    False
    >>> is_private("", "")  # senseless but consistent
    False


<EOL>return re.sub('<STR_LIT>', indent*'<STR_LIT:U+0020>', s)<EOL>

Add the given number of space characters to the beginning every
non-blank line in `s`, and return the result.


def _check_prefix(self, lines, prefix, name, lineno):

for i, line in enumerate(lines):<EOL><INDENT>if line and not line.startswith(prefix):<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(lineno+i+<NUM_LIT:1>, name, line))<EOL><DEDENT><DEDENT>

Check that every line in the given list starts with the given
prefix; if any line does not, then raise a ValueError.


def output_difference(self, example, got, optionflags):

want = example.want<EOL>if not (optionflags & DONT_ACCEPT_BLANKLINE):<EOL><INDENT>got = re.sub('<STR_LIT>', BLANKLINE_MARKER, got)<EOL><DEDENT>if self._do_a_fancy_diff(want, got, optionflags):<EOL><INDENT>want_lines = want.splitlines(True)  <EOL>got_lines = got.splitlines(True)<EOL>if optionflags & REPORT_UDIFF:<EOL><INDENT>diff = difflib.unified_diff(want_lines, got_lines, n=<NUM_LIT:2>)<EOL>diff = list(diff)[<NUM_LIT:2>:] <EOL>kind = '<STR_LIT>'<EOL><DEDENT>elif optionflags & REPORT_CDIFF:<EOL><INDENT>diff = difflib.context_diff(want_lines, got_lines, n=<NUM_LIT:2>)<EOL>diff = list(diff)[<NUM_LIT:2>:] <EOL>kind = '<STR_LIT>'<EOL><DEDENT>elif optionflags & REPORT_NDIFF:<EOL><INDENT>engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK)<EOL>diff = list(engine.compare(want_lines, got_lines))<EOL>kind = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>assert <NUM_LIT:0>, '<STR_LIT>'<EOL><DEDENT>diff = [line.rstrip() + '<STR_LIT:\n>' for line in diff]<EOL>return '<STR_LIT>' % kind + _indent('<STR_LIT>'.join(diff))<EOL><DEDENT>if want and got:<EOL><INDENT>return '<STR_LIT>' % (_indent(want), _indent(got))<EOL><DEDENT>elif want:<EOL><INDENT>return '<STR_LIT>' % _indent(want)<EOL><DEDENT>elif got:<EOL><INDENT>return '<STR_LIT>' % _indent(got)<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>

Return a string describing the differences between the
expected output for a given example (`example`) and the actual
output (`got`).  `optionflags` is the set of option flags used
to compare `want` and `got`.


line = line.rstrip()<EOL>if line:<EOL><INDENT>return '<STR_LIT>'+line<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT:#>'<EOL><DEDENT>

Return a commented form of the given line


if want.find(ELLIPSIS_MARKER)==-<NUM_LIT:1>:<EOL><INDENT>return want == got<EOL><DEDENT>ws = want.split(ELLIPSIS_MARKER)<EOL>assert len(ws) >= <NUM_LIT:2><EOL>startpos, endpos = <NUM_LIT:0>, len(got)<EOL>w = ws[<NUM_LIT:0>]<EOL>if w:   <EOL><INDENT>if got.startswith(w):<EOL><INDENT>startpos = len(w)<EOL>del ws[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>w = ws[-<NUM_LIT:1>]<EOL>if w:   <EOL><INDENT>if got.endswith(w):<EOL><INDENT>endpos -= len(w)<EOL>del ws[-<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>if startpos > endpos:<EOL><INDENT>return False<EOL><DEDENT>for w in ws:<EOL><INDENT>startpos = got.find(w, startpos, endpos)<EOL>if startpos < <NUM_LIT:0>:<EOL><INDENT>return False<EOL><DEDENT>startpos += len(w)<EOL><DEDENT>return True<EOL>

Essentially the only subtle case:
>>> _ellipsis_match('aa...aa', 'aaa')
False


def _check_prompt_blank(self, lines, indent, name, lineno):

for i, line in enumerate(lines):<EOL><INDENT>if len(line) >= indent+<NUM_LIT:4> and line[indent+<NUM_LIT:3>] != '<STR_LIT:U+0020>':<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(lineno+i+<NUM_LIT:1>, name,<EOL>line[indent:indent+<NUM_LIT:3>], line))<EOL><DEDENT><DEDENT>

Given the lines of a source string (including prompts and
leading indentation), check to make sure that every prompt is
followed by a space character.  If any line is not followed by
a space character, then raise ValueError.


def _find_lineno(self, obj, source_lines):

lineno = None<EOL>if inspect.ismodule(obj):<EOL><INDENT>lineno = <NUM_LIT:0><EOL><DEDENT>if inspect.isclass(obj):<EOL><INDENT>if source_lines is None:<EOL><INDENT>return None<EOL><DEDENT>pat = re.compile(r'<STR_LIT>' %<EOL>getattr(obj, '<STR_LIT>', '<STR_LIT:->'))<EOL>for i, line in enumerate(source_lines):<EOL><INDENT>if pat.match(line):<EOL><INDENT>lineno = i<EOL>break<EOL><DEDENT><DEDENT><DEDENT>if inspect.ismethod(obj): obj = obj.__func__<EOL>if inspect.isfunction(obj): obj = obj.__code__<EOL>if inspect.istraceback(obj): obj = obj.tb_frame<EOL>if inspect.isframe(obj): obj = obj.f_code<EOL>if inspect.iscode(obj):<EOL><INDENT>lineno = getattr(obj, '<STR_LIT>', None)-<NUM_LIT:1><EOL><DEDENT>if lineno is not None:<EOL><INDENT>if source_lines is None:<EOL><INDENT>return lineno+<NUM_LIT:1><EOL><DEDENT>pat = re.compile('<STR_LIT>')<EOL>for lineno in range(lineno, len(source_lines)):<EOL><INDENT>if pat.match(source_lines[lineno]):<EOL><INDENT>return lineno<EOL><DEDENT><DEDENT><DEDENT>return None<EOL>

Return a line number of the given object's docstring.  Note:
this method assumes that the object has a docstring.


suite = unittest.TestSuite()<EOL>if kw.get('<STR_LIT>', True):<EOL><INDENT>kw['<STR_LIT>'] = _normalize_module(kw.get('<STR_LIT>'))<EOL><DEDENT>for path in paths:<EOL><INDENT>suite.addTest(DocFileTest(path, **kw))<EOL><DEDENT>return suite<EOL>

A unittest suite for one or more doctest files.

    The path to each doctest file is given as a string; the
    interpretation of that string depends on the keyword argument
    "module_relative".

    A number of options may be provided as keyword arguments:

    module_relative
      If "module_relative" is True, then the given file paths are
      interpreted as os-independent module-relative paths.  By
      default, these paths are relative to the calling module's
      directory; but if the "package" argument is specified, then
      they are relative to that package.  To ensure os-independence,
      "filename" should use "/" characters to separate path
      segments, and may not be an absolute path (i.e., it may not
      begin with "/").

      If "module_relative" is False, then the given file paths are
      interpreted as os-specific paths.  These paths may be absolute
      or relative (to the current working directory).

    package
      A Python package or the name of a Python package whose directory
      should be used as the base directory for module relative paths.
      If "package" is not specified, then the calling module's
      directory is used as the base directory for module relative
      filenames.  It is an error to specify "package" if
      "module_relative" is False.

    setUp
      A set-up function.  This is called before running the
      tests in each file. The setUp function will be passed a DocTest
      object.  The setUp function can access the test globals as the
      globs attribute of the test passed.

    tearDown
      A tear-down function.  This is called after running the
      tests in each file.  The tearDown function will be passed a DocTest
      object.  The tearDown function can access the test globals as the
      globs attribute of the test passed.

    globs
      A dictionary containing initial global variables for the tests.

    optionflags
      A set of doctest option flags expressed as an integer.

    parser
      A DocTestParser (or subclass) that should be used to extract
      tests from the files.


if member not in container:<EOL><INDENT>standardMsg = '<STR_LIT>' % (safe_repr(member),<EOL>safe_repr(container))<EOL>self.fail(self._formatMessage(msg, standardMsg))<EOL><DEDENT>

assertIn and assertTrue does not exist in Python2.3


setup_py_file = tarfile.TarInfo(name='<STR_LIT>')<EOL>try:<EOL><INDENT>MemFile = StringIO.BytesIO<EOL><DEDENT>except AttributeError:<EOL><INDENT>MemFile = StringIO.StringIO<EOL><DEDENT>setup_py_bytes = MemFile(setup_py.encode('<STR_LIT:utf-8>'))<EOL>setup_py_file.size = len(setup_py_bytes.getvalue())<EOL>dist = tarfile.open(dist_path, '<STR_LIT>')<EOL>try:<EOL><INDENT>dist.addfile(setup_py_file, fileobj=setup_py_bytes)<EOL><DEDENT>finally:<EOL><INDENT>dist.close()<EOL><DEDENT>

Create a simple sdist tarball at dist_path, containing just a
    setup.py, the contents of which are provided by the setup_py string.


if not sys.platform.startswith('<STR_LIT:win32>'):<EOL><INDENT>return False<EOL><DEDENT>try:<EOL><INDENT>mod = __import__('<STR_LIT>')<EOL><DEDENT>except ImportError:<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Run this to determine if the local machine has win32com, and if it
does, include additional tests.


<EOL>time.sleep(<NUM_LIT:0.1>)<EOL>self._run = False<EOL>url = '<STR_LIT>' % vars(self)<EOL>try:<EOL><INDENT>if sys.version_info >= (<NUM_LIT:2>, <NUM_LIT:6>):<EOL><INDENT>urllib2.urlopen(url, timeout=<NUM_LIT:5>)<EOL><DEDENT>else:<EOL><INDENT>urllib2.urlopen(url)<EOL><DEDENT><DEDENT>except urllib2.URLError:<EOL><INDENT>pass<EOL><DEDENT>self.thread.join()<EOL>


distutils.core._setup_stop_after = "<STR_LIT>"<EOL>args.setdefault('<STR_LIT>',['<STR_LIT>'])<EOL>try:<EOL><INDENT>return setuptools.setup(**args)<EOL><DEDENT>finally:<EOL><INDENT>distutils.core._setup_stop_after = None<EOL><DEDENT>

Return distribution from 'setup(**args)', without executing commands


base, fragment = egg_info_for_url(url)<EOL>for dist in distros_for_location(url, base, metadata): yield dist<EOL>if fragment:<EOL><INDENT>match = EGG_FRAGMENT.match(fragment)<EOL>if match:<EOL><INDENT>for dist in interpret_distro_name(<EOL>url, match.group(<NUM_LIT:1>), metadata, precedence = CHECKOUT_DIST<EOL>):<EOL><INDENT>yield dist<EOL><DEDENT><DEDENT><DEDENT>

Yield egg or source distribution objects that might be found at a URL


def process_url(self, url, retrieve=False):

if url in self.scanned_urls and not retrieve:<EOL><INDENT>return<EOL><DEDENT>self.scanned_urls[url] = True<EOL>if not URL_SCHEME(url):<EOL><INDENT>self.process_filename(url)<EOL>return<EOL><DEDENT>else:<EOL><INDENT>dists = list(distros_for_url(url))<EOL>if dists:<EOL><INDENT>if not self.url_ok(url):<EOL><INDENT>return<EOL><DEDENT>self.debug("<STR_LIT>", url)<EOL><DEDENT><DEDENT>if dists or not retrieve or url in self.fetched_urls:<EOL><INDENT>list(map(self.add, dists))<EOL>return  <EOL><DEDENT>if not self.url_ok(url):<EOL><INDENT>self.fetched_urls[url] = True<EOL>return<EOL><DEDENT>self.info("<STR_LIT>", url)<EOL>f = self.open_url(url, "<STR_LIT>" % url)<EOL>if f is None: return<EOL>self.fetched_urls[url] = self.fetched_urls[f.url] = True<EOL>if '<STR_LIT:html>' not in f.headers.get('<STR_LIT>', '<STR_LIT>').lower():<EOL><INDENT>f.close()   <EOL>return<EOL><DEDENT>base = f.url     <EOL>page = f.read()<EOL>if not isinstance(page, str): <EOL><INDENT>if isinstance(f, urllib.error.HTTPError):<EOL><INDENT>charset = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>charset = f.headers.get_param('<STR_LIT>') or '<STR_LIT>'<EOL><DEDENT>page = page.decode(charset, "<STR_LIT:ignore>")<EOL><DEDENT>f.close()<EOL>for match in HREF.finditer(page):<EOL><INDENT>link = urllib.parse.urljoin(base, htmldecode(match.group(<NUM_LIT:1>)))<EOL>self.process_url(link)<EOL><DEDENT>if url.startswith(self.index_url) and getattr(f,'<STR_LIT:code>',None)!=<NUM_LIT>:<EOL><INDENT>page = self.process_index(url, page)<EOL><DEDENT>

Evaluate a URL as a possible download, and maybe retrieve it


scheme, server, path, param, query, frag = urllib.parse.urlparse(url)<EOL>filename = urllib.request.url2pathname(path)<EOL>if os.path.isfile(filename):<EOL><INDENT>return urllib.request.urlopen(url)<EOL><DEDENT>elif path.endswith('<STR_LIT:/>') and os.path.isdir(filename):<EOL><INDENT>files = []<EOL>for f in os.listdir(filename):<EOL><INDENT>if f=='<STR_LIT>':<EOL><INDENT>fp = open(os.path.join(filename,f),'<STR_LIT:rb>')<EOL>body = fp.read()<EOL>fp.close()<EOL>break<EOL><DEDENT>elif os.path.isdir(os.path.join(filename,f)):<EOL><INDENT>f+='<STR_LIT:/>'<EOL><DEDENT>files.append("<STR_LIT>" % (f,f))<EOL><DEDENT>else:<EOL><INDENT>body = ("<STR_LIT>" % url) +"<STR_LIT>" % '<STR_LIT:\n>'.join(files)<EOL><DEDENT>status, message = <NUM_LIT:200>, "<STR_LIT:OK>"<EOL><DEDENT>else:<EOL><INDENT>status, message, body = <NUM_LIT>, "<STR_LIT>", "<STR_LIT>"<EOL><DEDENT>return urllib.error.HTTPError(url, status, message,<EOL>{'<STR_LIT>':'<STR_LIT>'}, io.StringIO(body))<EOL>

Read a local path, with special support for directories


return entity_sub(decode_entity, text)<EOL>


def fetch_distribution(self,<EOL>requirement, tmpdir, force_scan=False, source=False, develop_ok=False,<EOL>local_index=None<EOL>):

<EOL>self.info("<STR_LIT>", requirement)<EOL>skipped = {}<EOL>dist = None<EOL>def find(req, env=None):<EOL><INDENT>if env is None:<EOL><INDENT>env = self<EOL><DEDENT>for dist in env[req.key]:<EOL><INDENT>if dist.precedence==DEVELOP_DIST and not develop_ok:<EOL><INDENT>if dist not in skipped:<EOL><INDENT>self.warn("<STR_LIT>",dist)<EOL>skipped[dist] = <NUM_LIT:1><EOL><DEDENT>continue<EOL><DEDENT>if dist in req and (dist.precedence<=SOURCE_DIST or not source):<EOL><INDENT>self.info("<STR_LIT>", dist)<EOL>return dist.clone(<EOL>location=self.download(dist.location, tmpdir)<EOL>)<EOL><DEDENT><DEDENT><DEDENT>if force_scan:<EOL><INDENT>self.prescan()<EOL>self.find_packages(requirement)<EOL>dist = find(requirement)<EOL><DEDENT>if local_index is not None:<EOL><INDENT>dist = dist or find(requirement, local_index)<EOL><DEDENT>if dist is None and self.to_scan is not None:<EOL><INDENT>self.prescan()<EOL>dist = find(requirement)<EOL><DEDENT>if dist is None and not force_scan:<EOL><INDENT>self.find_packages(requirement)<EOL>dist = find(requirement)<EOL><DEDENT>if dist is None:<EOL><INDENT>self.warn(<EOL>"<STR_LIT>",<EOL>(source and "<STR_LIT>" or "<STR_LIT>"),<EOL>requirement,<EOL>)<EOL><DEDENT>return dist<EOL>

Obtain a distribution suitable for fulfilling `requirement`

        `requirement` must be a ``pkg_resources.Requirement`` instance.
        If necessary, or if the `force_scan` flag is set, the requirement is
        searched for in the (online) package index as well as the locally
        installed packages.  If a distribution matching `requirement` is found,
        the returned distribution's ``location`` is the value you would have
        gotten from calling the ``download()`` method with the matching
        distribution's URL or filename.  If no matching distribution is found,
        ``None`` is returned.

        If the `source` flag is set, only source distributions and source
        checkout links will be considered.  Unless the `develop_ok` flag is
        set, development and system eggs (i.e., those using the ``.egg-info``
        format) will be ignored.


for url in urls:<EOL><INDENT>if (<EOL>self.to_scan is None        <EOL>or not URL_SCHEME(url)      <EOL>or url.startswith('<STR_LIT>')<EOL>or list(distros_for_url(url))   <EOL>):<EOL><INDENT>self.scan_url(url)<EOL><DEDENT>else:<EOL><INDENT>self.to_scan.append(url)<EOL><DEDENT><DEDENT>

Add `urls` to the list that will be prescanned for searches


def check_package(self, package, package_dir):

try:<EOL><INDENT>return self.packages_checked[package]<EOL><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT>init_py = _build_py.check_package(self, package, package_dir)<EOL>self.packages_checked[package] = init_py<EOL>if not init_py or not self.distribution.namespace_packages:<EOL><INDENT>return init_py<EOL><DEDENT>for pkg in self.distribution.namespace_packages:<EOL><INDENT>if pkg==package or pkg.startswith(package+'<STR_LIT:.>'):<EOL><INDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return init_py<EOL><DEDENT>f = open(init_py,'<STR_LIT>')<EOL>if '<STR_LIT>'.encode() not in f.read():<EOL><INDENT>from distutils import log<EOL>log.warn(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>'<STR_LIT>'<EOL>"<STR_LIT>", package<EOL>)<EOL><DEDENT>f.close()<EOL>return init_py<EOL>

Check namespace packages' __init__ for declare_namespace


def find_data_files(self, package, src_dir):

globs = (self.package_data.get('<STR_LIT>', [])<EOL>+ self.package_data.get(package, []))<EOL>files = self.manifest_files.get(package, [])[:]<EOL>for pattern in globs:<EOL><INDENT>files.extend(glob(os.path.join(src_dir, convert_path(pattern))))<EOL><DEDENT>return self.exclude_data_files(package, src_dir, files)<EOL>

Return filenames for package's data files in 'src_dir


self.analyze_manifest()<EOL>data = []<EOL>for package in self.packages or ():<EOL><INDENT>src_dir = self.get_package_dir(package)<EOL>build_dir = os.path.join(*([self.build_lib] + package.split('<STR_LIT:.>')))<EOL>plen = len(src_dir)+<NUM_LIT:1><EOL>filenames = [<EOL>file[plen:] for file in self.find_data_files(package, src_dir)<EOL>]<EOL>data.append( (package, src_dir, build_dir, filenames) )<EOL><DEDENT>return data<EOL>

Generate list of '(package,src_dir,build_dir,filenames)' tuples


lastdir = None<EOL>for package, src_dir, build_dir, filenames in self.data_files:<EOL><INDENT>for filename in filenames:<EOL><INDENT>target = os.path.join(build_dir, filename)<EOL>self.mkpath(os.path.dirname(target))<EOL>srcfile = os.path.join(src_dir, filename)<EOL>outf, copied = self.copy_file(srcfile, target)<EOL>srcfile = os.path.abspath(srcfile)<EOL>if copied and srcfile in self.distribution.convert_2to3_doctests:<EOL><INDENT>self.__doctests_2to3.append(outf)<EOL><DEDENT><DEDENT><DEDENT>


self._expand_attrs(['<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>',])<EOL>

Calls `os.path.expanduser` on install dirs.


try:<EOL><INDENT>fp = open(executable)<EOL>magic = fp.read(<NUM_LIT:2>)<EOL>fp.close()<EOL><DEDENT>except (OSError,IOError): return executable<EOL>return magic == '<STR_LIT>'<EOL>

Determine if the specified executable is a .sh (contains a #! line)


f = open(dist_filename,'<STR_LIT:rb>')<EOL>try:<EOL><INDENT>endrec = zipfile._EndRecData(f)<EOL>if endrec is None:<EOL><INDENT>return None<EOL><DEDENT>prepended = (endrec[<NUM_LIT:9>] - endrec[<NUM_LIT:5>]) - endrec[<NUM_LIT:6>]<EOL>if prepended < <NUM_LIT:12>:  <EOL><INDENT>return None<EOL><DEDENT>f.seek(prepended-<NUM_LIT:12>)<EOL>import struct, io, configparser<EOL>tag, cfglen, bmlen = struct.unpack("<STR_LIT>",f.read(<NUM_LIT:12>))<EOL>if tag not in (<NUM_LIT>, <NUM_LIT>):<EOL><INDENT>return None     <EOL><DEDENT>f.seek(prepended-(<NUM_LIT:12>+cfglen))<EOL>cfg = configparser.RawConfigParser({'<STR_LIT:version>':'<STR_LIT>','<STR_LIT>':'<STR_LIT>'})<EOL>try:<EOL><INDENT>part = f.read(cfglen)<EOL>if sys.version_info >= (<NUM_LIT:2>,<NUM_LIT:6>):<EOL><INDENT>null_byte = bytes([<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>null_byte = chr(<NUM_LIT:0>)<EOL><DEDENT>config = part.split(null_byte, <NUM_LIT:1>)[<NUM_LIT:0>]<EOL>config = config.decode('<STR_LIT:ascii>')<EOL>cfg.readfp(io.StringIO(config))<EOL><DEDENT>except configparser.Error:<EOL><INDENT>return None<EOL><DEDENT>if not cfg.has_section('<STR_LIT>') or not cfg.has_section('<STR_LIT>'):<EOL><INDENT>return None<EOL><DEDENT>return cfg<EOL><DEDENT>finally:<EOL><INDENT>f.close()<EOL><DEDENT>

Extract configuration data from a bdist_wininst .exe

    Returns a ConfigParser.RawConfigParser, or None


def write_script(self, script_name, contents, mode="<STR_LIT:t>", blockers=()):

self.delete_blockers(   <EOL>[os.path.join(self.script_dir,x) for x in blockers])<EOL>log.info("<STR_LIT>", script_name, self.script_dir)<EOL>target = os.path.join(self.script_dir, script_name)<EOL>self.add_output(target)<EOL>mask = current_umask()<EOL>if not self.dry_run:<EOL><INDENT>ensure_directory(target)<EOL>f = open(target,"<STR_LIT:w>"+mode)<EOL>f.write(contents)<EOL>f.close()<EOL>chmod(target, <NUM_LIT>-mask)<EOL><DEDENT>

Write an executable file to the scripts directory


def install_script(self, dist, script_name, script_text, dev_path=None):

spec = str(dist.as_requirement())<EOL>is_script = is_python_script(script_text, script_name)<EOL>def get_template(filename):<EOL><INDENT>"""<STR_LIT>"""<EOL>raw_bytes = resource_string('<STR_LIT>', template_name)<EOL>template_str = raw_bytes.decode('<STR_LIT:utf-8>')<EOL>clean_template = template_str.replace('<STR_LIT>', '<STR_LIT>')<EOL>return clean_template<EOL><DEDENT>if is_script:<EOL><INDENT>template_name = '<STR_LIT>'<EOL>if dev_path:<EOL><INDENT>template_name = template_name.replace('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>script_text = (get_script_header(script_text) +<EOL>get_template(template_name) % locals())<EOL><DEDENT>self.write_script(script_name, _to_ascii(script_text), '<STR_LIT:b>')<EOL>

Generate a legacy script wrapper and install it


if self.sitepy_installed:<EOL><INDENT>return  <EOL><DEDENT>sitepy = os.path.join(self.install_dir, "<STR_LIT>")<EOL>source = resource_string(Requirement.parse("<STR_LIT>"), "<STR_LIT>")<EOL>current = "<STR_LIT>"<EOL>if os.path.exists(sitepy):<EOL><INDENT>log.debug("<STR_LIT>", self.install_dir)<EOL>f = open(sitepy,'<STR_LIT:rb>')<EOL>current = f.read()<EOL>if sys.version_info >= (<NUM_LIT:3>,):<EOL><INDENT>current = current.decode()<EOL><DEDENT>f.close()<EOL>if not current.startswith('<STR_LIT>'):<EOL><INDENT>raise DistutilsError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % sitepy<EOL>)<EOL><DEDENT><DEDENT>if current != source:<EOL><INDENT>log.info("<STR_LIT>", sitepy)<EOL>if not self.dry_run:<EOL><INDENT>ensure_directory(sitepy)<EOL>f = open(sitepy,'<STR_LIT:wb>')<EOL>f.write(source)<EOL>f.close()<EOL><DEDENT>self.byte_compile([sitepy])<EOL><DEDENT>self.sitepy_installed = True<EOL>

Make sure there's a site.py in the target dir, if needed


def get_script_header(script_text, executable=sys_executable, wininst=False):

from distutils.command.build_scripts import first_line_re<EOL>if not isinstance(first_line_re.pattern, str):<EOL><INDENT>first_line_re = re.compile(first_line_re.pattern.decode())<EOL><DEDENT>first = (script_text+'<STR_LIT:\n>').splitlines()[<NUM_LIT:0>]<EOL>match = first_line_re.match(first)<EOL>options = '<STR_LIT>'<EOL>if match:<EOL><INDENT>options = match.group(<NUM_LIT:1>) or '<STR_LIT>'<EOL>if options: options = '<STR_LIT:U+0020>'+options<EOL><DEDENT>if wininst:<EOL><INDENT>executable = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>executable = nt_quote_arg(executable)<EOL><DEDENT>hdr = "<STR_LIT>" % locals()<EOL>if not isascii(hdr):<EOL><INDENT>if options:<EOL><INDENT>if options.strip().startswith('<STR_LIT:->'):<EOL><INDENT>options = '<STR_LIT>'+options.strip()[<NUM_LIT:1>:]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>options = '<STR_LIT>'<EOL><DEDENT><DEDENT>executable = fix_jython_executable(executable, options)<EOL>hdr = "<STR_LIT>" % locals()<EOL>return hdr<EOL>

Create a #! line, getting options (if any) from script_text


if not self.dirty:<EOL><INDENT>return<EOL><DEDENT>data = '<STR_LIT:\n>'.join(map(self.make_relative,self.paths))<EOL>if data:<EOL><INDENT>log.debug("<STR_LIT>", self.filename)<EOL>data = (<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>) % data<EOL>if os.path.islink(self.filename):<EOL><INDENT>os.unlink(self.filename)<EOL><DEDENT>f = open(self.filename,'<STR_LIT>')<EOL>f.write(data); f.close()<EOL><DEDENT>elif os.path.exists(self.filename):<EOL><INDENT>log.debug("<STR_LIT>", self.filename)<EOL>os.unlink(self.filename)<EOL><DEDENT>self.dirty = False<EOL>


<EOL>scheme = INSTALL_SCHEMES[name]<EOL>for key in SCHEME_KEYS:<EOL><INDENT>attrname = '<STR_LIT>' + key<EOL>if getattr(self, attrname) is None:<EOL><INDENT>setattr(self, attrname, scheme[key])<EOL><DEDENT><DEDENT>

Sets the install directories by applying the install schemes.


try:<EOL><INDENT>compile(text, filename, '<STR_LIT>')<EOL><DEDENT>except (SyntaxError, TypeError):<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>return True<EOL><DEDENT>


found = False<EOL>f = open(filename,'<STR_LIT>')<EOL>for line in iter(f.readline, '<STR_LIT>'):    <EOL><INDENT>parts = line.split()<EOL>if len(parts)==<NUM_LIT:2>:<EOL><INDENT>kind,length = parts<EOL>data = f.read(int(length))<EOL>if kind=='<STR_LIT>' and data=='<STR_LIT>':<EOL><INDENT>found = True<EOL><DEDENT>elif kind=='<STR_LIT>' and found:<EOL><INDENT>f.close()<EOL>break<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>f.close()<EOL>return<EOL><DEDENT>for line in data.splitlines():<EOL><INDENT>parts = line.split()<EOL>if parts:<EOL><INDENT>yield joinpath(dirname, parts[<NUM_LIT:0>])<EOL><DEDENT><DEDENT>


walker = os.walk(egg_dir)<EOL>base,dirs,files = next(walker)<EOL>if '<STR_LIT>' in dirs:<EOL><INDENT>dirs.remove('<STR_LIT>')<EOL><DEDENT>yield base,dirs,files<EOL>for bdf in walker:<EOL><INDENT>yield bdf<EOL><DEDENT>

Walk an unpacked egg's contents, skipping the metadata directory


def write_file(self, what, filename, data):

log.info("<STR_LIT>", what, filename)<EOL>if sys.version_info >= (<NUM_LIT:3>,):<EOL><INDENT>data = data.encode("<STR_LIT:utf-8>")<EOL><DEDENT>if not self.dry_run:<EOL><INDENT>f = open(filename, '<STR_LIT:wb>')<EOL>f.write(data)<EOL>f.close()<EOL><DEDENT>

Write `data` to `filename` (if not a dry run) after announcing it

        `what` is used in a log message to identify what is being written
        to the file.


log.info("<STR_LIT>", filename)<EOL>if not self.dry_run:<EOL><INDENT>os.unlink(filename)<EOL><DEDENT>

Delete `filename` (if not a dry run) after announcing it


manifest_filename = os.path.join(self.egg_info,"<STR_LIT>")<EOL>mm = manifest_maker(self.distribution)<EOL>mm.manifest = manifest_filename<EOL>mm.run()<EOL>self.filelist = mm.filelist<EOL>


if kind=='<STR_LIT>':<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>if kind=='<STR_LIT>':<EOL><INDENT>return os.path.join(<EOL>os.path.dirname(distutils.__file__),'<STR_LIT>'<EOL>)<EOL><DEDENT>if kind=='<STR_LIT:user>':<EOL><INDENT>dot = os.name=='<STR_LIT>' and '<STR_LIT:.>' or '<STR_LIT>'<EOL>return os.path.expanduser(convert_path("<STR_LIT>" % dot))<EOL><DEDENT>raise ValueError(<EOL>"<STR_LIT>", kind<EOL>)<EOL>

Get the filename of the distutils, local, global, or per-user config

    `kind` must be one of "local", "global", or "user"


all_files = []<EOL>for base, dirs, files in os.walk(dir):<EOL><INDENT>if base==os.curdir or base.startswith(os.curdir+os.sep):<EOL><INDENT>base = base[<NUM_LIT:2>:]<EOL><DEDENT>if base:<EOL><INDENT>files = [os.path.join(base, f) for f in files]<EOL><DEDENT>all_files.extend(filter(os.path.isfile, files))<EOL><DEDENT>return all_files<EOL>

Find all files under 'dir' and return the list of full filenames
    (relative to 'dir').


return self._validate_path(path)<EOL>


def open(self, file, flags, mode=<NUM_LIT>):

if flags & WRITE_FLAGS and not self._ok(file):<EOL><INDENT>self._violation("<STR_LIT>", file, flags, mode)<EOL><DEDENT>return _os.open(file,flags,mode)<EOL>


return getattr(self,self._feature_attrname(name))<EOL>

Return 1 if feature is included, 0 if excluded, 'None' if unknown


go = []<EOL>no = self.negative_opt.copy()<EOL>for name,feature in list(self.features.items()):<EOL><INDENT>self._set_feature(name,None)<EOL>feature.validate(self)<EOL>if feature.optional:<EOL><INDENT>descr = feature.description<EOL>incdef = '<STR_LIT>'<EOL>excdef='<STR_LIT>'<EOL>if not feature.include_by_default():<EOL><INDENT>excdef, incdef = incdef, excdef<EOL><DEDENT>go.append(('<STR_LIT>'+name, None, '<STR_LIT>'+descr+incdef))<EOL>go.append(('<STR_LIT>'+name, None, '<STR_LIT>'+descr+excdef))<EOL>no['<STR_LIT>'+name] = '<STR_LIT>'+name<EOL><DEDENT><DEDENT>self.global_options = self.feature_options = go + self.global_options<EOL>self.negative_opt = self.feature_negopt = no<EOL>

Add --with-X/--without-X options based on optional features


return self.available and self.standard<EOL>

Should this feature be included by default?


return '<STR_LIT>'+name.replace('<STR_LIT:->','<STR_LIT:_>')<EOL>

Convert feature name to corresponding option attribute name


dist.exclude(**self.extras)<EOL>if self.remove:<EOL><INDENT>for item in self.remove:<EOL><INDENT>dist.exclude_package(item)<EOL><DEDENT><DEDENT>

Ensure feature is excluded from distribution

        You may override this in a subclass to perform additional operations on
        the distribution.  This method will be called at most once per
        feature, and only after all included features have been asked to
        include themselves.


for pkg in self.packages or ():<EOL><INDENT>yield pkg<EOL><DEDENT>for module in self.py_modules or ():<EOL><INDENT>yield module<EOL><DEDENT>for ext in self.ext_modules or ():<EOL><INDENT>if isinstance(ext,tuple):<EOL><INDENT>name, buildinfo = ext<EOL><DEDENT>else:<EOL><INDENT>name = ext.name<EOL><DEDENT>if name.endswith('<STR_LIT>'):<EOL><INDENT>name = name[:-<NUM_LIT:6>]<EOL><DEDENT>yield name<EOL><DEDENT>

Yield all packages, modules, and extension names in distribution


import sys<EOL>if sys.version_info < (<NUM_LIT:3>,) or self.help_commands:<EOL><INDENT>return _Distribution.handle_display_options(self, option_order)<EOL><DEDENT>import io<EOL>if not isinstance(sys.stdout, io.TextIOWrapper):<EOL><INDENT>return _Distribution.handle_display_options(self, option_order)<EOL><DEDENT>if sys.stdout.encoding.lower() in ('<STR_LIT:utf-8>', '<STR_LIT:utf8>'):<EOL><INDENT>return _Distribution.handle_display_options(self, option_order)<EOL><DEDENT>encoding = sys.stdout.encoding<EOL>errors = sys.stdout.errors<EOL>newline = sys.platform != '<STR_LIT:win32>' and '<STR_LIT:\n>' or None<EOL>line_buffering = sys.stdout.line_buffering<EOL>sys.stdout = io.TextIOWrapper(<EOL>sys.stdout.detach(), '<STR_LIT:utf-8>', errors, newline, line_buffering)<EOL>try:<EOL><INDENT>return _Distribution.handle_display_options(self, option_order)<EOL><DEDENT>finally:<EOL><INDENT>sys.stdout = io.TextIOWrapper(<EOL>sys.stdout.detach(), encoding, errors, newline, line_buffering)<EOL><DEDENT>

If there were any non-global "display-only" options
        (--help-commands or the metadata display options) on the command
        line, display the requested info and return true; else return
        false.


def check_entry_points(dist, attr, value):

try:<EOL><INDENT>pkg_resources.EntryPoint.parse_map(value)<EOL><DEDENT>except ValueError as e:<EOL><INDENT>raise DistutilsSetupError(e)<EOL><DEDENT>

Verify that entry_points map is parseable


pfx = package+'<STR_LIT:.>'<EOL>if self.packages:<EOL><INDENT>self.packages = [<EOL>p for p in self.packages<EOL>if p != package and not p.startswith(pfx)<EOL>]<EOL><DEDENT>if self.py_modules:<EOL><INDENT>self.py_modules = [<EOL>p for p in self.py_modules<EOL>if p != package and not p.startswith(pfx)<EOL>]<EOL><DEDENT>if self.ext_modules:<EOL><INDENT>self.ext_modules = [<EOL>p for p in self.ext_modules<EOL>if p.name != package and not p.name.startswith(pfx)<EOL>]<EOL><DEDENT>

Remove packages, modules, and extensions in named package


def pyx_to_c(source):<EOL><INDENT>if source.endswith('<STR_LIT>'):<EOL><INDENT>source = source[:-<NUM_LIT:4>] + '<STR_LIT>'<EOL><DEDENT>return source<EOL><DEDENT>self.sources = map(pyx_to_c, self.sources)<EOL>


from array import array<EOL>from dis import HAVE_ARGUMENT, EXTENDED_ARG<EOL>bytes = array('<STR_LIT:b>',code.co_code)<EOL>eof = len(code.co_code)<EOL>ptr = <NUM_LIT:0><EOL>extended_arg = <NUM_LIT:0><EOL>while ptr<eof:<EOL><INDENT>op = bytes[ptr]<EOL>if op>=HAVE_ARGUMENT:<EOL><INDENT>arg = bytes[ptr+<NUM_LIT:1>] + bytes[ptr+<NUM_LIT:2>]*<NUM_LIT> + extended_arg<EOL>ptr += <NUM_LIT:3><EOL>if op==EXTENDED_ARG:<EOL><INDENT>extended_arg = arg * <NUM_LIT><EOL>continue<EOL><DEDENT><DEDENT>else:<EOL><INDENT>arg = None<EOL>ptr += <NUM_LIT:1><EOL><DEDENT>yield op,arg<EOL><DEDENT>

Yield '(op,arg)' pair for each operation in code object 'code


if self.requested_version is not None:<EOL><INDENT>return '<STR_LIT>' % (self.name,self.requested_version)<EOL><DEDENT>return self.name<EOL>

Return full package/distribution name, w/version


parts = module.split('<STR_LIT:.>')<EOL>while parts:<EOL><INDENT>part = parts.pop(<NUM_LIT:0>)<EOL>f, path, (suffix,mode,kind) = info = imp.find_module(part, paths)<EOL>if kind==PKG_DIRECTORY:<EOL><INDENT>parts = parts or ['<STR_LIT>']<EOL>paths = [path]<EOL><DEDENT>elif parts:<EOL><INDENT>raise ImportError("<STR_LIT>" % (parts,module))<EOL><DEDENT><DEDENT>return info<EOL>

Just like 'imp.find_module()', but with package support


if not isinstance(cls,type):<EOL><INDENT>class cls(cls,object): pass<EOL>return cls.__mro__[<NUM_LIT:1>:]<EOL><DEDENT>return cls.__mro__<EOL>


for attr in (<EOL>'<STR_LIT>', '<STR_LIT:version>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:location>',<EOL>'<STR_LIT>'<EOL>):<EOL><INDENT>kw.setdefault(attr, getattr(self,attr,None))<EOL><DEDENT>kw.setdefault('<STR_LIT>', self._provider)<EOL>return self.__class__(**kw)<EOL>

Copy this distribution, substituting in any changed keyword args


new = self.__class__([], platform=None, python=None)<EOL>for env in self, other:<EOL><INDENT>new += env<EOL><DEDENT>return new<EOL>

Add an environment or distribution to an environment


return name.replace('<STR_LIT:->','<STR_LIT:_>')<EOL>

Convert a project or version name to its filename-escaped form

    Any '-' characters are currently replaced with '_'.


importer = get_importer(path_item)<EOL>finder = _find_adapter(_distribution_finders, importer)<EOL>return finder(importer, path_item, only)<EOL>

Yield distributions accessible via `path_item`


self.zipinfo = zipimport._zip_directory_cache[importer.archive]<EOL>self.zip_pre = importer.archive+os.sep<EOL>self.loader = importer<EOL>if importer.prefix:<EOL><INDENT>self.module_path = os.path.join(importer.archive, importer.prefix)<EOL><DEDENT>else:<EOL><INDENT>self.module_path = importer.archive<EOL><DEDENT>self._setup_prefix()<EOL>

Create a metadata provider from a zipimporter


Yield named metadata resource as list of non-blank non-comment lines

       Leading and trailing whitespace is stripped from each line, and lines
       with ``#`` as the first non-blank character are omitted.


def file_ns_handler(importer, path_item, packageName, module):

subpath = os.path.join(path_item, packageName.split('<STR_LIT:.>')[-<NUM_LIT:1>])<EOL>normalized = _normalize_cached(subpath)<EOL>for item in module.__path__:<EOL><INDENT>if _normalize_cached(item)==normalized:<EOL><INDENT>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return subpath<EOL><DEDENT>

Compute an ns-package subpath for a filesystem or zipfile importer


def parse_group(cls, group, lines, dist=None):

if not MODULE(group):<EOL><INDENT>raise ValueError("<STR_LIT>", group)<EOL><DEDENT>this = {}<EOL>for line in yield_lines(lines):<EOL><INDENT>ep = cls.parse(line, dist)<EOL>if ep.name in this:<EOL><INDENT>raise ValueError("<STR_LIT>", group, ep.name)<EOL><DEDENT>this[ep.name]=ep<EOL><DEDENT>return this<EOL>


return self.get_entry_map(group).get(name)<EOL>

Return the EntryPoint object for `group`+`name`, or ``None``


try:<EOL><INDENT>attrs = extras = ()<EOL>name,value = src.split('<STR_LIT:=>',<NUM_LIT:1>)<EOL>if '<STR_LIT:[>' in value:<EOL><INDENT>value,extras = value.split('<STR_LIT:[>',<NUM_LIT:1>)<EOL>req = Requirement.parse("<STR_LIT>"+extras)<EOL>if req.specs: raise ValueError<EOL>extras = req.extras<EOL><DEDENT>if '<STR_LIT::>' in value:<EOL><INDENT>value,attrs = value.split('<STR_LIT::>',<NUM_LIT:1>)<EOL>if not MODULE(attrs.rstrip()):<EOL><INDENT>raise ValueError<EOL><DEDENT>attrs = attrs.rstrip().split('<STR_LIT:.>')<EOL><DEDENT><DEDENT>except ValueError:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>",<EOL>src<EOL>)<EOL><DEDENT>else:<EOL><INDENT>return cls(name.strip(), value.strip(), attrs, extras, dist)<EOL><DEDENT>

Parse a single entry point from string `src`

        Entry point syntax follows the form::

            name = some.module:some.attr [extra1,extra2]

        The entry name and module name are required, but the ``:attrs`` and
        ``[extras]`` parts are optional


importer = get_importer(path_item)<EOL>if importer is None:<EOL><INDENT>return None<EOL><DEDENT>loader = importer.find_module(packageName)<EOL>if loader is None:<EOL><INDENT>return None<EOL><DEDENT>module = sys.modules.get(packageName)<EOL>if module is None:<EOL><INDENT>module = sys.modules[packageName] = types.ModuleType(packageName)<EOL>module.__path__ = []<EOL>_set_parent_ns(packageName)<EOL><DEDENT>elif not hasattr(module,'<STR_LIT>'):<EOL><INDENT>raise TypeError("<STR_LIT>", packageName)<EOL><DEDENT>handler = _find_adapter(_namespace_handlers, importer)<EOL>subpath = handler(importer, path_item, packageName, module)<EOL>if subpath is not None:<EOL><INDENT>path = module.__path__<EOL>path.append(subpath)<EOL>loader.load_module(packageName)<EOL>for path_item in path:<EOL><INDENT>if path_item not in module.__path__:<EOL><INDENT>module.__path__.append(path_item)<EOL><DEDENT><DEDENT><DEDENT>return subpath<EOL>

Ensure that named package includes a subpath of path_item (if needed)


if callback in self.callbacks:<EOL><INDENT>return<EOL><DEDENT>self.callbacks.append(callback)<EOL>for dist in self:<EOL><INDENT>callback(dist)<EOL><DEDENT>

Invoke `callback` for all distributions (including existing ones)