self.zmq_socket.send_pyobj(message, copy=True)<EOL>reply = self.zmq_socket.recv_pyobj()<EOL>return reply<EOL>

This function needs to be fast at the same time aware of the possibility of
        ZMQ pipes overflowing.

        The timeout increases slowly if contention is detected on ZMQ pipes.
        We could set copy=False and get slightly better latency but this results
        in ZMQ sockets reaching a broken state once there are ~10k tasks in flight.
        This issue can be magnified if each the serialized buffer itself is larger.


def __init__(self, ip_address, port_range):

self.context = zmq.Context()<EOL>self.zmq_socket = self.context.socket(zmq.DEALER)<EOL>self.zmq_socket.set_hwm(<NUM_LIT:0>)<EOL>self.port = self.zmq_socket.bind_to_random_port("<STR_LIT>".format(ip_address),<EOL>min_port=port_range[<NUM_LIT:0>],<EOL>max_port=port_range[<NUM_LIT:1>])<EOL>self.poller = zmq.Poller()<EOL>self.poller.register(self.zmq_socket, zmq.POLLOUT)<EOL>

Parameters
----------

ip_address: str
   IP address of the client (where Parsl runs)
port_range: tuple(int, int)
   Port range for the comms between client and interchange


timeout_ms = <NUM_LIT:0><EOL>while True:<EOL><INDENT>socks = dict(self.poller.poll(timeout=timeout_ms))<EOL>if self.zmq_socket in socks and socks[self.zmq_socket] == zmq.POLLOUT:<EOL><INDENT>self.zmq_socket.send_pyobj(message, copy=True)<EOL>return<EOL><DEDENT>else:<EOL><INDENT>timeout_ms += <NUM_LIT:1><EOL>logger.debug("<STR_LIT>".format(timeout_ms))<EOL><DEDENT><DEDENT>


self.context = zmq.Context()<EOL>self.results_receiver = self.context.socket(zmq.DEALER)<EOL>self.results_receiver.set_hwm(<NUM_LIT:0>)<EOL>self.port = self.results_receiver.bind_to_random_port("<STR_LIT>".format(ip_address),<EOL>min_port=port_range[<NUM_LIT:0>],<EOL>max_port=port_range[<NUM_LIT:1>])<EOL>


user_ns = locals()<EOL>user_ns.update({'<STR_LIT>': __builtins__})<EOL>f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False)<EOL>prefix = "<STR_LIT>"<EOL>fname = prefix + "<STR_LIT:f>"<EOL>argname = prefix + "<STR_LIT:args>"<EOL>kwargname = prefix + "<STR_LIT>"<EOL>resultname = prefix + "<STR_LIT:result>"<EOL>user_ns.update({fname: f,<EOL>argname: args,<EOL>kwargname: kwargs,<EOL>resultname: resultname})<EOL>code = "<STR_LIT>".format(resultname, fname,<EOL>argname, kwargname)<EOL>try:<EOL><INDENT>exec(code, user_ns, user_ns)<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.warning("<STR_LIT>".format(e), exc_info=True)<EOL>raise e<EOL><DEDENT>else:<EOL><INDENT>return user_ns.get(resultname)<EOL><DEDENT>

Deserialize the buffer and execute the task.

    Returns the result or throws exception.


def worker(worker_id, pool_id, task_queue, result_queue, worker_queue):

start_file_logger('<STR_LIT>'.format(args.logdir, pool_id, worker_id),<EOL>worker_id,<EOL>name="<STR_LIT>",<EOL>level=logging.DEBUG if args.debug else logging.INFO)<EOL>logger.info('<STR_LIT>'.format(worker_id))<EOL>if args.debug:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL><DEDENT>while True:<EOL><INDENT>worker_queue.put(worker_id)<EOL>req = task_queue.get()<EOL>tid = req['<STR_LIT>']<EOL>logger.info("<STR_LIT>".format(tid))<EOL>try:<EOL><INDENT>worker_queue.get()<EOL><DEDENT>except queue.Empty:<EOL><INDENT>logger.warning("<STR_LIT>".format(worker_id))<EOL>pass<EOL><DEDENT>try:<EOL><INDENT>result = execute_task(req['<STR_LIT>'])<EOL>serialized_result = serialize_object(result)<EOL><DEDENT>except Exception:<EOL><INDENT>result_package = {'<STR_LIT>': tid, '<STR_LIT>': serialize_object(RemoteExceptionWrapper(*sys.exc_info()))}<EOL><DEDENT>else:<EOL><INDENT>result_package = {'<STR_LIT>': tid, '<STR_LIT:result>': serialized_result}<EOL><DEDENT>logger.info("<STR_LIT>".format(tid))<EOL>pkl_package = pickle.dumps(result_package)<EOL>result_queue.put(pkl_package)<EOL><DEDENT>

Put request token into queue
Get task from task_queue
Pop request from queue
Put result into result_queue


def start_file_logger(filename, rank, name='<STR_LIT>', level=logging.DEBUG, format_string=None):

if format_string is None:<EOL><INDENT>format_string = "<STR_LIT>".format(rank)<EOL><DEDENT>global logger<EOL>logger = logging.getLogger(name)<EOL>logger.setLevel(logging.DEBUG)<EOL>handler = logging.FileHandler(filename)<EOL>handler.setLevel(level)<EOL>formatter = logging.Formatter(format_string, datefmt='<STR_LIT>')<EOL>handler.setFormatter(formatter)<EOL>logger.addHandler(handler)<EOL>

Add a stream log handler.

    Args:
        - filename (string): Name of the file to write logs to
        - name (string): Logger name
        - level (logging.LEVEL): Set the logging level.
        - format_string (string): Set the format string

    Returns:
       -  None


def set_stream_logger(name='<STR_LIT>', level=logging.DEBUG, format_string=None):

if format_string is None:<EOL><INDENT>format_string = "<STR_LIT>"<EOL><DEDENT>global logger<EOL>logger = logging.getLogger(name)<EOL>logger.setLevel(logging.DEBUG)<EOL>handler = logging.StreamHandler()<EOL>handler.setLevel(level)<EOL>formatter = logging.Formatter(format_string, datefmt='<STR_LIT>')<EOL>handler.setFormatter(formatter)<EOL>logger.addHandler(handler)<EOL>

Add a stream log handler.

    Args:
         - name (string) : Set the logger name.
         - level (logging.LEVEL) : Set to logging.DEBUG by default.
         - format_string (sting) : Set to None by default.

    Returns:
         - None


logger.info("<STR_LIT>")<EOL>self.context = zmq.Context()<EOL>self.task_incoming = self.context.socket(zmq.DEALER)<EOL>self.task_incoming.setsockopt(zmq.IDENTITY, uid.encode('<STR_LIT:utf-8>'))<EOL>self.task_incoming.setsockopt(zmq.LINGER, <NUM_LIT:0>)<EOL>self.task_incoming.connect(task_q_url)<EOL>self.result_outgoing = self.context.socket(zmq.DEALER)<EOL>self.result_outgoing.setsockopt(zmq.IDENTITY, uid.encode('<STR_LIT:utf-8>'))<EOL>self.result_outgoing.setsockopt(zmq.LINGER, <NUM_LIT:0>)<EOL>self.result_outgoing.connect(result_q_url)<EOL>logger.info("<STR_LIT>")<EOL>self.uid = uid<EOL>self.block_id = block_id<EOL>cores_on_node = multiprocessing.cpu_count()<EOL>self.max_workers = max_workers<EOL>self.prefetch_capacity = prefetch_capacity<EOL>self.worker_count = min(max_workers,<EOL>math.floor(cores_on_node / cores_per_worker))<EOL>logger.info("<STR_LIT>".format(self.worker_count))<EOL>self.pending_task_queue = multiprocessing.Queue()<EOL>self.pending_result_queue = multiprocessing.Queue()<EOL>self.ready_worker_queue = multiprocessing.Queue()<EOL>self.max_queue_size = self.prefetch_capacity + self.worker_count<EOL>self.tasks_per_round = <NUM_LIT:1><EOL>self.heartbeat_period = heartbeat_period<EOL>self.heartbeat_threshold = heartbeat_threshold<EOL>self.poll_period = poll_period<EOL>

Parameters
----------
worker_url : str
     Worker url on which workers will attempt to connect back

uid : str
     string unique identifier

block_id : str
     Block identifier that maps managers to the provider blocks they belong to.

cores_per_worker : float
     cores to be assigned to each worker. Oversubscription is possible
     by setting cores_per_worker < 1.0. Default=1

max_workers : int
     caps the maximum number of workers that can be launched.
     default: infinity

prefetch_capacity : int
     Number of tasks that could be prefetched over available worker capacity.
     When there are a few tasks (<100) or when tasks are long running, this option should
     be set to 0 for better load balancing. Default is 0.

heartbeat_threshold : int
     Seconds since the last message from the interchange after which the
     interchange is assumed to be un-available, and the manager initiates shutdown. Default:120s

     Number of seconds since the last message from the interchange after which the worker
     assumes that the interchange is lost and the manager shuts down. Default:120

heartbeat_period : int
     Number of seconds after which a heartbeat message is sent to the interchange

poll_period : int
     Timeout period used by the manager in milliseconds. Default: 10ms


msg = {'<STR_LIT>': PARSL_VERSION,<EOL>'<STR_LIT>': "<STR_LIT>".format(sys.version_info.major,<EOL>sys.version_info.minor,<EOL>sys.version_info.micro),<EOL>'<STR_LIT>': self.worker_count,<EOL>'<STR_LIT>': self.block_id,<EOL>'<STR_LIT>': self.prefetch_capacity,<EOL>'<STR_LIT>': self.worker_count + self.prefetch_capacity,<EOL>'<STR_LIT>': platform.system(),<EOL>'<STR_LIT>': platform.node(),<EOL>'<STR_LIT>': os.getcwd(),<EOL>}<EOL>b_msg = json.dumps(msg).encode('<STR_LIT:utf-8>')<EOL>return b_msg<EOL>

Creates a registration message to identify the worker to the interchange


heartbeat = (HEARTBEAT_CODE).to_bytes(<NUM_LIT:4>, "<STR_LIT>")<EOL>r = self.task_incoming.send(heartbeat)<EOL>logger.debug("<STR_LIT>".format(r))<EOL>

Send heartbeat to the incoming task queue


logger.info("<STR_LIT>")<EOL>poller = zmq.Poller()<EOL>poller.register(self.task_incoming, zmq.POLLIN)<EOL>msg = self.create_reg_message()<EOL>logger.debug("<STR_LIT>".format(msg))<EOL>self.task_incoming.send(msg)<EOL>last_beat = time.time()<EOL>last_interchange_contact = time.time()<EOL>task_recv_counter = <NUM_LIT:0><EOL>poll_timer = self.poll_period<EOL>while not kill_event.is_set():<EOL><INDENT>ready_worker_count = self.ready_worker_queue.qsize()<EOL>pending_task_count = self.pending_task_queue.qsize()<EOL>logger.debug("<STR_LIT>".format(ready_worker_count,<EOL>pending_task_count))<EOL>if time.time() > last_beat + self.heartbeat_period:<EOL><INDENT>self.heartbeat()<EOL>last_beat = time.time()<EOL><DEDENT>if pending_task_count < self.max_queue_size and ready_worker_count > <NUM_LIT:0>:<EOL><INDENT>logger.debug("<STR_LIT>".format(ready_worker_count))<EOL>msg = ((ready_worker_count).to_bytes(<NUM_LIT:4>, "<STR_LIT>"))<EOL>self.task_incoming.send(msg)<EOL><DEDENT>socks = dict(poller.poll(timeout=poll_timer))<EOL>if self.task_incoming in socks and socks[self.task_incoming] == zmq.POLLIN:<EOL><INDENT>poll_timer = <NUM_LIT:0><EOL>_, pkl_msg = self.task_incoming.recv_multipart()<EOL>tasks = pickle.loads(pkl_msg)<EOL>last_interchange_contact = time.time()<EOL>if tasks == '<STR_LIT>':<EOL><INDENT>logger.critical("<STR_LIT>")<EOL>kill_event.set()<EOL>break<EOL><DEDENT>elif tasks == HEARTBEAT_CODE:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>task_recv_counter += len(tasks)<EOL>logger.debug("<STR_LIT>".format([t['<STR_LIT>'] for t in tasks],<EOL>task_recv_counter))<EOL>for task in tasks:<EOL><INDENT>self.pending_task_queue.put(task)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>if not poll_timer:<EOL><INDENT>poll_timer = self.poll_period<EOL><DEDENT>poll_timer = min(self.heartbeat_period * <NUM_LIT:1000>, poll_timer * <NUM_LIT:2>)<EOL>if time.time() > last_interchange_contact + self.heartbeat_threshold:<EOL><INDENT>logger.critical("<STR_LIT>")<EOL>kill_event.set()<EOL>logger.critical("<STR_LIT>")<EOL>break<EOL><DEDENT><DEDENT><DEDENT>

Pull tasks from the incoming tasks 0mq pipe onto the internal
        pending task queue

        Parameters:
        -----------
        kill_event : threading.Event
              Event to let the thread know when it is time to die.


logger.debug("<STR_LIT>")<EOL>push_poll_period = max(<NUM_LIT:10>, self.poll_period) / <NUM_LIT:1000>    <EOL>logger.debug("<STR_LIT>".format(push_poll_period))<EOL>last_beat = time.time()<EOL>items = []<EOL>while not kill_event.is_set():<EOL><INDENT>try:<EOL><INDENT>r = self.pending_result_queue.get(block=True, timeout=push_poll_period)<EOL>items.append(r)<EOL><DEDENT>except queue.Empty:<EOL><INDENT>pass<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.exception("<STR_LIT>".format(e))<EOL><DEDENT>if len(items) >= self.max_queue_size or time.time() > last_beat + push_poll_period:<EOL><INDENT>last_beat = time.time()<EOL>if items:<EOL><INDENT>self.result_outgoing.send_multipart(items)<EOL>items = []<EOL><DEDENT><DEDENT><DEDENT>logger.critical("<STR_LIT>")<EOL>

Listens on the pending_result_queue and sends out results via 0mq

        Parameters:
        -----------
        kill_event : threading.Event
              Event to let the thread know when it is time to die.


start = time.time()<EOL>self._kill_event = threading.Event()<EOL>self.procs = {}<EOL>for worker_id in range(self.worker_count):<EOL><INDENT>p = multiprocessing.Process(target=worker, args=(worker_id,<EOL>self.uid,<EOL>self.pending_task_queue,<EOL>self.pending_result_queue,<EOL>self.ready_worker_queue,<EOL>))<EOL>p.start()<EOL>self.procs[worker_id] = p<EOL><DEDENT>logger.debug("<STR_LIT>")<EOL>self._task_puller_thread = threading.Thread(target=self.pull_tasks,<EOL>args=(self._kill_event,))<EOL>self._result_pusher_thread = threading.Thread(target=self.push_results,<EOL>args=(self._kill_event,))<EOL>self._task_puller_thread.start()<EOL>self._result_pusher_thread.start()<EOL>logger.info("<STR_LIT>")<EOL>self._kill_event.wait()<EOL>logger.critical("<STR_LIT>")<EOL>self._task_puller_thread.join()<EOL>self._result_pusher_thread.join()<EOL>for proc_id in self.procs:<EOL><INDENT>self.procs[proc_id].terminate()<EOL>logger.critical("<STR_LIT>".format(self.procs[proc_id],<EOL>self.procs[proc_id].is_alive()))<EOL>self.procs[proc_id].join()<EOL>logger.debug("<STR_LIT>".format(self.procs[proc_id]))<EOL><DEDENT>self.task_incoming.close()<EOL>self.result_outgoing.close()<EOL>self.context.term()<EOL>delta = time.time() - start<EOL>logger.info("<STR_LIT>".format(delta))<EOL>return<EOL>

Start the worker processes.

        TODO: Move task receiving to a thread


def start_file_logger(filename, name='<STR_LIT>', level=logging.DEBUG, format_string=None):

if format_string is None:<EOL><INDENT>format_string = "<STR_LIT>"<EOL><DEDENT>global logger<EOL>logger = logging.getLogger(name)<EOL>logger.setLevel(level)<EOL>handler = logging.FileHandler(filename)<EOL>handler.setLevel(level)<EOL>formatter = logging.Formatter(format_string, datefmt='<STR_LIT>')<EOL>handler.setFormatter(formatter)<EOL>logger.addHandler(handler)<EOL>

Add a stream log handler.

    Parameters
    ---------

    filename: string
        Name of the file to write logs to. Required.
    name: string
        Logger name. Default="parsl.executors.interchange"
    level: logging.LEVEL
        Set the logging level. Default=logging.DEBUG
        - format_string (string): Set the format string
    format_string: string
        Format string to use.

    Returns
    -------
        None.


<EOL>ic = Interchange(*args, **kwargs)<EOL>comm_q.put((ic.worker_task_port,<EOL>ic.worker_result_port))<EOL>ic.start()<EOL>

Start the interchange process

    The executor is expected to call this function. The args, kwargs match that of the Interchange.__init__


self.logdir = logdir<EOL>try:<EOL><INDENT>os.makedirs(self.logdir)<EOL><DEDENT>except FileExistsError:<EOL><INDENT>pass<EOL><DEDENT>start_file_logger("<STR_LIT>".format(self.logdir), level=logging_level)<EOL>logger.debug("<STR_LIT>")<EOL>self.client_address = client_address<EOL>self.interchange_address = interchange_address<EOL>self.suppress_failure = suppress_failure<EOL>self.poll_period = poll_period<EOL>logger.info("<STR_LIT>".format(<EOL>client_address, client_ports[<NUM_LIT:0>], client_ports[<NUM_LIT:1>], client_ports[<NUM_LIT:2>]))<EOL>self.context = zmq.Context()<EOL>self.task_incoming = self.context.socket(zmq.DEALER)<EOL>self.task_incoming.set_hwm(<NUM_LIT:0>)<EOL>self.task_incoming.RCVTIMEO = <NUM_LIT:10>  <EOL>self.task_incoming.connect("<STR_LIT>".format(client_address, client_ports[<NUM_LIT:0>]))<EOL>self.results_outgoing = self.context.socket(zmq.DEALER)<EOL>self.results_outgoing.set_hwm(<NUM_LIT:0>)<EOL>self.results_outgoing.connect("<STR_LIT>".format(client_address, client_ports[<NUM_LIT:1>]))<EOL>self.command_channel = self.context.socket(zmq.REP)<EOL>self.command_channel.RCVTIMEO = <NUM_LIT:1000>  <EOL>self.command_channel.connect("<STR_LIT>".format(client_address, client_ports[<NUM_LIT:2>]))<EOL>logger.info("<STR_LIT>")<EOL>self.pending_task_queue = queue.Queue(maxsize=<NUM_LIT:10> ** <NUM_LIT:6>)<EOL>self.worker_ports = worker_ports<EOL>self.worker_port_range = worker_port_range<EOL>self.task_outgoing = self.context.socket(zmq.ROUTER)<EOL>self.task_outgoing.set_hwm(<NUM_LIT:0>)<EOL>self.results_incoming = self.context.socket(zmq.ROUTER)<EOL>self.results_incoming.set_hwm(<NUM_LIT:0>)<EOL>if self.worker_ports:<EOL><INDENT>self.worker_task_port = self.worker_ports[<NUM_LIT:0>]<EOL>self.worker_result_port = self.worker_ports[<NUM_LIT:1>]<EOL>self.task_outgoing.bind("<STR_LIT>".format(self.worker_task_port))<EOL>self.results_incoming.bind("<STR_LIT>".format(self.worker_result_port))<EOL><DEDENT>else:<EOL><INDENT>self.worker_task_port = self.task_outgoing.bind_to_random_port('<STR_LIT>',<EOL>min_port=worker_port_range[<NUM_LIT:0>],<EOL>max_port=worker_port_range[<NUM_LIT:1>], max_tries=<NUM_LIT:100>)<EOL>self.worker_result_port = self.results_incoming.bind_to_random_port('<STR_LIT>',<EOL>min_port=worker_port_range[<NUM_LIT:0>],<EOL>max_port=worker_port_range[<NUM_LIT:1>], max_tries=<NUM_LIT:100>)<EOL><DEDENT>logger.info("<STR_LIT>".format(<EOL>self.worker_task_port, self.worker_result_port))<EOL>self._ready_manager_queue = {}<EOL>self.heartbeat_threshold = heartbeat_threshold<EOL>self.current_platform = {'<STR_LIT>': PARSL_VERSION,<EOL>'<STR_LIT>': "<STR_LIT>".format(sys.version_info.major,<EOL>sys.version_info.minor,<EOL>sys.version_info.micro),<EOL>'<STR_LIT>': platform.system(),<EOL>'<STR_LIT>': platform.node(),<EOL>'<STR_LIT>': os.getcwd()}<EOL>logger.info("<STR_LIT>".format(self.current_platform))<EOL>

Parameters
----------
client_address : str
     The ip address at which the parsl client can be reached. Default: "127.0.0.1"

interchange_address : str
     The ip address at which the workers will be able to reach the Interchange. Default: "127.0.0.1"

client_ports : triple(int, int, int)
     The ports at which the client can be reached

worker_ports : tuple(int, int)
     The specific two ports at which workers will connect to the Interchange. Default: None

worker_port_range : tuple(int, int)
     The interchange picks ports at random from the range which will be used by workers.
     This is overridden when the worker_ports option is set. Defauls: (54000, 55000)

heartbeat_threshold : int
     Number of seconds since the last heartbeat after which worker is considered lost.

logdir : str
     Parsl log directory paths. Logs and temp files go here. Default: '.'

logging_level : int
     Logging level as defined in the logging module. Default: logging.INFO (20)

poll_period : int
     The main thread polling period, in milliseconds. Default: 10ms

suppress_failure : Bool
     When set to True, the interchange will attempt to suppress failures. Default: False


tasks = []<EOL>for i in range(<NUM_LIT:0>, count):<EOL><INDENT>try:<EOL><INDENT>x = self.pending_task_queue.get(block=False)<EOL><DEDENT>except queue.Empty:<EOL><INDENT>break<EOL><DEDENT>else:<EOL><INDENT>tasks.append(x)<EOL><DEDENT><DEDENT>return tasks<EOL>

Obtains a batch of tasks from the internal pending_task_queue

        Parameters
        ----------
        count: int
            Count of tasks to get from the queue

        Returns
        -------
        List of upto count tasks. May return fewer than count down to an empty list
            eg. [{'task_id':<x>, 'buffer':<buf>} ... ]


logger.info("<STR_LIT>")<EOL>task_counter = <NUM_LIT:0><EOL>poller = zmq.Poller()<EOL>poller.register(self.task_incoming, zmq.POLLIN)<EOL>while not kill_event.is_set():<EOL><INDENT>try:<EOL><INDENT>msg = self.task_incoming.recv_pyobj()<EOL><DEDENT>except zmq.Again:<EOL><INDENT>logger.debug("<STR_LIT>".format(self.pending_task_queue.qsize()))<EOL>continue<EOL><DEDENT>if msg == '<STR_LIT>':<EOL><INDENT>kill_event.set()<EOL>break<EOL><DEDENT>else:<EOL><INDENT>self.pending_task_queue.put(msg)<EOL>task_counter += <NUM_LIT:1><EOL>logger.debug("<STR_LIT>".format(task_counter))<EOL><DEDENT><DEDENT>


logger.debug("<STR_LIT>")<EOL>while not kill_event.is_set():<EOL><INDENT>try:<EOL><INDENT>command_req = self.command_channel.recv_pyobj()<EOL>logger.debug("<STR_LIT>".format(command_req))<EOL>if command_req == "<STR_LIT>":<EOL><INDENT>outstanding = self.pending_task_queue.qsize()<EOL>for manager in self._ready_manager_queue:<EOL><INDENT>outstanding += len(self._ready_manager_queue[manager]['<STR_LIT>'])<EOL><DEDENT>reply = outstanding<EOL><DEDENT>elif command_req == "<STR_LIT>":<EOL><INDENT>num_workers = <NUM_LIT:0><EOL>for manager in self._ready_manager_queue:<EOL><INDENT>num_workers += self._ready_manager_queue[manager]['<STR_LIT>']<EOL><DEDENT>reply = num_workers<EOL><DEDENT>elif command_req == "<STR_LIT>":<EOL><INDENT>reply = []<EOL>for manager in self._ready_manager_queue:<EOL><INDENT>resp = {'<STR_LIT>': manager.decode('<STR_LIT:utf-8>'),<EOL>'<STR_LIT>': self._ready_manager_queue[manager]['<STR_LIT>'],<EOL>'<STR_LIT>': self._ready_manager_queue[manager]['<STR_LIT>'],<EOL>'<STR_LIT>': len(self._ready_manager_queue[manager]['<STR_LIT>']),<EOL>'<STR_LIT>': self._ready_manager_queue[manager]['<STR_LIT>']}<EOL>reply.append(resp)<EOL><DEDENT><DEDENT>elif command_req.startswith("<STR_LIT>"):<EOL><INDENT>cmd, s_manager = command_req.split('<STR_LIT:;>')<EOL>manager = s_manager.encode('<STR_LIT:utf-8>')<EOL>logger.info("<STR_LIT>".format(manager))<EOL>if manager in self._ready_manager_queue:<EOL><INDENT>self._ready_manager_queue[manager]['<STR_LIT>'] = False<EOL>reply = True<EOL><DEDENT>else:<EOL><INDENT>reply = False<EOL><DEDENT><DEDENT>elif command_req == "<STR_LIT>":<EOL><INDENT>logger.info("<STR_LIT>")<EOL>kill_event.set()<EOL>reply = True<EOL><DEDENT>else:<EOL><INDENT>reply = None<EOL><DEDENT>logger.debug("<STR_LIT>".format(reply))<EOL>self.command_channel.send_pyobj(reply)<EOL><DEDENT>except zmq.Again:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>continue<EOL><DEDENT><DEDENT>

Command server to run async command to the interchange


logger.info("<STR_LIT>")<EOL>if poll_period is None:<EOL><INDENT>poll_period = self.poll_period<EOL><DEDENT>start = time.time()<EOL>count = <NUM_LIT:0><EOL>self._kill_event = threading.Event()<EOL>self._task_puller_thread = threading.Thread(target=self.migrate_tasks_to_internal,<EOL>args=(self._kill_event,))<EOL>self._task_puller_thread.start()<EOL>self._command_thread = threading.Thread(target=self._command_server,<EOL>args=(self._kill_event,))<EOL>self._command_thread.start()<EOL>poller = zmq.Poller()<EOL>poller.register(self.task_outgoing, zmq.POLLIN)<EOL>poller.register(self.results_incoming, zmq.POLLIN)<EOL>interesting_managers = set()<EOL>while not self._kill_event.is_set():<EOL><INDENT>self.socks = dict(poller.poll(timeout=poll_period))<EOL>if self.task_outgoing in self.socks and self.socks[self.task_outgoing] == zmq.POLLIN:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>message = self.task_outgoing.recv_multipart()<EOL>manager = message[<NUM_LIT:0>]<EOL>if manager not in self._ready_manager_queue:<EOL><INDENT>reg_flag = False<EOL>try:<EOL><INDENT>msg = json.loads(message[<NUM_LIT:1>].decode('<STR_LIT:utf-8>'))<EOL>reg_flag = True<EOL><DEDENT>except Exception:<EOL><INDENT>logger.warning("<STR_LIT>".format(<EOL>manager))<EOL>logger.debug("<STR_LIT>".format(message[<NUM_LIT:0>]))<EOL><DEDENT>self._ready_manager_queue[manager] = {'<STR_LIT>': time.time(),<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': True,<EOL>'<STR_LIT>': []}<EOL>if reg_flag is True:<EOL><INDENT>interesting_managers.add(manager)<EOL>logger.info("<STR_LIT>".format(manager))<EOL>self._ready_manager_queue[manager].update(msg)<EOL>logger.info("<STR_LIT>".format(manager, msg))<EOL>if (msg['<STR_LIT>'].rsplit("<STR_LIT:.>", <NUM_LIT:1>)[<NUM_LIT:0>] != self.current_platform['<STR_LIT>'].rsplit("<STR_LIT:.>", <NUM_LIT:1>)[<NUM_LIT:0>] or<EOL>msg['<STR_LIT>'] != self.current_platform['<STR_LIT>']):<EOL><INDENT>logger.warn("<STR_LIT>".format(manager))<EOL>if self.suppress_failure is False:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>self._kill_event.set()<EOL>e = ManagerLost(manager)<EOL>result_package = {'<STR_LIT>': -<NUM_LIT:1>, '<STR_LIT>': serialize_object(e)}<EOL>pkl_package = pickle.dumps(result_package)<EOL>self.results_outgoing.send(pkl_package)<EOL>logger.warning("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>logger.info("<STR_LIT>".format(manager, msg['<STR_LIT>']))<EOL>logger.info("<STR_LIT>".format(manager,<EOL>msg['<STR_LIT>'].rsplit("<STR_LIT:.>", <NUM_LIT:1>)[<NUM_LIT:0>]))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if self.suppress_failure is False:<EOL><INDENT>self._kill_event.set()<EOL>e = BadRegistration(manager, critical=True)<EOL>result_package = {'<STR_LIT>': -<NUM_LIT:1>, '<STR_LIT>': serialize_object(e)}<EOL>pkl_package = pickle.dumps(result_package)<EOL>self.results_outgoing.send(pkl_package)<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>".format(<EOL>manager))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>tasks_requested = int.from_bytes(message[<NUM_LIT:1>], "<STR_LIT>")<EOL>self._ready_manager_queue[manager]['<STR_LIT>'] = time.time()<EOL>if tasks_requested == HEARTBEAT_CODE:<EOL><INDENT>logger.debug("<STR_LIT>".format(manager))<EOL>self.task_outgoing.send_multipart([manager, b'<STR_LIT>', PKL_HEARTBEAT_CODE])<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>".format(manager, tasks_requested))<EOL>self._ready_manager_queue[manager]['<STR_LIT>'] = tasks_requested<EOL>interesting_managers.add(manager)<EOL><DEDENT><DEDENT>logger.debug("<STR_LIT>")<EOL><DEDENT>logger.debug("<STR_LIT>".format(len(self._ready_manager_queue),<EOL>len(interesting_managers)))<EOL>if interesting_managers and not self.pending_task_queue.empty():<EOL><INDENT>shuffled_managers = list(interesting_managers)<EOL>random.shuffle(shuffled_managers)<EOL>while shuffled_managers and not self.pending_task_queue.empty():  <EOL><INDENT>manager = shuffled_managers.pop()<EOL>tasks_inflight = len(self._ready_manager_queue[manager]['<STR_LIT>'])<EOL>real_capacity = min(self._ready_manager_queue[manager]['<STR_LIT>'],<EOL>self._ready_manager_queue[manager]['<STR_LIT>'] - tasks_inflight)<EOL>if (real_capacity and self._ready_manager_queue[manager]['<STR_LIT>']):<EOL><INDENT>tasks = self.get_tasks(real_capacity)<EOL>if tasks:<EOL><INDENT>self.task_outgoing.send_multipart([manager, b'<STR_LIT>', pickle.dumps(tasks)])<EOL>task_count = len(tasks)<EOL>count += task_count<EOL>tids = [t['<STR_LIT>'] for t in tasks]<EOL>self._ready_manager_queue[manager]['<STR_LIT>'] -= task_count<EOL>self._ready_manager_queue[manager]['<STR_LIT>'].extend(tids)<EOL>logger.debug("<STR_LIT>".format(tids, manager))<EOL>if self._ready_manager_queue[manager]['<STR_LIT>'] > <NUM_LIT:0>:<EOL><INDENT>logger.debug("<STR_LIT>".format(manager, self._ready_manager_queue[manager]['<STR_LIT>']))<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>".format(manager))<EOL>interesting_managers.remove(manager)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>interesting_managers.remove(manager)<EOL><DEDENT><DEDENT>logger.debug("<STR_LIT>".format(len(interesting_managers)))<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL><DEDENT>if self.results_incoming in self.socks and self.socks[self.results_incoming] == zmq.POLLIN:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>manager, *b_messages = self.results_incoming.recv_multipart()<EOL>if manager not in self._ready_manager_queue:<EOL><INDENT>logger.warning("<STR_LIT>".format(manager))<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>".format(len(b_messages)))<EOL>for b_message in b_messages:<EOL><INDENT>r = pickle.loads(b_message)<EOL>self._ready_manager_queue[manager]['<STR_LIT>'].remove(r['<STR_LIT>'])<EOL><DEDENT>self.results_outgoing.send_multipart(b_messages)<EOL>logger.debug("<STR_LIT>".format(self._ready_manager_queue[manager]['<STR_LIT>']))<EOL><DEDENT>logger.debug("<STR_LIT>")<EOL><DEDENT>logger.debug("<STR_LIT>")<EOL>bad_managers = [manager for manager in self._ready_manager_queue if<EOL>time.time() - self._ready_manager_queue[manager]['<STR_LIT>'] > self.heartbeat_threshold]<EOL>for manager in bad_managers:<EOL><INDENT>logger.debug("<STR_LIT>".format(self._ready_manager_queue[manager]['<STR_LIT>'], time.time()))<EOL>logger.warning("<STR_LIT>".format(manager))<EOL>for tid in self._ready_manager_queue[manager]['<STR_LIT>']:<EOL><INDENT>try:<EOL><INDENT>raise ManagerLost(manager)<EOL><DEDENT>except Exception:<EOL><INDENT>result_package = {'<STR_LIT>': tid, '<STR_LIT>': serialize_object(RemoteExceptionWrapper(*sys.exc_info()))}<EOL>pkl_package = pickle.dumps(result_package)<EOL>self.results_outgoing.send(pkl_package)<EOL>logger.warning("<STR_LIT>")<EOL><DEDENT><DEDENT>self._ready_manager_queue.pop(manager, '<STR_LIT:None>')<EOL><DEDENT>logger.debug("<STR_LIT>")<EOL>logger.debug("<STR_LIT>")<EOL><DEDENT>delta = time.time() - start<EOL>logger.info("<STR_LIT>".format(count, delta))<EOL>logger.warning("<STR_LIT>")<EOL>

Start the NeedNameQeueu

        Parameters:
        ----------

        TODO: Move task receiving to a thread


debug_opts = "<STR_LIT>" if self.worker_debug else "<STR_LIT>"<EOL>max_workers = "<STR_LIT>" if self.max_workers == float('<STR_LIT>') else "<STR_LIT>".format(self.max_workers)<EOL>worker_logdir = "<STR_LIT>".format(self.run_dir, self.label)<EOL>if self.worker_logdir_root is not None:<EOL><INDENT>worker_logdir = "<STR_LIT>".format(self.worker_logdir_root, self.label)<EOL><DEDENT>l_cmd = self.launch_cmd.format(debug=debug_opts,<EOL>prefetch_capacity=self.prefetch_capacity,<EOL>task_url=self.worker_task_url,<EOL>result_url=self.worker_result_url,<EOL>cores_per_worker=self.cores_per_worker,<EOL>max_workers=max_workers,<EOL>nodes_per_block=self.provider.nodes_per_block,<EOL>heartbeat_period=self.heartbeat_period,<EOL>heartbeat_threshold=self.heartbeat_threshold,<EOL>poll_period=self.poll_period,<EOL>logdir=worker_logdir)<EOL>self.launch_cmd = l_cmd<EOL>logger.debug("<STR_LIT>".format(self.launch_cmd))<EOL>self._scaling_enabled = self.provider.scaling_enabled<EOL>logger.debug("<STR_LIT>", self.provider)<EOL>if hasattr(self.provider, '<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>self.scale_out(blocks=self.provider.init_blocks)<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.error("<STR_LIT>".format(e))<EOL>raise e<EOL><DEDENT><DEDENT>

Compose the launch command and call the scale_out

        This should be implemented in the child classes to take care of
        executor specific oddities.


self.outgoing_q = zmq_pipes.TasksOutgoing("<STR_LIT:127.0.0.1>", self.interchange_port_range)<EOL>self.incoming_q = zmq_pipes.ResultsIncoming("<STR_LIT:127.0.0.1>", self.interchange_port_range)<EOL>self.command_client = zmq_pipes.CommandClient("<STR_LIT:127.0.0.1>", self.interchange_port_range)<EOL>self.is_alive = True<EOL>self._executor_bad_state = threading.Event()<EOL>self._executor_exception = None<EOL>self._queue_management_thread = None<EOL>self._start_queue_management_thread()<EOL>self._start_local_queue_process()<EOL>logger.debug("<STR_LIT>".format(self._queue_management_thread))<EOL>if self.provider:<EOL><INDENT>self.initialize_scaling()<EOL><DEDENT>else:<EOL><INDENT>self._scaling_enabled = False<EOL>logger.debug("<STR_LIT>")<EOL><DEDENT>

Create the Interchange process and connect to it.


logger.debug("<STR_LIT>")<EOL>while not self._executor_bad_state.is_set():<EOL><INDENT>try:<EOL><INDENT>msgs = self.incoming_q.get(timeout=<NUM_LIT:1>)<EOL><DEDENT>except queue.Empty:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>pass<EOL><DEDENT>except IOError as e:<EOL><INDENT>logger.exception("<STR_LIT>".format(e.errno, e))<EOL>return<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.exception("<STR_LIT>".format(e))<EOL>return<EOL><DEDENT>else:<EOL><INDENT>if msgs is None:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return<EOL><DEDENT>else:<EOL><INDENT>for serialized_msg in msgs:<EOL><INDENT>try:<EOL><INDENT>msg = pickle.loads(serialized_msg)<EOL>tid = msg['<STR_LIT>']<EOL><DEDENT>except pickle.UnpicklingError:<EOL><INDENT>raise BadMessage("<STR_LIT>")<EOL><DEDENT>except Exception:<EOL><INDENT>raise BadMessage("<STR_LIT>")<EOL><DEDENT>if tid == -<NUM_LIT:1> and '<STR_LIT>' in msg:<EOL><INDENT>logger.warning("<STR_LIT>")<EOL>self._executor_exception, _ = deserialize_object(msg['<STR_LIT>'])<EOL>logger.exception("<STR_LIT>".format(self._executor_exception))<EOL>self._executor_bad_state.set()<EOL>for task in self.tasks:<EOL><INDENT>self.tasks[task].set_exception(self._executor_exception)<EOL><DEDENT>break<EOL><DEDENT>task_fut = self.tasks[tid]<EOL>if '<STR_LIT:result>' in msg:<EOL><INDENT>result, _ = deserialize_object(msg['<STR_LIT:result>'])<EOL>task_fut.set_result(result)<EOL><DEDENT>elif '<STR_LIT>' in msg:<EOL><INDENT>try:<EOL><INDENT>s, _ = deserialize_object(msg['<STR_LIT>'])<EOL>try:<EOL><INDENT>s.reraise()<EOL><DEDENT>except Exception as e:<EOL><INDENT>task_fut.set_exception(e)<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>task_fut.set_exception(<EOL>DeserializationError("<STR_LIT>".format(e)))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise BadMessage("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if not self.is_alive:<EOL><INDENT>break<EOL><DEDENT><DEDENT>logger.info("<STR_LIT>")<EOL>

Listen to the queue for task status messages and handle them.

        Depending on the message, tasks will be updated with results, exceptions,
        or updates. It expects the following messages:

        .. code:: python

            {
               "task_id" : <task_id>
               "result"  : serialized result object, if task succeeded
               ... more tags could be added later
            }

            {
               "task_id" : <task_id>
               "exception" : serialized exception object, on failure
            }

        We do not support these yet, but they could be added easily.

        .. code:: python

            {
               "task_id" : <task_id>
               "cpu_stat" : <>
               "mem_stat" : <>
               "io_stat"  : <>
               "started"  : tstamp
            }

        The `None` message is a die request.


q.put(None)<EOL>


comm_q = Queue(maxsize=<NUM_LIT:10>)<EOL>self.queue_proc = Process(target=interchange.starter,<EOL>args=(comm_q,),<EOL>kwargs={"<STR_LIT>": (self.outgoing_q.port,<EOL>self.incoming_q.port,<EOL>self.command_client.port),<EOL>"<STR_LIT>": self.worker_ports,<EOL>"<STR_LIT>": self.worker_port_range,<EOL>"<STR_LIT>": "<STR_LIT>".format(self.run_dir, self.label),<EOL>"<STR_LIT>": self.suppress_failure,<EOL>"<STR_LIT>": self.heartbeat_threshold,<EOL>"<STR_LIT>": self.poll_period,<EOL>"<STR_LIT>": logging.DEBUG if self.worker_debug else logging.INFO<EOL>},<EOL>)<EOL>self.queue_proc.start()<EOL>try:<EOL><INDENT>(worker_task_port, worker_result_port) = comm_q.get(block=True, timeout=<NUM_LIT>)<EOL><DEDENT>except queue.Empty:<EOL><INDENT>logger.error("<STR_LIT>")<EOL>raise Exception("<STR_LIT>")<EOL><DEDENT>self.worker_task_url = "<STR_LIT>".format(self.address, worker_task_port)<EOL>self.worker_result_url = "<STR_LIT>".format(self.address, worker_result_port)<EOL>

Starts the interchange process locally

        Starts the interchange process locally and uses an internal command queue to
        get the worker task and result ports that the interchange has bound to.


if self._queue_management_thread is None:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>self._queue_management_thread = threading.Thread(target=self._queue_management_worker)<EOL>self._queue_management_thread.daemon = True<EOL>self._queue_management_thread.start()<EOL>logger.debug("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL><DEDENT>

Method to start the management thread as a daemon.

        Checks if a thread already exists, then starts it.
        Could be used later as a restart if the management thread dies.


c = self.command_client.run("<STR_LIT>".format(worker_id))<EOL>logger.debug("<STR_LIT>".format(worker_id))<EOL>return c<EOL>

Puts a worker on hold, preventing scheduling of additional tasks to it.

        This is called "hold" mostly because this only stops scheduling of tasks,
        and does not actually kill the worker.

        Parameters
        ----------

        worker_id : str
            Worker id to be put on hold


managers = self.connected_managers<EOL>for manager in managers:<EOL><INDENT>if manager['<STR_LIT>'] == block_id:<EOL><INDENT>logger.debug("<STR_LIT>".format(manager['<STR_LIT>']))<EOL>self.hold_worker(manager['<STR_LIT>'])<EOL><DEDENT><DEDENT>

Sends hold command to all managers which are in a specific block

        Parameters
        ----------
        block_id : str
             Block identifier of the block to be put on hold


def submit(self, func, *args, **kwargs):

if self._executor_bad_state.is_set():<EOL><INDENT>raise self._executor_exception<EOL><DEDENT>self._task_counter += <NUM_LIT:1><EOL>task_id = self._task_counter<EOL>logger.debug("<STR_LIT>".format(func, args))<EOL>self.tasks[task_id] = Future()<EOL>fn_buf = pack_apply_message(func, args, kwargs,<EOL>buffer_threshold=<NUM_LIT> * <NUM_LIT>,<EOL>item_threshold=<NUM_LIT>)<EOL>msg = {"<STR_LIT>": task_id,<EOL>"<STR_LIT>": fn_buf}<EOL>self.outgoing_q.put(msg)<EOL>return self.tasks[task_id]<EOL>

Submits work to the the outgoing_q.

        The outgoing_q is an external process listens on this
        queue for new work. This method behaves like a
        submit call as described here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_

        Args:
            - func (callable) : Callable function
            - *args (list) : List of arbitrary positional arguments.

        Kwargs:
            - **kwargs (dict) : A dictionary of arbitrary keyword args for func.

        Returns:
              Future


r = []<EOL>for i in range(blocks):<EOL><INDENT>if self.provider:<EOL><INDENT>external_block_id = str(len(self.blocks))<EOL>launch_cmd = self.launch_cmd.format(block_id=external_block_id)<EOL>internal_block = self.provider.submit(launch_cmd, <NUM_LIT:1>, <NUM_LIT:1>)<EOL>logger.debug("<STR_LIT>".format(external_block_id, internal_block))<EOL>if not internal_block:<EOL><INDENT>raise(ScalingFailed(self.provider.label,<EOL>"<STR_LIT>"))<EOL><DEDENT>r.extend([external_block_id])<EOL>self.blocks[external_block_id] = internal_block<EOL><DEDENT>else:<EOL><INDENT>logger.error("<STR_LIT>")<EOL>r = None<EOL><DEDENT><DEDENT>return r<EOL>

Scales out the number of blocks by "blocks"

        Raises:
             NotImplementedError


def scale_in(self, blocks=None, block_ids=[]):

if block_ids:<EOL><INDENT>block_ids_to_kill = block_ids<EOL><DEDENT>else:<EOL><INDENT>block_ids_to_kill = list(self.blocks.keys())[:blocks]<EOL><DEDENT>for block_id in block_ids_to_kill:<EOL><INDENT>self._hold_block(block_id)<EOL><DEDENT>to_kill = [self.blocks.pop(bid) for bid in block_ids_to_kill]<EOL>if self.provider:<EOL><INDENT>r = self.provider.cancel(to_kill)<EOL><DEDENT>return r<EOL>

Scale in the number of active blocks by specified amount.

        The scale in method here is very rude. It doesn't give the workers
        the opportunity to finish current tasks or cleanup. This is tracked
        in issue #530

        Parameters
        ----------

        blocks : int
             Number of blocks to terminate and scale_in by

        block_ids : list
             List of specific block ids to terminate. Optional

        Raises:
             NotImplementedError


status = []<EOL>if self.provider:<EOL><INDENT>status = self.provider.status(self.blocks.values())<EOL><DEDENT>return status<EOL>


def shutdown(self, hub=True, targets='<STR_LIT:all>', block=False):

logger.info("<STR_LIT>")<EOL>self.queue_proc.terminate()<EOL>logger.info("<STR_LIT>")<EOL>return True<EOL>

Shutdown the executor, including all workers and controllers.

        This is not implemented.

        Kwargs:
            - hub (Bool): Whether the hub should be shutdown, Default:True,
            - targets (list of ints| 'all'): List of block id's to kill, Default:'all'
            - block (Bool): To block for confirmations or not

        Raises:
             NotImplementedError


user_ns = locals()<EOL>user_ns.update({'<STR_LIT>': __builtins__})<EOL>f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False)<EOL>fname = getattr(f, '<STR_LIT>', '<STR_LIT:f>')<EOL>prefix = "<STR_LIT>"<EOL>fname = prefix + "<STR_LIT:f>"<EOL>argname = prefix + "<STR_LIT:args>"<EOL>kwargname = prefix + "<STR_LIT>"<EOL>resultname = prefix + "<STR_LIT:result>"<EOL>user_ns.update({fname: f,<EOL>argname: args,<EOL>kwargname: kwargs,<EOL>resultname: resultname})<EOL>code = "<STR_LIT>".format(resultname, fname,<EOL>argname, kwargname)<EOL>try:<EOL><INDENT>logger.debug("<STR_LIT>".format(code))<EOL>exec(code, user_ns, user_ns)<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.warning("<STR_LIT>".format(e))<EOL>raise e<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>".format(user_ns.get(resultname)))<EOL>return user_ns.get(resultname)<EOL><DEDENT>

Deserialize the buffer and execute the task.

    Returns the serialized result or exception.


if format_string is None:<EOL><INDENT>format_string = "<STR_LIT>".format(rank)<EOL><DEDENT>global logger<EOL>logger = logging.getLogger(name)<EOL>logger.setLevel(logging.DEBUG)<EOL>handler = logging.FileHandler(filename)<EOL>handler.setLevel(level)<EOL>formatter = logging.Formatter(format_string, datefmt='<STR_LIT>')<EOL>handler.setFormatter(formatter)<EOL>logger.addHandler(handler)<EOL>

Add a stream log handler.

    Args:
        - filename (string): Name of the file to write logs to
        - name (string): Logger name
        - level (logging.LEVEL): Set the logging level.
        - format_string (string): Set the format string

    Returns:
       -  None


if format_string is None:<EOL><INDENT>format_string = "<STR_LIT>"<EOL><DEDENT>global logger<EOL>logger = logging.getLogger(name)<EOL>logger.setLevel(logging.DEBUG)<EOL>handler = logging.StreamHandler()<EOL>handler.setLevel(level)<EOL>formatter = logging.Formatter(format_string, datefmt='<STR_LIT>')<EOL>handler.setFormatter(formatter)<EOL>logger.addHandler(handler)<EOL>

Add a stream log handler.

    Args:
         - name (string) : Set the logger name.
         - level (logging.LEVEL) : Set to logging.DEBUG by default.
         - format_string (sting) : Set to None by default.

    Returns:
         - None


self.uid = uid<EOL>self.context = zmq.Context()<EOL>self.task_incoming = self.context.socket(zmq.DEALER)<EOL>self.task_incoming.setsockopt(zmq.IDENTITY, uid.encode('<STR_LIT:utf-8>'))<EOL>self.task_incoming.setsockopt(zmq.LINGER, <NUM_LIT:0>)<EOL>self.task_incoming.connect(task_q_url)<EOL>self.result_outgoing = self.context.socket(zmq.DEALER)<EOL>self.result_outgoing.setsockopt(zmq.IDENTITY, uid.encode('<STR_LIT:utf-8>'))<EOL>self.result_outgoing.setsockopt(zmq.LINGER, <NUM_LIT:0>)<EOL>self.result_outgoing.connect(result_q_url)<EOL>logger.info("<STR_LIT>")<EOL>self.max_queue_size = max_queue_size + comm.size<EOL>self.pending_task_queue = queue.Queue()<EOL>self.pending_result_queue = queue.Queue()<EOL>self.ready_worker_queue = queue.Queue()<EOL>self.tasks_per_round = <NUM_LIT:1><EOL>self.heartbeat_period = heartbeat_period<EOL>self.heartbeat_threshold = heartbeat_threshold<EOL>self.comm = comm<EOL>self.rank = rank<EOL>

Parameters
----------
worker_url : str
     Worker url on which workers will attempt to connect back

heartbeat_threshold : int
     Number of seconds since the last message from the interchange after which the worker
     assumes that the interchange is lost and the manager shuts down. Default:120

heartbeat_period : int
     Number of seconds after which a heartbeat message is sent to the interchange


msg = {'<STR_LIT>': PARSL_VERSION,<EOL>'<STR_LIT>': "<STR_LIT>".format(sys.version_info.major,<EOL>sys.version_info.minor,<EOL>sys.version_info.micro),<EOL>'<STR_LIT>': platform.system(),<EOL>'<STR_LIT>': platform.node(),<EOL>'<STR_LIT>': os.getcwd(),<EOL>}<EOL>b_msg = json.dumps(msg).encode('<STR_LIT:utf-8>')<EOL>return b_msg<EOL>

Creates a registration message to identify the worker to the interchange


heartbeat = (HEARTBEAT_CODE).to_bytes(<NUM_LIT:4>, "<STR_LIT>")<EOL>r = self.task_incoming.send(heartbeat)<EOL>logger.debug("<STR_LIT>".format(r))<EOL>

Send heartbeat to the incoming task queue


info = MPI.Status()<EOL>result = self.comm.recv(source=MPI.ANY_SOURCE, tag=RESULT_TAG, status=info)<EOL>logger.debug("<STR_LIT>".format(result))<EOL>return result<EOL>

Receives a results from the MPI worker pool and send it out via 0mq

        Returns:
        --------
            result: task result from the workers


info = MPI.Status()<EOL>comm.recv(source=MPI.ANY_SOURCE, tag=TASK_REQUEST_TAG, status=info)<EOL>worker_rank = info.Get_source()<EOL>logger.info("<STR_LIT>".format(worker_rank))<EOL>return worker_rank<EOL>

Receives 1 task request from MPI comm

        Returns:
        --------
            worker_rank: worker_rank id


logger.info("<STR_LIT>")<EOL>poller = zmq.Poller()<EOL>poller.register(self.task_incoming, zmq.POLLIN)<EOL>msg = self.create_reg_message()<EOL>logger.debug("<STR_LIT>".format(msg))<EOL>self.task_incoming.send(msg)<EOL>last_beat = time.time()<EOL>last_interchange_contact = time.time()<EOL>task_recv_counter = <NUM_LIT:0><EOL>poll_timer = <NUM_LIT:1><EOL>while not kill_event.is_set():<EOL><INDENT>time.sleep(LOOP_SLOWDOWN)<EOL>ready_worker_count = self.ready_worker_queue.qsize()<EOL>pending_task_count = self.pending_task_queue.qsize()<EOL>logger.debug("<STR_LIT>".format(ready_worker_count,<EOL>pending_task_count))<EOL>if time.time() > last_beat + self.heartbeat_period:<EOL><INDENT>self.heartbeat()<EOL>last_beat = time.time()<EOL><DEDENT>if pending_task_count < self.max_queue_size and ready_worker_count > <NUM_LIT:0>:<EOL><INDENT>logger.debug("<STR_LIT>".format(ready_worker_count))<EOL>msg = ((ready_worker_count).to_bytes(<NUM_LIT:4>, "<STR_LIT>"))<EOL>self.task_incoming.send(msg)<EOL><DEDENT>socks = dict(poller.poll(timeout=poll_timer))<EOL>if self.task_incoming in socks and socks[self.task_incoming] == zmq.POLLIN:<EOL><INDENT>_, pkl_msg = self.task_incoming.recv_multipart()<EOL>tasks = pickle.loads(pkl_msg)<EOL>last_interchange_contact = time.time()<EOL>if tasks == '<STR_LIT>':<EOL><INDENT>logger.critical("<STR_LIT>")<EOL>kill_event.set()<EOL>break<EOL><DEDENT>elif tasks == HEARTBEAT_CODE:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>poll_timer = <NUM_LIT:1><EOL>task_recv_counter += len(tasks)<EOL>logger.debug("<STR_LIT>".format([t['<STR_LIT>'] for t in tasks],<EOL>task_recv_counter))<EOL>for task in tasks:<EOL><INDENT>self.pending_task_queue.put(task)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>poll_timer = min(self.heartbeat_period * <NUM_LIT:1000>, poll_timer * <NUM_LIT:2>)<EOL>if time.time() > last_interchange_contact + self.heartbeat_threshold:<EOL><INDENT>logger.critical("<STR_LIT>")<EOL>kill_event.set()<EOL>logger.critical("<STR_LIT>")<EOL>break<EOL><DEDENT><DEDENT><DEDENT>

Pulls tasks from the incoming tasks 0mq pipe onto the internal
        pending task queue

        Parameters:
        -----------
        kill_event : threading.Event
              Event to let the thread know when it is time to die.


<EOL>timeout = <NUM_LIT:0.1><EOL>logger.debug("<STR_LIT>")<EOL>while not kill_event.is_set():<EOL><INDENT>time.sleep(LOOP_SLOWDOWN)<EOL>try:<EOL><INDENT>items = []<EOL>while not self.pending_result_queue.empty():<EOL><INDENT>r = self.pending_result_queue.get(block=True)<EOL>items.append(r)<EOL><DEDENT>if items:<EOL><INDENT>self.result_outgoing.send_multipart(items)<EOL><DEDENT><DEDENT>except queue.Empty:<EOL><INDENT>logger.debug("<STR_LIT>".format(timeout))<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.exception("<STR_LIT>".format(e))<EOL><DEDENT><DEDENT>logger.critical("<STR_LIT>")<EOL>

Listens on the pending_result_queue and sends out results via 0mq

        Parameters:
        -----------
        kill_event : threading.Event
              Event to let the thread know when it is time to die.


self.comm.Barrier()<EOL>logger.debug("<STR_LIT>")<EOL>self._kill_event = threading.Event()<EOL>self._task_puller_thread = threading.Thread(target=self.pull_tasks,<EOL>args=(self._kill_event,))<EOL>self._result_pusher_thread = threading.Thread(target=self.push_results,<EOL>args=(self._kill_event,))<EOL>self._task_puller_thread.start()<EOL>self._result_pusher_thread.start()<EOL>start = None<EOL>result_counter = <NUM_LIT:0><EOL>task_recv_counter = <NUM_LIT:0><EOL>task_sent_counter = <NUM_LIT:0><EOL>logger.info("<STR_LIT>")<EOL>while not self._kill_event.is_set():<EOL><INDENT>time.sleep(LOOP_SLOWDOWN)<EOL>timer = time.time() + <NUM_LIT><EOL>counter = min(<NUM_LIT:10>, comm.size)<EOL>while time.time() < timer:<EOL><INDENT>info = MPI.Status()<EOL>if counter > <NUM_LIT:10>:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>break<EOL><DEDENT>if not self.comm.Iprobe(status=info):<EOL><INDENT>logger.debug("<STR_LIT>".format(counter))<EOL>break<EOL><DEDENT>else:<EOL><INDENT>tag = info.Get_tag()<EOL>logger.info("<STR_LIT>".format(tag))<EOL>counter += <NUM_LIT:1><EOL>if tag == RESULT_TAG:<EOL><INDENT>result = self.recv_result_from_workers()<EOL>self.pending_result_queue.put(result)<EOL>result_counter += <NUM_LIT:1><EOL><DEDENT>elif tag == TASK_REQUEST_TAG:<EOL><INDENT>worker_rank = self.recv_task_request_from_workers()<EOL>self.ready_worker_queue.put(worker_rank)<EOL><DEDENT>else:<EOL><INDENT>logger.error("<STR_LIT>".format(tag))<EOL><DEDENT><DEDENT><DEDENT>available_worker_cnt = self.ready_worker_queue.qsize()<EOL>available_task_cnt = self.pending_task_queue.qsize()<EOL>logger.debug("<STR_LIT>".format(available_worker_cnt,<EOL>available_task_cnt))<EOL>this_round = min(available_worker_cnt, available_task_cnt)<EOL>for i in range(this_round):<EOL><INDENT>worker_rank = self.ready_worker_queue.get()<EOL>task = self.pending_task_queue.get()<EOL>comm.send(task, dest=worker_rank, tag=worker_rank)<EOL>task_sent_counter += <NUM_LIT:1><EOL>logger.debug("<STR_LIT>".format(worker_rank, task['<STR_LIT>']))<EOL><DEDENT>if not start:<EOL><INDENT>start = time.time()<EOL><DEDENT>logger.debug("<STR_LIT>".format(<EOL>task_recv_counter, task_sent_counter, result_counter))<EOL><DEDENT>self._task_puller_thread.join()<EOL>self._result_pusher_thread.join()<EOL>self.task_incoming.close()<EOL>self.result_outgoing.close()<EOL>self.context.term()<EOL>delta = time.time() - start<EOL>logger.info("<STR_LIT>".format(delta))<EOL>

Start the Manager process.

        The worker loops on this:

        1. If the last message sent was older than heartbeat period we send a heartbeat
        2.

        TODO: Move task receiving to a thread


@abstractmethod<EOL><INDENT>def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):<DEDENT>

pass<EOL>

Wraps the command with the Launcher calls.
        *MUST* be implemented by the concrete child classes


def __call__(self, command, tasks_per_node, nodes_per_block, walltime=None):

return command<EOL>

Args:
- command (string): The command string to be launched
- task_block (string) : bash evaluated string.

KWargs:
- walltime (int) : This is not used by this launcher.


task_blocks = tasks_per_node * nodes_per_block<EOL>x =


self.overrides = overrides<EOL>

Parameters
----------

overrides: str
     This string will be passed to the srun launcher. Default: ''


self.overrides = overrides<EOL>

Parameters
----------

overrides: str
     This string will be passed to the launcher. Default: ''


self.overrides = overrides<EOL>

Parameters
----------

overrides: str
     This string will be passed to the aprun launcher. Default: ''


tasks_per_block = tasks_per_node * nodes_per_block<EOL>x =

Args:
- command (string): The command string to be launched
- tasks_per_node (int) : Workers to launch per node
- nodes_per_block (int) : Number of nodes in a block

KWargs:
- walltime (int) : This is not used by this launcher.


if(not os.path.isdir(rundir)):<EOL><INDENT>return []<EOL><DEDENT>dirs = sorted(os.listdir(rundir))<EOL>checkpoints = []<EOL>for runid in dirs:<EOL><INDENT>checkpoint = os.path.abspath('<STR_LIT>'.format(rundir, runid))<EOL>if os.path.isdir(checkpoint):<EOL><INDENT>checkpoints.append(checkpoint)<EOL><DEDENT><DEDENT>return checkpoints<EOL>

Finds the checkpoints from all last runs.

    Note that checkpoints are incremental, and this helper will not find
    previous checkpoints from earlier than the most recent run. It probably
    should be made to do so.

    Kwargs:
       - rundir(str) : Path to the runinfo directory

    Returns:
       - a list suitable for the checkpointFiles parameter of DataFlowKernel
         constructor


if not os.path.isdir(rundir):<EOL><INDENT>return []<EOL><DEDENT>dirs = sorted(os.listdir(rundir))<EOL>if len(dirs) == <NUM_LIT:0>:<EOL><INDENT>return []<EOL><DEDENT>last_runid = dirs[-<NUM_LIT:1>]<EOL>last_checkpoint = os.path.abspath('<STR_LIT>'.format(rundir, last_runid))<EOL>if(not(os.path.isdir(last_checkpoint))):<EOL><INDENT>return []<EOL><DEDENT>return [last_checkpoint]<EOL>

Find the checkpoint from the last run, if one exists.

    Note that checkpoints are incremental, and this helper will not find
    previous checkpoints from earlier than the most recent run. It probably
    should be made to do so.

    Kwargs:
       - rundir(str) : Path to the runinfo directory

    Returns:
     - a list suitable for checkpointFiles parameter of DataFlowKernel
       constructor, with 0 or 1 elements


hours, mins, seconds = time_string.split('<STR_LIT::>')<EOL>total_mins = int(hours) * <NUM_LIT> + int(mins)<EOL>if total_mins < <NUM_LIT:1>:<EOL><INDENT>logger.warning("<STR_LIT>".format(time_string, total_mins))<EOL><DEDENT>return total_mins<EOL>

wtime_to_minutes

    Convert standard wallclock time string to minutes.

    Args:
        - Time_string in HH:MM:SS format

    Returns:
        (int) minutes


p_this = Path(__file__)<EOL>to_remove_list = [<EOL>p_this.change(new_basename="<STR_LIT>"),<EOL>]<EOL>for p in p_this.parent.select_by_ext("<STR_LIT>"):<EOL><INDENT>to_remove_list.append(p)<EOL><DEDENT>for p in to_remove_list:<EOL><INDENT>if p.is_file():<EOL><INDENT>p.remove()<EOL><DEDENT>elif p.is_dir():<EOL><INDENT>shutil.rmtree(p.abspath)<EOL><DEDENT><DEDENT>


p_this = Path(__file__)<EOL>to_remove_list = list(p_this.parent.select_by_ext("<STR_LIT>"))<EOL>for p in to_remove_list:<EOL><INDENT>if p.exists():<EOL><INDENT>p.remove()<EOL><DEDENT><DEDENT>


p = Path(__file__).change(new_basename="<STR_LIT>")<EOL>try:<EOL><INDENT>shutil.copytree(p.abspath, p.change(new_basename="<STR_LIT>").abspath)<EOL><DEDENT>except Exception as e:<EOL><INDENT>pass<EOL><DEDENT>p = Path(__file__).change(new_basename="<STR_LIT>")<EOL>with open(p.abspath, "<STR_LIT:wb>") as f:<EOL><INDENT>f.write("<STR_LIT>".encode("<STR_LIT:utf-8>"))<EOL><DEDENT>p = Path(__file__).change(new_basename="<STR_LIT>")<EOL>with open(p.abspath, "<STR_LIT:wb>") as f:<EOL><INDENT>f.write("<STR_LIT>".encode("<STR_LIT:utf-8>"))<EOL><DEDENT>

Create temp file and dir for test.

- create a new folder ``/wow``
- create two file `/`wow/file_to_move.txt``, ``wow/file_to_copy.txt``


p_this = Path(__file__)<EOL>to_remove_list = [<EOL>p_this.change(new_basename="<STR_LIT>"),<EOL>p_this.change(new_basename="<STR_LIT>"),<EOL>p_this.change(new_basename="<STR_LIT>"),<EOL>p_this.change(new_basename="<STR_LIT>"),<EOL>p_this.change(new_basename="<STR_LIT>"),<EOL>p_this.change(new_basename="<STR_LIT>"),<EOL>p_this.change(new_basename="<STR_LIT>"),<EOL>]<EOL>for p in to_remove_list:<EOL><INDENT>if p.is_file():<EOL><INDENT>p.remove()<EOL><DEDENT>elif p.is_dir():<EOL><INDENT>shutil.rmtree(p.abspath)<EOL><DEDENT><DEDENT>


def get_text_fingerprint(text, hash_meth, encoding="<STR_LIT:utf-8>"):

m = hash_meth()<EOL>m.update(text.encode(encoding))<EOL>return m.hexdigest()<EOL>

Use default hash method to return hash value of a piece of string
default setting use 'utf-8' encoding.


def md5file(abspath, nbytes=<NUM_LIT:0>, chunk_size=DEFAULT_CHUNK_SIZE):

return get_file_fingerprint(abspath, hashlib.md5, nbytes=nbytes, chunk_size=chunk_size)<EOL>

Return md5 hash value of a piece of a file

Estimate processing time on:

:param abspath: the absolute path to the file
:param nbytes: only has first N bytes of the file. if 0 or None,
  hash all file

CPU = i7-4600U 2.10GHz - 2.70GHz, RAM = 8.00 GB
1 second can process 0.25GB data

- 0.59G - 2.43 sec
- 1.3G - 5.68 sec
- 1.9G - 7.72 sec
- 2.5G - 10.32 sec
- 3.9G - 16.0 sec


def sha256file(abspath, nbytes=<NUM_LIT:0>, chunk_size=DEFAULT_CHUNK_SIZE):

return get_file_fingerprint(abspath, hashlib.sha256, nbytes=nbytes, chunk_size=chunk_size)<EOL>

Return sha256 hash value of a piece of a file

Estimate processing time on:

:param abspath: the absolute path to the file
:param nbytes: only has first N bytes of the file. if 0 or None,
  hash all file


def sha512file(abspath, nbytes=<NUM_LIT:0>, chunk_size=DEFAULT_CHUNK_SIZE):

return get_file_fingerprint(abspath, hashlib.sha512, nbytes=nbytes, chunk_size=chunk_size)<EOL>

Return sha512 hash value of a piece of a file

Estimate processing time on:

:param abspath: the absolute path to the file
:param nbytes: only has first N bytes of the file. if 0 or None,
  hash all file


return md5file(abspath=self.abspath, nbytes=nbytes)<EOL>

Return md5 check sum of first n bytes of this file.


return md5file(self.abspath)<EOL>


return sha256file(abspath=self.abspath, nbytes=nbytes)<EOL>

Return sha256 check sum of first n bytes of this file.


return sha256file(self.abspath)<EOL>


return sha512file(abspath=self.abspath, nbytes=nbytes)<EOL>

Return sha512 check sum of first n bytes of this file.


return sha512file(self.abspath)<EOL>


self.assert_exists()<EOL>if dst is None:<EOL><INDENT>dst = self._auto_zip_archive_dst()<EOL><DEDENT>else:<EOL><INDENT>dst = self.change(new_abspath=dst)<EOL><DEDENT>if not dst.basename.lower().endswith("<STR_LIT>"):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if dst.exists():<EOL><INDENT>if not overwrite:<EOL><INDENT>raise IOError("<STR_LIT>" % dst)<EOL><DEDENT><DEDENT>if compress:<EOL><INDENT>compression = ZIP_DEFLATED<EOL><DEDENT>else:<EOL><INDENT>compression = ZIP_STORED<EOL><DEDENT>if not dst.parent.exists():<EOL><INDENT>if makedirs:<EOL><INDENT>os.makedirs(dst.parent.abspath)<EOL><DEDENT><DEDENT>if verbose:<EOL><INDENT>msg = "<STR_LIT>" % self<EOL>print(msg)<EOL><DEDENT>current_dir = os.getcwd()<EOL>if self.is_dir():<EOL><INDENT>total_size = <NUM_LIT:0><EOL>selected = list()<EOL>for p in self.glob("<STR_LIT>"):<EOL><INDENT>if filters(p):<EOL><INDENT>selected.append(p)<EOL>total_size += p.size<EOL><DEDENT><DEDENT>if verbose:<EOL><INDENT>msg = "<STR_LIT>".format(<EOL>len(selected), repr_data_size(total_size),<EOL>)<EOL>print(msg)<EOL><DEDENT>with ZipFile(dst.abspath, "<STR_LIT:w>", compression) as f:<EOL><INDENT>os.chdir(self.abspath)<EOL>for p in selected:<EOL><INDENT>relpath = p.relative_to(self).__str__()<EOL>f.write(relpath)<EOL><DEDENT><DEDENT><DEDENT>elif self.is_file():<EOL><INDENT>with ZipFile(dst.abspath, "<STR_LIT:w>", compression) as f:<EOL><INDENT>os.chdir(self.parent.abspath)<EOL>f.write(self.basename)<EOL><DEDENT><DEDENT>os.chdir(current_dir)<EOL>if verbose:<EOL><INDENT>msg = "<STR_LIT>".format(dst.size_in_text)<EOL>print(msg)<EOL><DEDENT>

Make a zip archive.

:param dst: output file path. if not given, will be automatically assigned.
:param filters: custom path filter. By default it allows any file.
:param compress: compress or not.
:param overwrite: overwrite exists or not.
:param verbose: display log or not.
:return:


def preprocess_arg(arg):  <EOL><INDENT>if arg is None:<EOL><INDENT>return []<EOL><DEDENT>if isinstance(arg, (tuple, list)):<EOL><INDENT>return list(arg)<EOL><DEDENT>else:<EOL><INDENT>return [arg, ]<EOL><DEDENT><DEDENT>self.assert_is_dir_and_exists()<EOL>ignore = preprocess_arg(ignore)<EOL>for i in ignore:<EOL><INDENT>if i.startswith("<STR_LIT:/>") or i.startswith("<STR_LIT:\\>"):<EOL><INDENT>raise ValueError<EOL><DEDENT><DEDENT>ignore_ext = preprocess_arg(ignore_ext)<EOL>for ext in ignore_ext:<EOL><INDENT>if not ext.startswith("<STR_LIT:.>"):<EOL><INDENT>raise ValueError<EOL><DEDENT><DEDENT>ignore_pattern = preprocess_arg(ignore_pattern)<EOL>if case_sensitive:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>ignore = [i.lower() for i in ignore]<EOL>ignore_ext = [i.lower() for i in ignore_ext]<EOL>ignore_pattern = [i.lower() for i in ignore_pattern]<EOL><DEDENT>def filters(p):<EOL><INDENT>relpath = p.relative_to(self).abspath<EOL>if not case_sensitive:<EOL><INDENT>relpath = relpath.lower()<EOL><DEDENT>for i in ignore:<EOL><INDENT>if relpath.startswith(i):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>if case_sensitive:<EOL><INDENT>ext = p.ext<EOL><DEDENT>else:<EOL><INDENT>ext = p.ext.lower()<EOL><DEDENT>if ext in ignore_ext:<EOL><INDENT>return False<EOL><DEDENT>for pattern in ignore_pattern:<EOL><INDENT>if pattern in relpath:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>if ignore_size_smaller_than:<EOL><INDENT>if p.size < ignore_size_smaller_than:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>if ignore_size_larger_than:<EOL><INDENT>if p.size > ignore_size_larger_than:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL><DEDENT>self.make_zip_archive(<EOL>dst=dst, filters=filters, compress=True, overwrite=False, verbose=True,<EOL>)<EOL>

Create a compressed zip archive backup for a directory.

:param dst: the output file path.
:param ignore: file or directory defined in this list will be ignored.
:param ignore_ext: file with extensions defined in this list will be ignored.
:param ignore_pattern: any file or directory that contains this pattern
    will be ignored.
:param ignore_size_smaller_than: any file size smaller than this
    will be ignored.
:param ignore_size_larger_than: any file size larger than this
    will be ignored.

**中文文档**

为一个目录创建一个备份压缩包。可以通过过滤器选择你要备份的文件。


return self.absolute().__str__()<EOL>

r"""
        Absolute path.

        Example: ``C:\User\admin\readme.txt`` for ``C:\User\admin\readme.txt``


return encode_hexstr(self.abspath)<EOL>

Return absolute path encoded in hex string.


return self.parent.abspath<EOL>

r"""
        Parent dir full absolute path.

        Example: ``C:\User\admin`` for ``C:\User\admin\readme.txt``


return encode_hexstr(self.dirpath)<EOL>

Return dir full absolute path encoded in hex string.


return self.parent.name<EOL>

r"""
        Parent dir name.

        Example: ``admin`` for ``C:\User\admin\readme.txt``


return encode_hexstr(self.dirname)<EOL>


return self.name<EOL>

r"""
        File name with extension, path is not included.

        Example: ``readme.txt`` for ``C:\User\admin\readme.txt``


return encode_hexstr(self.basename)<EOL>

File name with extension encoded in hex string.


return self.stem<EOL>

r"""
        File name without extension.

        Example: ``readme`` for ``C:\User\admin\readme.txt``


return encode_hexstr(self.fname)<EOL>


return self.suffix<EOL>

r"""
        File extension. If it's a dir, then return empty str.

        Example: ``.txt`` for ``C:\User\admin\readme.txt``


try:<EOL><INDENT>return self._stat.st_size<EOL><DEDENT>except:  <EOL><INDENT>self._stat = self.stat()<EOL>return self.size<EOL><DEDENT>


return repr_data_size(self.size, precision=<NUM_LIT:2>)<EOL>


try:<EOL><INDENT>return self._stat.st_mtime<EOL><DEDENT>except:  <EOL><INDENT>self._stat = self.stat()<EOL>return self.mtime<EOL><DEDENT>

Get most recent modify time in timestamp.


try:<EOL><INDENT>return self._stat.st_atime<EOL><DEDENT>except:  <EOL><INDENT>self._stat = self.stat()<EOL>return self.atime<EOL><DEDENT>

Get most recent access time in timestamp.


try:<EOL><INDENT>return self._stat.st_ctime<EOL><DEDENT>except:  <EOL><INDENT>self._stat = self.stat()<EOL>return self.ctime<EOL><DEDENT>

Get most recent create time in timestamp.


return datetime.fromtimestamp(self.mtime)<EOL>

Get most recent modify time in datetime.


return datetime.fromtimestamp(self.atime)<EOL>

Get most recent access time in datetime.


return datetime.fromtimestamp(self.ctime)<EOL>

Get most recent create time in datetime.


def join_parsed_parts(self, drv, root, parts, drv2, root2, parts2):

if root2:<EOL><INDENT>if not drv2 and drv:<EOL><INDENT>return drv, root2, [drv + root2] + parts2[<NUM_LIT:1>:]<EOL><DEDENT><DEDENT>elif drv2:<EOL><INDENT>if drv2 == drv or self.casefold(drv2) == self.casefold(drv):<EOL><INDENT>return drv, root, parts + parts2[<NUM_LIT:1>:]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return drv, root, parts + parts2<EOL><DEDENT>return drv2, root2, parts2<EOL>

Join the two paths represented by the respective
(drive, root, parts) tuples.  Return a new (drive, root, parts) tuple.


path_cls = type(parent_path)<EOL>is_dir = path_cls.is_dir<EOL>exists = path_cls.exists<EOL>scandir = parent_path._accessor.scandir<EOL>if not is_dir(parent_path):<EOL><INDENT>return iter([])<EOL><DEDENT>return self._select_from(parent_path, is_dir, exists, scandir)<EOL>

Iterate over all child paths of `parent_path` matched by this
        selector.  This can contain parent_path itself.


if cls is PurePath:<EOL><INDENT>cls = PureWindowsPath if os.name == '<STR_LIT>' else PurePosixPath<EOL><DEDENT>return cls._from_parts(args)<EOL>

Construct a PurePath from one or several strings and or existing
        PurePath objects.  The strings and path objects are combined so as
        to yield a canonicalized path, which is incorporated into the
        new PurePath object.


try:<EOL><INDENT>return self._str<EOL><DEDENT>except AttributeError:<EOL><INDENT>self._str = self._format_parsed_parts(self._drv, self._root,<EOL>self._parts) or '<STR_LIT:.>'<EOL>return self._str<EOL><DEDENT>

Return the string representation of the path, suitable for
        passing to system calls.


f = self._flavour<EOL>return str(self).replace(f.sep, '<STR_LIT:/>')<EOL>

Return the string representation of the path with forward (/)
        slashes.


if sys.version_info < (<NUM_LIT:3>, <NUM_LIT:2>):<EOL><INDENT>raise NotImplementedError("<STR_LIT>")<EOL><DEDENT>return os.fsencode(str(self))<EOL>

Return the bytes representation of the path.  This is only
        recommended to use under Unix.


if not self.is_absolute():<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>return self._flavour.make_uri(self)<EOL>


anchor = self._drv + self._root<EOL>return anchor<EOL>

The concatenation of the drive and root, or ''.


parts = self._parts<EOL>if len(parts) == (<NUM_LIT:1> if (self._drv or self._root) else <NUM_LIT:0>):<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>return parts[-<NUM_LIT:1>]<EOL>


name = self.name<EOL>i = name.rfind('<STR_LIT:.>')<EOL>if <NUM_LIT:0> < i < len(name) - <NUM_LIT:1>:<EOL><INDENT>return name[i:]<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>

The final component's last suffix, if any.


name = self.name<EOL>if name.endswith('<STR_LIT:.>'):<EOL><INDENT>return []<EOL><DEDENT>name = name.lstrip('<STR_LIT:.>')<EOL>return ['<STR_LIT:.>' + suffix for suffix in name.split('<STR_LIT:.>')[<NUM_LIT:1>:]]<EOL>

A list of the final component's suffixes, if any.


name = self.name<EOL>i = name.rfind('<STR_LIT:.>')<EOL>if <NUM_LIT:0> < i < len(name) - <NUM_LIT:1>:<EOL><INDENT>return name[:i]<EOL><DEDENT>else:<EOL><INDENT>return name<EOL><DEDENT>

The final path component, minus its last suffix.


if not self.name:<EOL><INDENT>raise ValueError("<STR_LIT>" % (self,))<EOL><DEDENT>drv, root, parts = self._flavour.parse_parts((name,))<EOL>if (not name or name[-<NUM_LIT:1>] in [self._flavour.sep, self._flavour.altsep]<EOL>or drv or root or len(parts) != <NUM_LIT:1>):<EOL><INDENT>raise ValueError("<STR_LIT>" % (name))<EOL><DEDENT>return self._from_parsed_parts(self._drv, self._root,<EOL>self._parts[:-<NUM_LIT:1>] + [name])<EOL>

Return a new path with the file name changed.


<EOL>f = self._flavour<EOL>if f.sep in suffix or f.altsep and f.altsep in suffix:<EOL><INDENT>raise ValueError("<STR_LIT>" % (suffix))<EOL><DEDENT>if suffix and not suffix.startswith('<STR_LIT:.>') or suffix == '<STR_LIT:.>':<EOL><INDENT>raise ValueError("<STR_LIT>" % (suffix))<EOL><DEDENT>name = self.name<EOL>if not name:<EOL><INDENT>raise ValueError("<STR_LIT>" % (self,))<EOL><DEDENT>old_suffix = self.suffix<EOL>if not old_suffix:<EOL><INDENT>name = name + suffix<EOL><DEDENT>else:<EOL><INDENT>name = name[:-len(old_suffix)] + suffix<EOL><DEDENT>return self._from_parsed_parts(self._drv, self._root,<EOL>self._parts[:-<NUM_LIT:1>] + [name])<EOL>

Return a new path with the file suffix changed (or added, if
        none).


<EOL>if not other:<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>parts = self._parts<EOL>drv = self._drv<EOL>root = self._root<EOL>if root:<EOL><INDENT>abs_parts = [drv, root] + parts[<NUM_LIT:1>:]<EOL><DEDENT>else:<EOL><INDENT>abs_parts = parts<EOL><DEDENT>to_drv, to_root, to_parts = self._parse_args(other)<EOL>if to_root:<EOL><INDENT>to_abs_parts = [to_drv, to_root] + to_parts[<NUM_LIT:1>:]<EOL><DEDENT>else:<EOL><INDENT>to_abs_parts = to_parts<EOL><DEDENT>n = len(to_abs_parts)<EOL>cf = self._flavour.casefold_parts<EOL>if (root or drv) if n == <NUM_LIT:0> else cf(abs_parts[:n]) != cf(to_abs_parts):<EOL><INDENT>formatted = self._format_parsed_parts(to_drv, to_root, to_parts)<EOL>raise ValueError("<STR_LIT>"<EOL>.format(str(self), str(formatted)))<EOL><DEDENT>return self._from_parsed_parts('<STR_LIT>', root if n == <NUM_LIT:1> else '<STR_LIT>',<EOL>abs_parts[n:])<EOL>

Return the relative path to another path identified by the passed
        arguments.  If the operation is not possible (because this is not
        a subpath of the other path), raise ValueError.


<EOL>try:<EOL><INDENT>return self._pparts<EOL><DEDENT>except AttributeError:<EOL><INDENT>self._pparts = tuple(self._parts)<EOL>return self._pparts<EOL><DEDENT>

An object providing sequence-like access to the
        components in the filesystem path.


return self._make_child(args)<EOL>

Combine this path with one or several arguments, and return a
        new path representing either a subpath (if all arguments are relative
        paths) or a totally different path (if one of the arguments is
        anchored).


drv = self._drv<EOL>root = self._root<EOL>parts = self._parts<EOL>if len(parts) == <NUM_LIT:1> and (drv or root):<EOL><INDENT>return self<EOL><DEDENT>return self._from_parsed_parts(drv, root, parts[:-<NUM_LIT:1>])<EOL>


return _PathParents(self)<EOL>

A sequence of this path's logical parents.


if not self._root:<EOL><INDENT>return False<EOL><DEDENT>return not self._flavour.has_drv or bool(self._drv)<EOL>

True if the path is absolute (has both a root and, if applicable,
        a drive).


return self._flavour.is_reserved(self._parts)<EOL>

Return True if the path contains one of the special names reserved
        by the system, if any.


cf = self._flavour.casefold<EOL>path_pattern = cf(path_pattern)<EOL>drv, root, pat_parts = self._flavour.parse_parts((path_pattern,))<EOL>if not pat_parts:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if drv and drv != cf(self._drv):<EOL><INDENT>return False<EOL><DEDENT>if root and root != cf(self._root):<EOL><INDENT>return False<EOL><DEDENT>parts = self._cparts<EOL>if drv or root:<EOL><INDENT>if len(pat_parts) != len(parts):<EOL><INDENT>return False<EOL><DEDENT>pat_parts = pat_parts[<NUM_LIT:1>:]<EOL><DEDENT>elif len(pat_parts) > len(parts):<EOL><INDENT>return False<EOL><DEDENT>for part, pat in zip(reversed(parts), reversed(pat_parts)):<EOL><INDENT>if not fnmatch.fnmatchcase(part, pat):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>

Return True if this path matches the given pattern.


def _raw_open(self, flags, mode=<NUM_LIT>):

if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT>return self._accessor.open(self, flags, mode)<EOL>

Open the file pointed by this path and return a file descriptor,
as os.open() does.


return cls(os.getcwd())<EOL>

Return a new path pointing to the current working directory
        (as returned by os.getcwd()).


return cls(cls()._flavour.gethomedir(None))<EOL>

Return a new path pointing to the user's home directory (as
        returned by os.path.expanduser('~')).


if hasattr(os.path, "<STR_LIT>"):<EOL><INDENT>st = self.stat()<EOL>try:<EOL><INDENT>other_st = other_path.stat()<EOL><DEDENT>except AttributeError:<EOL><INDENT>other_st = os.stat(other_path)<EOL><DEDENT>return os.path.samestat(st, other_st)<EOL><DEDENT>else:<EOL><INDENT>filename1 = six.text_type(self)<EOL>filename2 = six.text_type(other_path)<EOL>st1 = _win32_get_unique_path_id(filename1)<EOL>st2 = _win32_get_unique_path_id(filename2)<EOL>return st1 == st2<EOL><DEDENT>

Return whether other_path is the same or not as this file
        (as returned by os.path.samefile()).


if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT>for name in self._accessor.listdir(self):<EOL><INDENT>if name in ('<STR_LIT:.>', '<STR_LIT:..>'):<EOL><INDENT>continue<EOL><DEDENT>yield self._make_child_relpath(name)<EOL>if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT><DEDENT>

Iterate over the files in this directory.  Does not yield any
        result for the special paths '.' and '..'.


if not pattern:<EOL><INDENT>raise ValueError("<STR_LIT>".format(pattern))<EOL><DEDENT>pattern = self._flavour.casefold(pattern)<EOL>drv, root, pattern_parts = self._flavour.parse_parts((pattern,))<EOL>if drv or root:<EOL><INDENT>raise NotImplementedError("<STR_LIT>")<EOL><DEDENT>selector = _make_selector(tuple(pattern_parts))<EOL>for p in selector.select_from(self):<EOL><INDENT>yield p<EOL><DEDENT>

Iterate over this subtree and yield all existing files (of any
        kind, including directories) matching the given pattern.


pattern = self._flavour.casefold(pattern)<EOL>drv, root, pattern_parts = self._flavour.parse_parts((pattern,))<EOL>if drv or root:<EOL><INDENT>raise NotImplementedError("<STR_LIT>")<EOL><DEDENT>selector = _make_selector(("<STR_LIT>",) + tuple(pattern_parts))<EOL>for p in selector.select_from(self):<EOL><INDENT>yield p<EOL><DEDENT>

Recursively yield all existing files (of any kind, including
        directories) matching the given pattern, anywhere in this subtree.


<EOL>if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT>if self.is_absolute():<EOL><INDENT>return self<EOL><DEDENT>obj = self._from_parts([os.getcwd()] + self._parts, init=False)<EOL>obj._init(template=self)<EOL>return obj<EOL>

Return an absolute version of this path.  This function works
        even if the path doesn't point to anything.

        No normalization is done, i.e. all '.' and '..' will be kept along.
        Use resolve() to get the canonical path to a file.


if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT>s = self._flavour.resolve(self, strict=strict)<EOL>if s is None:<EOL><INDENT>self.stat()<EOL>s = str(self.absolute())<EOL><DEDENT>normed = self._flavour.pathmod.normpath(s)<EOL>obj = self._from_parts((normed,), init=False)<EOL>obj._init(template=self)<EOL>return obj<EOL>

Make the path absolute, resolving all symlinks on the way and also
normalizing it (for example turning slashes into backslashes under
Windows).


return self._accessor.stat(self)<EOL>

Return the result of the stat() system call on this path, like
os.stat() does.


import pwd<EOL>return pwd.getpwuid(self.stat().st_uid).pw_name<EOL>

Return the login name of the file owner.


import grp<EOL>return grp.getgrgid(self.stat().st_gid).gr_name<EOL>


def open(self, mode='<STR_LIT:r>', buffering=-<NUM_LIT:1>, encoding=None,<EOL>errors=None, newline=None):

if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT>if sys.version_info >= (<NUM_LIT:3>, <NUM_LIT:3>):<EOL><INDENT>return io.open(<EOL>str(self), mode, buffering, encoding, errors, newline,<EOL>opener=self._opener)<EOL><DEDENT>else:<EOL><INDENT>return io.open(str(self), mode, buffering,<EOL>encoding, errors, newline)<EOL><DEDENT>

Open the file pointed by this path and return a file object, as
the built-in open() function does.


with self.open(mode='<STR_LIT:rb>') as f:<EOL><INDENT>return f.read()<EOL><DEDENT>

Open the file in bytes mode, read it, and close the file.


def read_text(self, encoding=None, errors=None):

with self.open(mode='<STR_LIT:r>', encoding=encoding, errors=errors) as f:<EOL><INDENT>return f.read()<EOL><DEDENT>

Open the file in text mode, read it, and close the file.


if not isinstance(data, six.binary_type):<EOL><INDENT>raise TypeError(<EOL>'<STR_LIT>' %<EOL>(six.binary_type.__name__, data.__class__.__name__))<EOL><DEDENT>with self.open(mode='<STR_LIT:wb>') as f:<EOL><INDENT>return f.write(data)<EOL><DEDENT>

Open the file in bytes mode, write to it, and close the file.


def write_text(self, data, encoding=None, errors=None):

if not isinstance(data, six.text_type):<EOL><INDENT>raise TypeError(<EOL>'<STR_LIT>' %<EOL>(six.text_type.__name__, data.__class__.__name__))<EOL><DEDENT>with self.open(mode='<STR_LIT:w>', encoding=encoding, errors=errors) as f:<EOL><INDENT>return f.write(data)<EOL><DEDENT>

Open the file in text mode, write to it, and close the file.


def touch(self, mode=<NUM_LIT>, exist_ok=True):

if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT>if exist_ok:<EOL><INDENT>try:<EOL><INDENT>self._accessor.utime(self, None)<EOL><DEDENT>except OSError:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>return<EOL><DEDENT><DEDENT>flags = os.O_CREAT | os.O_WRONLY<EOL>if not exist_ok:<EOL><INDENT>flags |= os.O_EXCL<EOL><DEDENT>fd = self._raw_open(flags, mode)<EOL>os.close(fd)<EOL>

Create this file with the given access mode, if it doesn't exist.


def mkdir(self, mode=<NUM_LIT>, parents=False, exist_ok=False):

if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT>def _try_func():<EOL><INDENT>self._accessor.mkdir(self, mode)<EOL><DEDENT>def _exc_func(exc):<EOL><INDENT>if not parents or self.parent == self:<EOL><INDENT>raise exc<EOL><DEDENT>self.parent.mkdir(parents=True, exist_ok=True)<EOL>self.mkdir(mode, parents=False, exist_ok=exist_ok)<EOL><DEDENT>try:<EOL><INDENT>_try_except_filenotfounderror(_try_func, _exc_func)<EOL><DEDENT>except OSError:<EOL><INDENT>if not exist_ok or not self.is_dir():<EOL><INDENT>raise<EOL><DEDENT><DEDENT>

Create a new directory at this given path.


if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT>self._accessor.chmod(self, mode)<EOL>

Change the permissions of the path, like os.chmod().


if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT>self._accessor.lchmod(self, mode)<EOL>

Like chmod(), except if the path points to a symlink, the symlink's
permissions are changed, rather than its target's.


if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT>self._accessor.unlink(self)<EOL>

Remove this file or link.
If the path is a directory, use rmdir() instead.


if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT>self._accessor.rmdir(self)<EOL>

Remove this directory.  The directory must be empty.


if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT>return self._accessor.lstat(self)<EOL>

Like stat(), except if the path points to a symlink, the symlink's
status information is returned, rather than its target's.


if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT>self._accessor.rename(self, target)<EOL>


if sys.version_info < (<NUM_LIT:3>, <NUM_LIT:3>):<EOL><INDENT>raise NotImplementedError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT>self._accessor.replace(self, target)<EOL>

Rename this path to the given path, clobbering the existing
destination if it exists.


def symlink_to(self, target, target_is_directory=False):

if self._closed:<EOL><INDENT>self._raise_closed()<EOL><DEDENT>self._accessor.symlink(target, self, target_is_directory)<EOL>

Make this path a symlink pointing to the given path.
Note the order of arguments (self, target) is the reverse of
os.symlink's.


try:<EOL><INDENT>self.stat()<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno not in (ENOENT, ENOTDIR):<EOL><INDENT>raise<EOL><DEDENT>return False<EOL><DEDENT>return True<EOL>


try:<EOL><INDENT>return S_ISDIR(self.stat().st_mode)<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno not in (ENOENT, ENOTDIR):<EOL><INDENT>raise<EOL><DEDENT>return False<EOL><DEDENT>


try:<EOL><INDENT>return S_ISREG(self.stat().st_mode)<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno not in (ENOENT, ENOTDIR):<EOL><INDENT>raise<EOL><DEDENT>return False<EOL><DEDENT>

Whether this path is a regular file (also True for symlinks pointing
to regular files).


try:<EOL><INDENT>return S_ISLNK(self.lstat().st_mode)<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno not in (ENOENT, ENOTDIR):<EOL><INDENT>raise<EOL><DEDENT>return False<EOL><DEDENT>


try:<EOL><INDENT>return S_ISBLK(self.stat().st_mode)<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno not in (ENOENT, ENOTDIR):<EOL><INDENT>raise<EOL><DEDENT>return False<EOL><DEDENT>


try:<EOL><INDENT>return S_ISCHR(self.stat().st_mode)<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno not in (ENOENT, ENOTDIR):<EOL><INDENT>raise<EOL><DEDENT>return False<EOL><DEDENT>

Whether this path is a character device.


try:<EOL><INDENT>return S_ISFIFO(self.stat().st_mode)<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno not in (ENOENT, ENOTDIR):<EOL><INDENT>raise<EOL><DEDENT>return False<EOL><DEDENT>


try:<EOL><INDENT>return S_ISSOCK(self.stat().st_mode)<EOL><DEDENT>except OSError as e:<EOL><INDENT>if e.errno not in (ENOENT, ENOTDIR):<EOL><INDENT>raise<EOL><DEDENT>return False<EOL><DEDENT>


if (not (self._drv or self._root)<EOL>and self._parts and self._parts[<NUM_LIT:0>][:<NUM_LIT:1>] == '<STR_LIT>'):<EOL><INDENT>homedir = self._flavour.gethomedir(self._parts[<NUM_LIT:0>][<NUM_LIT:1>:])<EOL>return self._from_parts([homedir] + self._parts[<NUM_LIT:1>:])<EOL><DEDENT>return self<EOL>

Return a new path with expanded ~ and ~user constructs
        (as returned by os.path.expanduser)


@staticmethod<EOL>def sort_by(p_list, reverse=False):<EOL><INDENT>return sorted(<EOL>p_list,<EOL>key=lambda p: getattr(p, key),<EOL>reverse=reverse,<EOL>)<EOL><DEDENT>return sort_by<EOL>


if not self.is_file():<EOL><INDENT>msg = "<STR_LIT>" % self<EOL>raise EnvironmentError(msg)<EOL><DEDENT>

Assert it is a directory and exists in file system.


if not self.is_dir():<EOL><INDENT>msg = "<STR_LIT>" % self<EOL>raise EnvironmentError(msg)<EOL><DEDENT>


if not self.exists():<EOL><INDENT>msg = "<STR_LIT>" % self<EOL>raise EnvironmentError(msg)<EOL><DEDENT>


def select(self, filters=all_true, recursive=True):

self.assert_is_dir_and_exists()<EOL>if recursive:<EOL><INDENT>for p in self.glob("<STR_LIT>"):<EOL><INDENT>if filters(p):<EOL><INDENT>yield p<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for p in self.iterdir():<EOL><INDENT>if filters(p):<EOL><INDENT>yield p<EOL><DEDENT><DEDENT><DEDENT>

Select path by criterion.

        :param filters: a lambda function that take a `pathlib.Path` as input,
          boolean as a output.
        :param recursive: include files in subfolder or not.

        **中文文档**

        根据filters中定义的条件选择路径。


def select_file(self, filters=all_true, recursive=True):

for p in self.select(filters, recursive):<EOL><INDENT>if p.is_file():<EOL><INDENT>yield p<EOL><DEDENT><DEDENT>

Select file path by criterion.

        **中文文档**

        根据filters中定义的条件选择文件。


def select_dir(self, filters=all_true, recursive=True):

for p in self.select(filters, recursive):<EOL><INDENT>if p.is_dir():<EOL><INDENT>yield p<EOL><DEDENT><DEDENT>

Select dir path by criterion.

        **中文文档**

        根据filters中定义的条件选择文件夹。


self.assert_is_dir_and_exists()<EOL>n = <NUM_LIT:0><EOL>for _ in self.select_file(recursive=True):<EOL><INDENT>n += <NUM_LIT:1><EOL><DEDENT>return n<EOL>

Count how many files in this directory. Including file in sub folder.


self.assert_is_dir_and_exists()<EOL>n = <NUM_LIT:0><EOL>for _ in self.select_dir(recursive=True):<EOL><INDENT>n += <NUM_LIT:1><EOL><DEDENT>return n<EOL>

Count how many folders in this directory. Including folder in sub folder.


self.assert_is_dir_and_exists()<EOL>n = <NUM_LIT:0><EOL>for _ in self.select_file(recursive=False):<EOL><INDENT>n += <NUM_LIT:1><EOL><DEDENT>return n<EOL>

Count how many files in this directory (doesn't include files in
sub folders).


self.assert_is_dir_and_exists()<EOL>n = <NUM_LIT:0><EOL>for _ in self.select_dir(recursive=False):<EOL><INDENT>n += <NUM_LIT:1><EOL><DEDENT>return n<EOL>

Count how many folders in this directory (doesn't include folder in
sub folders).


def select_by_ext(self, ext, recursive=True):

ext = [ext.strip().lower() for ext in ensure_list(ext)]<EOL>def filters(p): return p.suffix.lower() in ext<EOL>return self.select_file(filters, recursive)<EOL>

Select file path by extension.

:param ext:

**中文文档**

选择与预定义的若干个扩展名匹配的文件。


if case_sensitive:<EOL><INDENT>def filters(p):<EOL><INDENT>return pattern in p.fname<EOL><DEDENT><DEDENT>else:<EOL><INDENT>pattern = pattern.lower()<EOL>def filters(p):<EOL><INDENT>return pattern in p.fname.lower()<EOL><DEDENT><DEDENT>return self.select_file(filters, recursive)<EOL>

Select file path by text pattern in file name.

**中文文档**

选择文件名中包含指定子字符串的文件。


if case_sensitive:<EOL><INDENT>def filters(p):<EOL><INDENT>return pattern in p.abspath<EOL><DEDENT><DEDENT>else:<EOL><INDENT>pattern = pattern.lower()<EOL>def filters(p):<EOL><INDENT>return pattern in p.abspath.lower()<EOL><DEDENT><DEDENT>return self.select_file(filters, recursive)<EOL>

Select file path by text pattern in absolute path.

**中文文档**

选择绝对路径中包含指定子字符串的文件。


def select_by_size(self, min_size=<NUM_LIT:0>, max_size=<NUM_LIT:1> << <NUM_LIT>, recursive=True):

def filters(p): return min_size <= p.size <= max_size<EOL>return self.select_file(filters, recursive)<EOL>

Select file path by size.

**中文文档**

选择所有文件大小在一定范围内的文件。


def select_by_mtime(self, min_time=<NUM_LIT:0>, max_time=ts_2100,<EOL>recursive=True):

def filters(p): return min_time <= p.mtime <= max_time<EOL>return self.select_file(filters, recursive)<EOL>

Select file path by modify time.

:param min_time: lower bound timestamp
:param max_time: upper bound timestamp

**中文文档**

选择所有 :attr:`pathlib_mate.pathlib2.Path.mtime` 在一定范围内的文件。


def select_by_atime(self, min_time=<NUM_LIT:0>, max_time=ts_2100, recursive=True):

def filters(p): return min_time <= p.atime <= max_time<EOL>return self.select_file(filters, recursive)<EOL>

Select file path by access time.

:param min_time: lower bound timestamp
:param max_time: upper bound timestamp

**中文文档**

选择所有 :attr:`pathlib_mate.pathlib2.Path.atime` 在一定范围内的文件。


def select_by_ctime(self, min_time=<NUM_LIT:0>, max_time=ts_2100,<EOL>recursive=True):

def filters(p): return min_time <= p.ctime <= max_time<EOL>return self.select_file(filters, recursive)<EOL>

Select file path by create time.

:param min_time: lower bound timestamp
:param max_time: upper bound timestamp

**中文文档**

选择所有 :attr:`pathlib_mate.pathlib2.Path.ctime` 在一定范围内的文件。


return self.select_by_ext(self._image_ext, recursive)<EOL>


return self.select_by_ext(self._audio_ext, recursive)<EOL>


return self.select_by_ext(self._video_ext, recursive)<EOL>


return self.select_by_ext(self._ms_word_ext, recursive)<EOL>


return self.select_by_ext(self._ms_excel_ext, recursive)<EOL>


return self.select_by_ext(self._archive_ext, recursive)<EOL>


total = <NUM_LIT:0><EOL>for p in self.select_file(recursive=True):<EOL><INDENT>try:<EOL><INDENT>total += p.size<EOL><DEDENT>except:  <EOL><INDENT>print("<STR_LIT>" % p)<EOL><DEDENT><DEDENT>return total<EOL>

Return total file size (include sub folder). Symlink doesn't count.


if isinstance(value, six.string_types):<EOL><INDENT>return value<EOL><DEDENT>else:<EOL><INDENT>return six.text_type(value)<EOL><DEDENT>


if isinstance(path_or_path_list, (tuple, list, set)):<EOL><INDENT>return [ensure_str(path) for path in path_or_path_list]<EOL><DEDENT>else:<EOL><INDENT>return [ensure_str(path_or_path_list), ]<EOL><DEDENT>

Pre-process input argument, whether if it is:

1. abspath
2. Path instance
3. string
4. list or set of any of them

It returns list of path.

:return path_or_path_list: always return list of path in string

**中文文档**

预处理输入参数。


if size_in_bytes < <NUM_LIT>:<EOL><INDENT>return "<STR_LIT>" % size_in_bytes<EOL><DEDENT>index = <NUM_LIT:0><EOL>while <NUM_LIT:1>:<EOL><INDENT>index += <NUM_LIT:1><EOL>size_in_bytes, mod = divmod(size_in_bytes, <NUM_LIT>)<EOL>if size_in_bytes < <NUM_LIT>:<EOL><INDENT>break<EOL><DEDENT><DEDENT>template = "<STR_LIT>" % precision<EOL>s = template.format(size_in_bytes + mod / <NUM_LIT>, MAGNITUDE_OF_DATA[index])<EOL>return s<EOL>

Return human readable string represent of a file size. Doesn't support
size greater than 1EB.

For example:

- 100 bytes => 100 B
- 100,000 bytes => 97.66 KB
- 100,000,000 bytes => 95.37 MB
- 100,000,000,000 bytes => 93.13 GB
- 100,000,000,000,000 bytes => 90.95 TB
- 100,000,000,000,000,000 bytes => 88.82 PB
...

Magnitude of data::

    1000         kB    kilobyte
    1000 ** 2    MB    megabyte
    1000 ** 3    GB    gigabyte
    1000 ** 4    TB    terabyte
    1000 ** 5    PB    petabyte
    1000 ** 6    EB    exabyte
    1000 ** 7    ZB    zettabyte
    1000 ** 8    YB    yottabyte


if os.path.exists(dst):<EOL><INDENT>print("<STR_LIT>" % dst)<EOL>return<EOL><DEDENT>src, dst = os.path.abspath(src), os.path.abspath(dst)<EOL>cwd = os.getcwd()<EOL>todo = list()<EOL>dirname, basename = os.path.split(src)<EOL>os.chdir(dirname)<EOL>for dirname, _, fnamelist in os.walk(basename):<EOL><INDENT>for fname in fnamelist:<EOL><INDENT>newname = os.path.join(dirname, fname)<EOL>todo.append(newname)<EOL><DEDENT><DEDENT>with ZipFile(dst, "<STR_LIT:w>") as f:<EOL><INDENT>for newname in todo:<EOL><INDENT>f.write(newname)<EOL><DEDENT><DEDENT>os.chdir(cwd)<EOL>

Add a folder and everything inside to zip archive.

Example::

    |---paper
        |--- algorithm.pdf
        |--- images
            |--- 1.jpg

    zip_a_folder("paper", "paper.zip")

    paper.zip
        |---paper
            |--- algorithm.pdf
            |--- images
                |--- 1.jpg

**中文文档**

将整个文件夹添加到压缩包, 包括根目录本身。


if os.path.exists(dst):<EOL><INDENT>print("<STR_LIT>" % dst)<EOL>return<EOL><DEDENT>src, dst = os.path.abspath(src), os.path.abspath(dst)<EOL>cwd = os.getcwd()<EOL>todo = list()<EOL>os.chdir(src)<EOL>for dirname, _, fnamelist in os.walk(os.getcwd()):<EOL><INDENT>for fname in fnamelist:<EOL><INDENT>newname = os.path.relpath(os.path.join(dirname, fname), src)<EOL>todo.append(newname)<EOL><DEDENT><DEDENT>with ZipFile(dst, "<STR_LIT:w>") as f:<EOL><INDENT>for newname in todo:<EOL><INDENT>f.write(newname)<EOL><DEDENT><DEDENT>os.chdir(cwd)<EOL>

Add everything in a folder except the root folder it self to zip archive.

Example::

    |---paper
        |--- algorithm.pdf
        |--- images
            |--- 1.jpg

    zip_everything_in_folder("paper", "paper.zip")

    paper.zip
        |--- algorithm.pdf
        |--- images
            |--- 1.jpg

**中文文档**

将目录内部的所有文件添加到压缩包, 不包括根目录本身。


if os.path.exists(dst):<EOL><INDENT>print("<STR_LIT>" % dst)<EOL>return<EOL><DEDENT>base_dir = os.getcwd()<EOL>with ZipFile(dst, "<STR_LIT:w>") as f:<EOL><INDENT>for abspath in list_of_abspath:<EOL><INDENT>dirname, basename = os.path.split(abspath)<EOL>os.chdir(dirname)<EOL>f.write(basename)<EOL><DEDENT><DEDENT>os.chdir(base_dir)<EOL>

Add many files to a zip archive.

**中文文档**

将一系列的文件压缩到一个压缩包中, 若有重复的文件名, 在zip中保留所有的副本。


if self.exists():<EOL><INDENT>if self.is_file():<EOL><INDENT>return self.size == <NUM_LIT:0><EOL><DEDENT>elif self.is_dir():<EOL><INDENT>if strict:<EOL><INDENT>return len(list(self.select(recursive=True))) == <NUM_LIT:0><EOL><DEDENT>else:  <EOL><INDENT>return len(list(self.select_file(recursive=True))) == <NUM_LIT:0><EOL><DEDENT><DEDENT>else:  <EOL><INDENT>msg = "<STR_LIT>" % self<EOL>raise EnvironmentError(msg)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise EnvironmentError("<STR_LIT>" % self)<EOL><DEDENT>

- If it's a file, check if it is a empty file. (0 bytes content)
- If it's a directory, check if there's no file and dir in it.
    But if ``strict = False``, then only check if there's no file in it.

:param strict: only useful when it is a directory. if True, only
    return True if this dir has no dir and file. if False, return True
    if it doesn't have any file.


self_basename = self.basename<EOL>self_basename_lower = self.basename.lower()<EOL>if case_sensitive:  <EOL><INDENT>def match(basename):<EOL><INDENT>return basename.startswith(self_basename)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>def match(basename):<EOL><INDENT>return basename.lower().startswith(self_basename_lower)<EOL><DEDENT><DEDENT>choices = list()<EOL>if self.is_dir():<EOL><INDENT>choices.append(self)<EOL>for p in self.sort_by_abspath(self.select(recursive=False)):<EOL><INDENT>choices.append(p)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>p_parent = self.parent<EOL>if p_parent.is_dir():<EOL><INDENT>for p in self.sort_by_abspath(p_parent.select(recursive=False)):<EOL><INDENT>if match(p.basename):<EOL><INDENT>choices.append(p)<EOL><DEDENT><DEDENT><DEDENT>else:  <EOL><INDENT>raise ValueError("<STR_LIT>" % p_parent)<EOL><DEDENT><DEDENT>return choices<EOL>

A command line auto complete similar behavior. Find all item with same
prefix of this one.

:param case_sensitive: toggle if it is case sensitive.
:return: list of :class:`pathlib_mate.pathlib2.Path`.


self.assert_is_dir_and_exists()<EOL>size_table = sorted(<EOL>[(p, p.dirsize) for p in self.select_dir(recursive=False)],<EOL>key=lambda x: x[<NUM_LIT:1>],<EOL>reverse=True,<EOL>)<EOL>for p, size in size_table[:top_n]:<EOL><INDENT>print("<STR_LIT>".format(repr_data_size(size), p.abspath))<EOL><DEDENT>


self.assert_is_dir_and_exists()<EOL>size_table = sorted(<EOL>[(p, p.size) for p in self.select_file(recursive=True)],<EOL>key=lambda x: x[<NUM_LIT:1>],<EOL>reverse=True,<EOL>)<EOL>for p, size in size_table[:top_n]:<EOL><INDENT>print("<STR_LIT>".format(repr_data_size(size), p.abspath))<EOL><DEDENT>


self.assert_is_dir_and_exists()<EOL>size_table1 = sorted(<EOL>[(p, p.dirsize) for p in self.select_dir(recursive=False)],<EOL>key=lambda x: x[<NUM_LIT:1>],<EOL>reverse=True,<EOL>)<EOL>for p1, size1 in size_table1[:top_n]:<EOL><INDENT>print("<STR_LIT>".format(repr_data_size(size1), p1.abspath))<EOL>size_table2 = sorted(<EOL>[(p, p.size) for p in p1.select_file(recursive=True)],<EOL>key=lambda x: x[<NUM_LIT:1>],<EOL>reverse=True,<EOL>)<EOL>for p2, size2 in size_table2[:top_n]:<EOL><INDENT>print("<STR_LIT>".format(<EOL>repr_data_size(size2), p2.abspath))<EOL><DEDENT><DEDENT>

Print ``top_n`` big dir and ``top_n`` big file in each dir.


self.assert_is_dir_and_exists()<EOL>from collections import OrderedDict<EOL>stat = OrderedDict()<EOL>stat[self.abspath] = {"<STR_LIT:file>": <NUM_LIT:0>, "<STR_LIT>": <NUM_LIT:0>, "<STR_LIT:size>": <NUM_LIT:0>}<EOL>for p in self.select(filters=filters, recursive=True):<EOL><INDENT>if p.is_file():<EOL><INDENT>size = p.size<EOL>while <NUM_LIT:1>:<EOL><INDENT>parent = p.parent<EOL>stat[parent.abspath]["<STR_LIT:file>"] += <NUM_LIT:1><EOL>stat[parent.abspath]["<STR_LIT:size>"] += size<EOL>if parent.abspath == self.abspath:<EOL><INDENT>break<EOL><DEDENT>p = parent<EOL><DEDENT><DEDENT>elif p.is_dir():<EOL><INDENT>stat[p.abspath] = {"<STR_LIT:file>": <NUM_LIT:0>, "<STR_LIT>": <NUM_LIT:0>, "<STR_LIT:size>": <NUM_LIT:0>}<EOL>while <NUM_LIT:1>:<EOL><INDENT>parent = p.parent<EOL>stat[parent.abspath]["<STR_LIT>"] += <NUM_LIT:1><EOL>if parent.abspath == self.abspath:<EOL><INDENT>break<EOL><DEDENT>p = parent<EOL><DEDENT><DEDENT><DEDENT>return stat<EOL>

Find out how many files, directories and total size (Include file in
it's sub-folder) it has for each folder and sub-folder.

:returns: stat, a dict like ``{"directory path": {
  "file": number of files, "dir": number of directories,
  "size": total size in bytes}}``

**中文文档**

返回一个目录中的每个子目录的, 文件, 文件夹, 大小的统计数据。


self.assert_is_dir_and_exists()<EOL>stat = {"<STR_LIT:file>": <NUM_LIT:0>, "<STR_LIT>": <NUM_LIT:0>, "<STR_LIT:size>": <NUM_LIT:0>}<EOL>for p in self.select(filters=filters, recursive=True):<EOL><INDENT>if p.is_file():<EOL><INDENT>stat["<STR_LIT:file>"] += <NUM_LIT:1><EOL>stat["<STR_LIT:size>"] += p.size<EOL><DEDENT>elif p.is_dir():<EOL><INDENT>stat["<STR_LIT>"] += <NUM_LIT:1><EOL><DEDENT><DEDENT>return stat<EOL>

Find out how many files, directorys and total size (Include file in
        it's sub-folder).

        :returns: stat, a dict like ``{"file": number of files,
          "dir": number of directorys, "size": total size in bytes}``

        **中文文档**

        返回一个目录中的文件, 文件夹, 大小的统计数据。


self.assert_is_dir_and_exists()<EOL>src = self.abspath<EOL>dst = os.path.abspath(dst)<EOL>if os.path.exists(dst):  <EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>folder_to_create = list()<EOL>file_to_create = list()<EOL>for current_folder, _, file_list in os.walk(self.abspath):<EOL><INDENT>current_folder = current_folder.replace(src, dst)<EOL>try:<EOL><INDENT>os.mkdir(current_folder)<EOL><DEDENT>except:  <EOL><INDENT>pass<EOL><DEDENT>for basename in file_list:<EOL><INDENT>abspath = os.path.join(current_folder, basename)<EOL>with open(abspath, "<STR_LIT:wb>") as _:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>

Create a new folder having exactly same structure with this directory.
However, all files are just empty file with same file name.

:param dst: destination directory. The directory can't exists before
you execute this.

**中文文档**

创建一个目录的镜像拷贝, 与拷贝操作不同的是, 文件的副本只是在文件名上
与原件一致, 但是是空文件, 完全没有内容, 文件大小为0。


import subprocess<EOL>self.assert_is_dir_and_exists()<EOL>if py_exe is None:<EOL><INDENT>if six.PY2:<EOL><INDENT>py_exe = "<STR_LIT>"<EOL><DEDENT>elif six.PY3:<EOL><INDENT>py_exe = "<STR_LIT>"<EOL><DEDENT><DEDENT>for p in self.select_by_ext("<STR_LIT>"):<EOL><INDENT>subprocess.Popen('<STR_LIT>' % (py_exe, p.abspath))<EOL><DEDENT>

Execute every ``.py`` file as main script.

:param py_exe: str, python command or python executable path.

**中文文档**

将目录下的所有Python文件作为主脚本用当前解释器运行。


def trail_space(self, filters=lambda p: p.ext == "<STR_LIT>"):

self.assert_is_dir_and_exists()<EOL>for p in self.select_file(filters):<EOL><INDENT>try:<EOL><INDENT>with open(p.abspath, "<STR_LIT:rb>") as f:<EOL><INDENT>lines = list()<EOL>for line in f:<EOL><INDENT>lines.append(line.decode("<STR_LIT:utf-8>").rstrip())<EOL><DEDENT><DEDENT>with open(p.abspath, "<STR_LIT:wb>") as f:<EOL><INDENT>f.write("<STR_LIT:\n>".join(lines).encode("<STR_LIT:utf-8>"))<EOL><DEDENT><DEDENT>except Exception as e:  <EOL><INDENT>raise e<EOL><DEDENT><DEDENT>

Trail white space at end of each line for every ``.py`` file.

**中文文档**

将目录下的所有被选择的文件中行末的空格删除。


self.assert_is_dir_and_exists()<EOL>for p in self.select_by_ext("<STR_LIT>"):<EOL><INDENT>with open(p.abspath, "<STR_LIT:rb>") as f:<EOL><INDENT>code = f.read().decode("<STR_LIT:utf-8>")<EOL><DEDENT>formatted_code = autopep8.fix_code(code, **kwargs)<EOL>with open(p.abspath, "<STR_LIT:wb>") as f:<EOL><INDENT>f.write(formatted_code.encode("<STR_LIT:utf-8>"))<EOL><DEDENT><DEDENT>

Auto convert your python code in a directory to pep8 styled code.

:param kwargs: arguments for ``autopep8.fix_code`` method.

**中文文档**

将目录下的所有Python文件用pep8风格格式化。增加其可读性和规范性。


return binascii.b2a_hex(text.encode("<STR_LIT:utf-8>")).decode("<STR_LIT:utf-8>")<EOL>

Convert any utf-8 string to hex string.

**中文文档**

将任意utf-8字符串编码为16进制字符串。


return binascii.a2b_hex(text.encode("<STR_LIT:utf-8>")).decode("<STR_LIT:utf-8>")<EOL>

Reverse operation of :func:`encode_hexstr`.

**中文文档**

将16进制字符串解码为原字符串。