def App(apptype, data_flow_kernel=None, walltime=<NUM_LIT>, cache=False, executors='<STR_LIT:all>'):

from parsl.app.python import PythonApp<EOL>from parsl.app.bash import BashApp<EOL>logger.warning("<STR_LIT>")<EOL>if apptype == '<STR_LIT>':<EOL><INDENT>app_class = PythonApp<EOL><DEDENT>elif apptype == '<STR_LIT>':<EOL><INDENT>app_class = BashApp<EOL><DEDENT>else:<EOL><INDENT>raise InvalidAppTypeError("<STR_LIT>".format(apptype))<EOL><DEDENT>def wrapper(f):<EOL><INDENT>return app_class(f,<EOL>data_flow_kernel=data_flow_kernel,<EOL>walltime=walltime,<EOL>cache=cache,<EOL>executors=executors)<EOL><DEDENT>return wrapper<EOL>

The App decorator function.

    Args:
        - apptype (string) : Apptype can be bash|python

    Kwargs:
        - data_flow_kernel (DataFlowKernel): The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for
          managing this app. This can be omitted only
          after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`.
        - walltime (int) : Walltime for app in seconds,
             default=60
        - executors (str|list) : Labels of the executors that this app can execute over. Default is 'all'.
        - cache (Bool) : Enable caching of the app call
             default=False

    Returns:
         A PythonApp or BashApp object, which when called runs the apps through the executor.


def python_app(function=None, data_flow_kernel=None, walltime=<NUM_LIT>, cache=False, executors='<STR_LIT:all>'):

from parsl.app.python import PythonApp<EOL>def decorator(func):<EOL><INDENT>def wrapper(f):<EOL><INDENT>return PythonApp(f,<EOL>data_flow_kernel=data_flow_kernel,<EOL>walltime=walltime,<EOL>cache=cache,<EOL>executors=executors)<EOL><DEDENT>return wrapper(func)<EOL><DEDENT>if function is not None:<EOL><INDENT>return decorator(function)<EOL><DEDENT>return decorator<EOL>

Decorator function for making python apps.

    Parameters
    ----------
    function : function
        Do not pass this keyword argument directly. This is needed in order to allow for omitted parenthesis,
        for example, `@python_app` if using all defaults or `@python_app(walltime=120)`. If the
        decorator is used alone, function will be the actual function being decorated, whereas if it
        is called with arguments, function will be None. Default is None.
    data_flow_kernel : DataFlowKernel
        The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for managing this app. This can
        be omitted only after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`. Default is None.
    walltime : int
        Walltime for app in seconds. Default is 60.
    executors : string or list
        Labels of the executors that this app can execute over. Default is 'all'.
    cache : bool
        Enable caching of the app call. Default is False.


def bash_app(function=None, data_flow_kernel=None, walltime=<NUM_LIT>, cache=False, executors='<STR_LIT:all>'):

from parsl.app.bash import BashApp<EOL>def decorator(func):<EOL><INDENT>def wrapper(f):<EOL><INDENT>return BashApp(f,<EOL>data_flow_kernel=data_flow_kernel,<EOL>walltime=walltime,<EOL>cache=cache,<EOL>executors=executors)<EOL><DEDENT>return wrapper(func)<EOL><DEDENT>if function is not None:<EOL><INDENT>return decorator(function)<EOL><DEDENT>return decorator<EOL>

Decorator function for making bash apps.

    Parameters
    ----------
    function : function
        Do not pass this keyword argument directly. This is needed in order to allow for omitted parenthesis,
        for example, `@bash_app` if using all defaults or `@bash_app(walltime=120)`. If the
        decorator is used alone, function will be the actual function being decorated, whereas if it
        is called with arguments, function will be None. Default is None.
    data_flow_kernel : DataFlowKernel
        The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for managing this app. This can
        be omitted only after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`. Default is None.
    walltime : int
        Walltime for app in seconds. Default is 60.
    executors : string or list
        Labels of the executors that this app can execute over. Default is 'all'.
    cache : bool
        Enable caching of the app call. Default is False.


def __init__(self, func, data_flow_kernel=None, walltime=<NUM_LIT>, executors='<STR_LIT:all>', cache=False):

self.__name__ = func.__name__<EOL>self.func = func<EOL>self.data_flow_kernel = data_flow_kernel<EOL>self.status = '<STR_LIT>'<EOL>self.executors = executors<EOL>self.cache = cache<EOL>if not (isinstance(executors, list) or isinstance(executors, str)):<EOL><INDENT>logger.error("<STR_LIT>".format(<EOL>func.__name__))<EOL><DEDENT>if cache is True:<EOL><INDENT>try:<EOL><INDENT>self.fn_source = getsource(func)<EOL><DEDENT>except OSError:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>self.fn_source = func.__name__<EOL><DEDENT>self.func_hash = md5(self.fn_source.encode('<STR_LIT:utf-8>')).hexdigest()<EOL><DEDENT>else:<EOL><INDENT>self.func_hash = func.__name__<EOL><DEDENT>params = signature(func).parameters<EOL>self.kwargs = {}<EOL>if '<STR_LIT>' in params:<EOL><INDENT>self.kwargs['<STR_LIT>'] = params['<STR_LIT>'].default<EOL><DEDENT>if '<STR_LIT>' in params:<EOL><INDENT>self.kwargs['<STR_LIT>'] = params['<STR_LIT>'].default<EOL><DEDENT>self.outputs = params['<STR_LIT>'].default if '<STR_LIT>' in params else []<EOL>self.inputs = params['<STR_LIT>'].default if '<STR_LIT>' in params else []<EOL>

Construct the App object.

        Args:
             - func (function): Takes the function to be made into an App

        Kwargs:
             - data_flow_kernel (DataFlowKernel): The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for
               managing this app. This can be omitted only
               after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`.
             - walltime (int) : Walltime in seconds for the app execution.
             - executors (str|list) : Labels of the executors that this app can execute over. Default is 'all'.
             - cache (Bool) : Enable caching of this app ?

        Returns:
             - App object.


@typeguard.typechecked<EOL>def set_stream_logger(name: str = '<STR_LIT>', level: int = logging.DEBUG, format_string: Optional[str] = None):

if format_string is None:<EOL><INDENT>format_string = "<STR_LIT>"<EOL><DEDENT>logger = logging.getLogger(name)<EOL>logger.setLevel(logging.DEBUG)<EOL>handler = logging.StreamHandler()<EOL>handler.setLevel(level)<EOL>formatter = logging.Formatter(format_string, datefmt='<STR_LIT>')<EOL>handler.setFormatter(formatter)<EOL>logger.addHandler(handler)<EOL>futures_logger = logging.getLogger("<STR_LIT>")<EOL>futures_logger.addHandler(handler)<EOL>

Add a stream log handler.

    Args:
         - name (string) : Set the logger name.
         - level (logging.LEVEL) : Set to logging.DEBUG by default.
         - format_string (string) : Set to None by default.

    Returns:
         - None


@typeguard.typechecked<EOL>def set_file_logger(filename: str, name: str = '<STR_LIT>', level: int = logging.DEBUG, format_string: Optional[str] = None):

if format_string is None:<EOL><INDENT>format_string = "<STR_LIT>"<EOL><DEDENT>logger = logging.getLogger(name)<EOL>logger.setLevel(logging.DEBUG)<EOL>handler = logging.FileHandler(filename)<EOL>handler.setLevel(level)<EOL>formatter = logging.Formatter(format_string, datefmt='<STR_LIT>')<EOL>handler.setFormatter(formatter)<EOL>logger.addHandler(handler)<EOL>futures_logger = logging.getLogger("<STR_LIT>")<EOL>futures_logger.addHandler(handler)<EOL>

Add a stream log handler.

    Args:
        - filename (string): Name of the file to write logs to
        - name (string): Logger name
        - level (logging.LEVEL): Set the logging level.
        - format_string (string): Set the format string

    Returns:
       -  None


self.session = self.create_session()<EOL>self.client = self.session.client('<STR_LIT>')<EOL>self.ec2 = self.session.resource('<STR_LIT>')<EOL>self.instances = []<EOL>self.instance_states = {}<EOL>self.vpc_id = <NUM_LIT:0><EOL>self.sg_id = <NUM_LIT:0><EOL>self.sn_ids = []<EOL>


try:<EOL><INDENT>fh = open(state_file, '<STR_LIT:r>')<EOL>state = json.load(fh)<EOL>self.vpc_id = state['<STR_LIT>']<EOL>self.sg_id = state['<STR_LIT>']<EOL>self.sn_ids = state['<STR_LIT>']<EOL>self.instances = state['<STR_LIT>']<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.debug("<STR_LIT>".format(e))<EOL>raise e<EOL><DEDENT>logger.debug("<STR_LIT>")<EOL>

Read the state file, if it exists.

        If this script has been run previously, resource IDs will have been written to a
        state file. On starting a run, a state file will be looked for before creating new
        infrastructure. Information on VPCs, security groups, and subnets are saved, as
        well as running instances and their states.

        AWS has a maximum number of VPCs per region per account, so we do not want to
        clutter users' AWS accounts with security groups and VPCs that will be used only
        once.


fh = open('<STR_LIT>', '<STR_LIT:w>')<EOL>state = {}<EOL>state['<STR_LIT>'] = self.vpc_id<EOL>state['<STR_LIT>'] = self.sg_id<EOL>state['<STR_LIT>'] = self.sn_ids<EOL>state['<STR_LIT>'] = self.instances<EOL>state["<STR_LIT>"] = self.instance_states<EOL>fh.write(json.dumps(state, indent=<NUM_LIT:4>))<EOL>

Save information that must persist to a file.

        We do not want to create a new VPC and new identical security groups, so we save
        information about them in a file between runs.


session = None<EOL>if self.key_file is not None:<EOL><INDENT>credfile = os.path.expandvars(os.path.expanduser(self.key_file))<EOL>try:<EOL><INDENT>with open(credfile, '<STR_LIT:r>') as f:<EOL><INDENT>creds = json.load(f)<EOL><DEDENT><DEDENT>except json.JSONDecodeError as e:<EOL><INDENT>logger.error(<EOL>"<STR_LIT>".format(self.label, credfile)<EOL>)<EOL>raise e<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.debug(<EOL>"<STR_LIT>".format(<EOL>self.label, credfile<EOL>)<EOL>)<EOL>raise e<EOL><DEDENT>logger.debug("<STR_LIT>".format(self.label))<EOL>session = boto3.session.Session(region_name=self.region, **creds)<EOL><DEDENT>elif self.profile is not None:<EOL><INDENT>logger.debug("<STR_LIT>".format(self.label))<EOL>session = boto3.session.Session(<EOL>profile_name=self.profile, region_name=self.region<EOL>)<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>".format(self.label))<EOL>session = boto3.session.Session(region_name=self.region)<EOL><DEDENT>return session<EOL>

Create a session.

        First we look in self.key_file for a path to a json file with the
        credentials. The key file should have 'AWSAccessKeyId' and 'AWSSecretKey'.

        Next we look at self.profile for a profile name and try
        to use the Session call to automatically pick up the keys for the profile from
        the user default keys file ~/.aws/config.

        Finally, boto3 will look for the keys in environment variables:
        AWS_ACCESS_KEY_ID: The access key for your AWS account.
        AWS_SECRET_ACCESS_KEY: The secret key for your AWS account.
        AWS_SESSION_TOKEN: The session key for your AWS account.
        This is only needed when you are using temporary credentials.
        The AWS_SECURITY_TOKEN environment variable can also be used,
        but is only supported for backwards compatibility purposes.
        AWS_SESSION_TOKEN is supported by multiple AWS SDKs besides python.


try:<EOL><INDENT>vpc = self.ec2.create_vpc(<EOL>CidrBlock='<STR_LIT>',<EOL>AmazonProvidedIpv6CidrBlock=False,<EOL>)<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.error("<STR_LIT>".format(e))<EOL>raise e<EOL><DEDENT>internet_gateway = self.ec2.create_internet_gateway()<EOL>internet_gateway.attach_to_vpc(VpcId=vpc.vpc_id)  <EOL>self.internet_gateway = internet_gateway.id<EOL>route_table = self.config_route_table(vpc, internet_gateway)<EOL>self.route_table = route_table.id<EOL>availability_zones = self.client.describe_availability_zones()<EOL>for num, zone in enumerate(availability_zones['<STR_LIT>']):<EOL><INDENT>if zone['<STR_LIT>'] == "<STR_LIT>":<EOL><INDENT>subnet = vpc.create_subnet(<EOL>CidrBlock='<STR_LIT>'.format(<NUM_LIT:16> * num), AvailabilityZone=zone['<STR_LIT>']<EOL>)<EOL>subnet.meta.client.modify_subnet_attribute(<EOL>SubnetId=subnet.id, MapPublicIpOnLaunch={"<STR_LIT>": True}<EOL>)<EOL>route_table.associate_with_subnet(SubnetId=subnet.id)<EOL>self.sn_ids.append(subnet.id)<EOL><DEDENT>else:<EOL><INDENT>logger.info("<STR_LIT>".format(zone['<STR_LIT>']))<EOL><DEDENT><DEDENT>self.security_group(vpc)<EOL>self.vpc_id = vpc.id<EOL>return vpc<EOL>

Create and configure VPC

        We create a VPC with CIDR 10.0.0.0/16, which provides up to 64,000 instances.

        We attach a subnet for each availability zone within the region specified in the
        config. We give each subnet an ip range like 10.0.X.0/20, which is large enough
        for approx. 4000 instances.

        Security groups are configured in function security_group.


sg = vpc.create_security_group(<EOL>GroupName="<STR_LIT>", Description="<STR_LIT>"<EOL>)<EOL>ip_ranges = [{'<STR_LIT>': '<STR_LIT>'}]<EOL>in_permissions = [<EOL>{<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT>,<EOL>'<STR_LIT>': ip_ranges,<EOL>}, {<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT>,<EOL>'<STR_LIT>': ip_ranges,<EOL>}, {<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': -<NUM_LIT:1>,<EOL>'<STR_LIT>': -<NUM_LIT:1>,<EOL>'<STR_LIT>': [{<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>}],<EOL>}, {<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': <NUM_LIT>,<EOL>'<STR_LIT>': <NUM_LIT>,<EOL>'<STR_LIT>': [{<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>}],<EOL>}<EOL>]<EOL>out_permissions = [<EOL>{<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT>,<EOL>'<STR_LIT>': [{<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>}],<EOL>},<EOL>{<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT>,<EOL>'<STR_LIT>': ip_ranges,<EOL>},<EOL>{<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': <NUM_LIT:0>,<EOL>'<STR_LIT>': <NUM_LIT>,<EOL>'<STR_LIT>': ip_ranges,<EOL>},<EOL>]<EOL>sg.authorize_ingress(IpPermissions=in_permissions)<EOL>sg.authorize_egress(IpPermissions=out_permissions)<EOL>self.sg_id = sg.id<EOL>return sg<EOL>

Create and configure a new security group.

        Allows all ICMP in, all TCP and UDP in within VPC.

        This security group is very open. It allows all incoming ping requests on all
        ports. It also allows all outgoing traffic on all ports. This can be limited by
        changing the allowed port ranges.

        Parameters
        ----------
        vpc : VPC instance
            VPC in which to set up security group.


def config_route_table(self, vpc, internet_gateway):

route_table = vpc.create_route_table()<EOL>route_table.create_route(<EOL>DestinationCidrBlock='<STR_LIT>', GatewayId=internet_gateway.internet_gateway_id<EOL>)<EOL>return route_table<EOL>

Configure route table for Virtual Private Cloud (VPC).

        Parameters
        ----------
        vpc : dict
            Representation of the VPC (created by create_vpc()).
        internet_gateway : dict
            Representation of the internet gateway (created by create_vpc()).


def spin_up_instance(self, command, job_name):

command = Template(template_string).substitute(jobname=job_name,<EOL>user_script=command,<EOL>linger=str(self.linger).lower(),<EOL>worker_init=self.worker_init)<EOL>instance_type = self.instance_type<EOL>subnet = self.sn_ids[<NUM_LIT:0>]<EOL>ami_id = self.image_id<EOL>total_instances = len(self.instances)<EOL>if float(self.spot_max_bid) > <NUM_LIT:0>:<EOL><INDENT>spot_options = {<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': str(self.spot_max_bid),<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>}<EOL>}<EOL><DEDENT>else:<EOL><INDENT>spot_options = {}<EOL><DEDENT>if total_instances > self.max_nodes:<EOL><INDENT>logger.warn("<STR_LIT>".format(self.max_nodes))<EOL>return [None]<EOL><DEDENT>try:<EOL><INDENT>tag_spec = [{"<STR_LIT>": "<STR_LIT>", "<STR_LIT>": [{'<STR_LIT>': '<STR_LIT:Name>', '<STR_LIT>': job_name}]}]<EOL>instance = self.ec2.create_instances(<EOL>MinCount=<NUM_LIT:1>,<EOL>MaxCount=<NUM_LIT:1>,<EOL>InstanceType=instance_type,<EOL>ImageId=ami_id,<EOL>KeyName=self.key_name,<EOL>SubnetId=subnet,<EOL>SecurityGroupIds=[self.sg_id],<EOL>TagSpecifications=tag_spec,<EOL>InstanceMarketOptions=spot_options,<EOL>InstanceInitiatedShutdownBehavior='<STR_LIT>',<EOL>IamInstanceProfile={'<STR_LIT>': self.iam_instance_profile_arn},<EOL>UserData=command<EOL>)<EOL><DEDENT>except ClientError as e:<EOL><INDENT>print(e)<EOL>logger.error(e.response)<EOL>return [None]<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.error("<STR_LIT>".format(e))<EOL>return [None]<EOL><DEDENT>self.instances.append(instance[<NUM_LIT:0>].id)<EOL>logger.info(<EOL>"<STR_LIT>".format(instance[<NUM_LIT:0>].id, instance_type)<EOL>)<EOL>return instance<EOL>

Start an instance in the VPC in the first available subnet.

        N instances will be started if nodes_per_block > 1.
        Not supported. We only do 1 node per block.

        Parameters
        ----------
        command : str
            Command string to execute on the node.
        job_name : str
            Name associated with the instances.


if instances and len(self.instances) > <NUM_LIT:0>:<EOL><INDENT>print(instances)<EOL>try:<EOL><INDENT>print([i.id for i in instances])<EOL><DEDENT>except Exception as e:<EOL><INDENT>print(e)<EOL><DEDENT>term = self.client.terminate_instances(InstanceIds=instances)<EOL>logger.info("<STR_LIT>".format(len(instances), str(instances)))<EOL><DEDENT>elif len(self.instances) > <NUM_LIT:0>:<EOL><INDENT>instance = self.instances.pop()<EOL>term = self.client.terminate_instances(InstanceIds=[instance])<EOL>logger.info("<STR_LIT>".format(instance))<EOL><DEDENT>else:<EOL><INDENT>logger.warn("<STR_LIT>")<EOL>return -<NUM_LIT:1><EOL><DEDENT>self.get_instance_state()<EOL>return term<EOL>

Shut down a list of instances, if provided.

        If no instance is provided, the last instance started up will be shut down.


if instances:<EOL><INDENT>desc = self.client.describe_instances(InstanceIds=instances)<EOL><DEDENT>else:<EOL><INDENT>desc = self.client.describe_instances(InstanceIds=self.instances)<EOL><DEDENT>for i in range(len(desc['<STR_LIT>'])):<EOL><INDENT>instance = desc['<STR_LIT>'][i]['<STR_LIT>'][<NUM_LIT:0>]<EOL>self.instance_states[instance['<STR_LIT>']] = instance['<STR_LIT>']['<STR_LIT:Name>']<EOL><DEDENT>return self.instance_states<EOL>

Get states of all instances on EC2 which were started by this file.


all_states = []<EOL>status = self.client.describe_instances(InstanceIds=job_ids)<EOL>for r in status['<STR_LIT>']:<EOL><INDENT>for i in r['<STR_LIT>']:<EOL><INDENT>instance_id = i['<STR_LIT>']<EOL>instance_state = translate_table.get(i['<STR_LIT>']['<STR_LIT:Name>'], '<STR_LIT>')<EOL>self.resources[instance_id]['<STR_LIT:status>'] = instance_state<EOL>all_states.extend([instance_state])<EOL><DEDENT><DEDENT>return all_states<EOL>

Get the status of a list of jobs identified by their ids.

        Parameters
        ----------
        job_ids : list of str
            Identifiers for the jobs.

        Returns
        -------
        list of int
            The status codes of the requsted jobs.


def submit(self, command='<STR_LIT>', blocksize=<NUM_LIT:1>, tasks_per_node=<NUM_LIT:1>, job_name="<STR_LIT>"):

job_name = "<STR_LIT>".format(time.time())<EOL>wrapped_cmd = self.launcher(command,<EOL>tasks_per_node,<EOL>self.nodes_per_block)<EOL>[instance, *rest] = self.spin_up_instance(command=wrapped_cmd, job_name=job_name)<EOL>if not instance:<EOL><INDENT>logger.error("<STR_LIT>")<EOL>return None<EOL><DEDENT>logger.debug("<STR_LIT>".format(instance.instance_id))<EOL>state = translate_table.get(instance.state['<STR_LIT:Name>'], "<STR_LIT>")<EOL>self.resources[instance.instance_id] = {<EOL>"<STR_LIT>": instance.instance_id,<EOL>"<STR_LIT>": instance,<EOL>"<STR_LIT:status>": state<EOL>}<EOL>return instance.instance_id<EOL>

Submit the command onto a freshly instantiated AWS EC2 instance.

        Submit returns an ID that corresponds to the task that was just submitted.

        Parameters
        ----------
        command : str
            Command to be invoked on the remote side.
        blocksize : int
            Number of blocks requested.
        tasks_per_node : int (default=1)
            Number of command invocations to be launched per node
        job_name : str
            Prefix for the job name.

        Returns
        -------
        None or str
            If at capacity, None will be returned. Otherwise, the job identifier will be returned.


if self.linger is True:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return [False for x in job_ids]<EOL><DEDENT>try:<EOL><INDENT>self.client.terminate_instances(InstanceIds=list(job_ids))<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.error("<STR_LIT>".format(job_ids))<EOL>raise e<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>".format(job_ids))<EOL><DEDENT>for job_id in job_ids:<EOL><INDENT>self.resources[job_id]["<STR_LIT:status>"] = "<STR_LIT>"<EOL><DEDENT>for job_id in job_ids:<EOL><INDENT>self.instances.remove(job_id)<EOL><DEDENT>return [True for x in job_ids]<EOL>

Cancel the jobs specified by a list of job ids.

        Parameters
        ----------
        job_ids : list of str
            List of of job identifiers

        Returns
        -------
        list of bool
            Each entry in the list will contain False if the operation fails. Otherwise, the entry will be True.


self.get_instance_state()<EOL>status_string = "<STR_LIT>".format(<EOL>self.vpc_id, self.sn_ids, self.sg_id, self.instances<EOL>)<EOL>status_string += "<STR_LIT>"<EOL>self.get_instance_state()<EOL>for state in self.instance_states.keys():<EOL><INDENT>status_string += "<STR_LIT>".format(<EOL>state, self.instance_states[state]<EOL>)<EOL><DEDENT>status_string += "<STR_LIT:\n>"<EOL>logger.info(status_string)<EOL>return status_string<EOL>

Print human readable summary of current AWS state to log and to console.


self.shut_down_instance(self.instances)<EOL>self.instances = []<EOL>try:<EOL><INDENT>self.client.delete_internet_gateway(InternetGatewayId=self.internet_gateway)<EOL>self.internet_gateway = None<EOL>self.client.delete_route_table(RouteTableId=self.route_table)<EOL>self.route_table = None<EOL>for subnet in list(self.sn_ids):<EOL><INDENT>self.client.delete_subnet(SubnetId=subnet)<EOL>self.sn_ids.remove(subnet)<EOL><DEDENT>self.client.delete_security_group(GroupId=self.sg_id)<EOL>self.sg_id = None<EOL>self.client.delete_vpc(VpcId=self.vpc_id)<EOL>self.vpc_id = None<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.error("<STR_LIT:{}>".format(e))<EOL>raise e<EOL><DEDENT>self.show_summary()<EOL>os.remove(self.config['<STR_LIT>'])<EOL>

Teardown the EC2 infastructure.

        Terminate all EC2 instances, delete all subnets, delete security group, delete VPC,
        and reset all instance variables.


return len(self.instances)<EOL>


job_id_list = '<STR_LIT:U+002C>'.join(self.resources.keys())<EOL>cmd = "<STR_LIT>".format(job_id_list)<EOL>retcode, stdout, stderr = super().execute_wait(cmd)<EOL>if retcode != <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>jobs_missing = list(self.resources.keys())<EOL>for line in stdout.split('<STR_LIT:\n>'):<EOL><INDENT>parts = line.split()<EOL>if parts and parts[<NUM_LIT:0>] != '<STR_LIT>':<EOL><INDENT>job_id = parts[<NUM_LIT:0>]<EOL>status = translate_table.get(parts[<NUM_LIT:4>], '<STR_LIT>')<EOL>self.resources[job_id]['<STR_LIT:status>'] = status<EOL>jobs_missing.remove(job_id)<EOL><DEDENT><DEDENT>for missing_job in jobs_missing:<EOL><INDENT>if self.resources[missing_job]['<STR_LIT:status>'] in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>self.resources[missing_job]['<STR_LIT:status>'] = '<STR_LIT>'<EOL><DEDENT><DEDENT>

Internal: Do not call. Returns the status list for a list of job_ids

        Args:
              self

        Returns:
              [status...] : Status list of all jobs


def submit(self, command, blocksize, tasks_per_node, job_name="<STR_LIT>"):

if self.provisioned_blocks >= self.max_blocks:<EOL><INDENT>logger.warn("<STR_LIT>".format(self.label))<EOL>return None<EOL><DEDENT>job_name = "<STR_LIT>".format(job_name, time.time())<EOL>script_path = "<STR_LIT>".format(self.script_dir, job_name)<EOL>script_path = os.path.abspath(script_path)<EOL>logger.debug("<STR_LIT>".format(self.nodes_per_block))<EOL>job_config = {}<EOL>job_config["<STR_LIT>"] = self.channel.script_dir<EOL>job_config["<STR_LIT>"] = self.nodes_per_block<EOL>job_config["<STR_LIT>"] = tasks_per_node<EOL>job_config["<STR_LIT>"] = wtime_to_minutes(self.walltime)<EOL>job_config["<STR_LIT>"] = self.scheduler_options<EOL>job_config["<STR_LIT>"] = self.worker_init<EOL>job_config["<STR_LIT>"] = self.partition<EOL>job_config["<STR_LIT>"] = command<EOL>job_config["<STR_LIT>"] = self.launcher(command,<EOL>tasks_per_node,<EOL>self.nodes_per_block)<EOL>logger.debug("<STR_LIT>")<EOL>self._write_submit_script(template_string, script_path, job_name, job_config)<EOL>if self.move_files:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>channel_script_path = script_path<EOL><DEDENT>retcode, stdout, stderr = super().execute_wait("<STR_LIT>".format(channel_script_path))<EOL>job_id = None<EOL>if retcode == <NUM_LIT:0>:<EOL><INDENT>for line in stdout.split('<STR_LIT:\n>'):<EOL><INDENT>if line.startswith("<STR_LIT>"):<EOL><INDENT>job_id = line.split("<STR_LIT>")[<NUM_LIT:1>].strip()<EOL>self.resources[job_id] = {'<STR_LIT>': job_id, '<STR_LIT:status>': '<STR_LIT>', '<STR_LIT>': blocksize}<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL>logger.error("<STR_LIT>", retcode, stdout.strip(), stderr.strip())<EOL><DEDENT>return job_id<EOL>

Submit the command as a slurm job of blocksize parallel elements.

        Parameters
        ----------
        command : str
            Command to be made on the remote side.
        blocksize : int
            Not implemented.
        tasks_per_node : int
            Command invocations to be launched per node
        job_name : str
            Name for the job (must be unique).
        Returns
        -------
        None or str
            If at capacity, returns None; otherwise, a string identifier for the job


job_id_list = '<STR_LIT:U+0020>'.join(job_ids)<EOL>retcode, stdout, stderr = super().execute_wait("<STR_LIT>".format(job_id_list))<EOL>rets = None<EOL>if retcode == <NUM_LIT:0>:<EOL><INDENT>for jid in job_ids:<EOL><INDENT>self.resources[jid]['<STR_LIT:status>'] = translate_table['<STR_LIT>']  <EOL><DEDENT>rets = [True for i in job_ids]<EOL><DEDENT>else:<EOL><INDENT>rets = [False for i in job_ids]<EOL><DEDENT>return rets<EOL>

Cancels the jobs specified by a list of job ids

        Args:
        job_ids : [<job_id> ...]

        Returns :
        [True/False...] : If the cancel operation fails the entire list will be False.


def get_configs(self, command, tasks_per_node):

logger.debug("<STR_LIT>".format(<EOL>self.nodes_per_block, tasks_per_node))<EOL>job_config = {}<EOL>job_config["<STR_LIT>"] = self.channel.script_dir<EOL>job_config["<STR_LIT>"] = self.nodes_per_block<EOL>job_config["<STR_LIT>"] = wtime_to_minutes(self.walltime)<EOL>job_config["<STR_LIT>"] = self.scheduler_options<EOL>job_config["<STR_LIT>"] = self.worker_init<EOL>job_config["<STR_LIT>"] = command<EOL>job_config["<STR_LIT>"] = self.launcher(command,<EOL>tasks_per_node,<EOL>self.nodes_per_block)<EOL>return job_config<EOL>

Compose a dictionary with information for writing the submit script.


<EOL>if blocksize < self.nodes_per_block:<EOL><INDENT>blocksize = self.nodes_per_block<EOL><DEDENT>job_name = "<STR_LIT>".format(job_name, time.time())<EOL>script_path = "<STR_LIT>".format(self.script_dir, job_name)<EOL>script_path = os.path.abspath(script_path)<EOL>job_config = self.get_configs(command, tasks_per_node)<EOL>logger.debug("<STR_LIT>")<EOL>self._write_submit_script(template_string, script_path, job_name, job_config)<EOL>channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)<EOL>cmd = "<STR_LIT>".format(channel_script_path)<EOL>retcode, stdout, stderr = super().execute_wait(cmd, <NUM_LIT:10>)<EOL>if retcode == <NUM_LIT:0>:<EOL><INDENT>for line in stdout.split('<STR_LIT:\n>'):<EOL><INDENT>job_id = line.strip()<EOL>if not job_id:<EOL><INDENT>continue<EOL><DEDENT>self.resources[job_id] = {'<STR_LIT>': job_id, '<STR_LIT:status>': '<STR_LIT>', '<STR_LIT>': blocksize}<EOL>return job_id<EOL><DEDENT><DEDENT>else:<EOL><INDENT>print("<STR_LIT>")<EOL>logger.error("<STR_LIT>", retcode, stdout.strip(), stderr.strip())<EOL><DEDENT>

The submit method takes the command string to be executed upon
        instantiation of a resource most often to start a pilot (such as IPP engine
        or even Swift-T engines).

        Args :
             - command (str) : The bash command string to be executed.
             - blocksize (int) : Blocksize to be requested
             - tasks_per_node (int) : command invocations to be launched per node

        KWargs:
             - job_name (str) : Human friendly name to be assigned to the job request

        Returns:
             - A job identifier, this could be an integer, string etc

        Raises:
             - ExecutionProviderException or its subclasses


cmd = "<STR_LIT>"<EOL>retcode, stdout, stderr = super().execute_wait(cmd)<EOL>if retcode != <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>jobs_missing = list(self.resources.keys())<EOL>for line in stdout.split('<STR_LIT:\n>'):<EOL><INDENT>parts = line.split()<EOL>if parts and parts[<NUM_LIT:0>].lower().lower() != '<STR_LIT>'and not parts[<NUM_LIT:0>].startswith('<STR_LIT>'):<EOL><INDENT>job_id = parts[<NUM_LIT:0>]<EOL>status = translate_table.get(parts[<NUM_LIT:4>].lower(), '<STR_LIT>')<EOL>if job_id in self.resources:<EOL><INDENT>self.resources[job_id]['<STR_LIT:status>'] = status<EOL>jobs_missing.remove(job_id)<EOL><DEDENT><DEDENT><DEDENT>for missing_job in jobs_missing:<EOL><INDENT>if self.resources[missing_job]['<STR_LIT:status>'] in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>self.resources[missing_job]['<STR_LIT:status>'] = '<STR_LIT>'<EOL><DEDENT><DEDENT>

Get the status of a list of jobs identified by the job identifiers
        returned from the submit request.

        Returns:
             - A list of status from ['PENDING', 'RUNNING', 'CANCELLED', 'COMPLETED',
               'FAILED', 'TIMEOUT'] corresponding to each job_id in the job_ids list.

        Raises:
             - ExecutionProviderException or its subclasses


job_id_list = '<STR_LIT:U+0020>'.join(job_ids)<EOL>cmd = "<STR_LIT>".format(job_id_list)<EOL>retcode, stdout, stderr = super().execute_wait(cmd, <NUM_LIT:3>)<EOL>rets = None<EOL>if retcode == <NUM_LIT:0>:<EOL><INDENT>for jid in job_ids:<EOL><INDENT>self.resources[jid]['<STR_LIT:status>'] = "<STR_LIT>"<EOL><DEDENT>rets = [True for i in job_ids]<EOL><DEDENT>else:<EOL><INDENT>rets = [False for i in job_ids]<EOL><DEDENT>return rets<EOL>

Cancels the resources identified by the job_ids provided by the user.

        Args:
             - job_ids (list): A list of job identifiers

        Returns:
             - A list of status from cancelling the job which can be True, False

        Raises:
             - ExecutionProviderException or its subclasses


def submit(self, cmd_string, blocksize, tasks_per_node, job_name="<STR_LIT>"):

if not self.resources:<EOL><INDENT>cur_timestamp = str(time.time() * <NUM_LIT:1000>).split("<STR_LIT:.>")[<NUM_LIT:0>]<EOL>job_name = "<STR_LIT>".format(job_name, cur_timestamp)<EOL>if not self.deployment_name:<EOL><INDENT>deployment_name = '<STR_LIT>'.format(job_name)<EOL><DEDENT>else:<EOL><INDENT>deployment_name = '<STR_LIT>'.format(self.deployment_name,<EOL>cur_timestamp)<EOL><DEDENT>formatted_cmd = template_string.format(command=cmd_string,<EOL>worker_init=self.worker_init)<EOL>self.deployment_obj = self._create_deployment_object(job_name,<EOL>self.image,<EOL>deployment_name,<EOL>cmd_string=formatted_cmd,<EOL>replicas=self.init_blocks,<EOL>volumes=self.persistent_volumes)<EOL>logger.debug("<STR_LIT>".format(deployment_name))<EOL>self._create_deployment(self.deployment_obj)<EOL>self.resources[deployment_name] = {'<STR_LIT:status>': '<STR_LIT>',<EOL>'<STR_LIT>': self.init_blocks}<EOL><DEDENT>return deployment_name<EOL>

Submit a job
        Args:
             - cmd_string  :(String) - Name of the container to initiate
             - blocksize   :(float) - Number of replicas
             - tasks_per_node (int) : command invocations to be launched per node

        Kwargs:
             - job_name (String): Name for job, must be unique
        Returns:
             - None: At capacity, cannot provision more
             - job_id: (string) Identifier for the job


self._status()<EOL>return ['<STR_LIT>' for jid in job_ids]<EOL>

Get the status of a list of jobs identified by the job identifiers
        returned from the submit request.
        Args:
             - job_ids (list) : A list of job identifiers
        Returns:
             - A list of status from ['PENDING', 'RUNNING', 'CANCELLED', 'COMPLETED',
               'FAILED', 'TIMEOUT'] corresponding to each job_id in the job_ids list.
        Raises:
             - ExecutionProviderExceptions or its subclasses


for job in job_ids:<EOL><INDENT>logger.debug("<STR_LIT>".format(job))<EOL>self._delete_deployment(job)<EOL>self.resources[job]['<STR_LIT:status>'] = '<STR_LIT>'<EOL><DEDENT>rets = [True for i in job_ids]<EOL>return rets<EOL>

Cancels the jobs specified by a list of job ids
        Args:
        job_ids : [<job_id> ...]
        Returns :
        [True/False...] : If the cancel operation fails the entire list will be False.


jobs_ids = list(self.resources.keys())<EOL>return jobs_ids<EOL>

Internal: Do not call. Returns the status list for a list of job_ids
        Args:
              self
        Returns:
              [status...] : Status list of all jobs


<EOL>security_context = None<EOL>if self.user_id and self.group_id:<EOL><INDENT>security_context = client.V1SecurityContext(run_as_group=self.group_id,<EOL>run_as_user=self.user_id,<EOL>run_as_non_root=self.run_as_non_root)<EOL><DEDENT>environment_vars = client.V1EnvVar(name="<STR_LIT>", value="<STR_LIT>")<EOL>launch_args = ["<STR_LIT:-c>", "<STR_LIT>".format(cmd_string)]<EOL>volume_mounts = []<EOL>for volume in volumes:<EOL><INDENT>volume_mounts.append(client.V1VolumeMount(mount_path=volume[<NUM_LIT:1>],<EOL>name=volume[<NUM_LIT:0>]))<EOL><DEDENT>container = None<EOL>if security_context:<EOL><INDENT>container = client.V1Container(<EOL>name=job_name,<EOL>image=job_image,<EOL>ports=[client.V1ContainerPort(container_port=port)],<EOL>volume_mounts=volume_mounts,<EOL>command=['<STR_LIT>'],<EOL>args=launch_args,<EOL>env=[environment_vars],<EOL>security_context=security_context)<EOL><DEDENT>else:<EOL><INDENT>container = client.V1Container(<EOL>name=job_name,<EOL>image=job_image,<EOL>ports=[client.V1ContainerPort(container_port=port)],<EOL>volume_mounts=volume_mounts,<EOL>command=['<STR_LIT>'],<EOL>args=launch_args,<EOL>env=[environment_vars])<EOL><DEDENT>secret = None<EOL>if self.secret:<EOL><INDENT>secret = client.V1LocalObjectReference(name=self.secret)<EOL><DEDENT>volume_defs = []<EOL>for volume in volumes:<EOL><INDENT>volume_defs.append(client.V1Volume(name=volume[<NUM_LIT:0>],<EOL>persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(<EOL>claim_name=volume[<NUM_LIT:0>])))<EOL><DEDENT>template = client.V1PodTemplateSpec(<EOL>metadata=client.V1ObjectMeta(labels={"<STR_LIT>": job_name}),<EOL>spec=client.V1PodSpec(containers=[container],<EOL>image_pull_secrets=[secret],<EOL>volumes=volume_defs<EOL>))<EOL>spec = client.ExtensionsV1beta1DeploymentSpec(replicas=replicas,<EOL>template=template)<EOL>deployment = client.ExtensionsV1beta1Deployment(<EOL>api_version="<STR_LIT>",<EOL>kind="<STR_LIT>",<EOL>metadata=client.V1ObjectMeta(name=deployment_name),<EOL>spec=spec)<EOL>return deployment<EOL>

Create a kubernetes deployment for the job.
        Args:
              - job_name (string) : Name of the job and deployment
              - job_image (string) : Docker image to launch
        KWargs:
             - port (integer) : Container port
             - replicas : Number of replica containers to maintain
        Returns:
              - True: The deployment object to launch


api_response = self.kube_client.create_namespaced_deployment(<EOL>body=deployment,<EOL>namespace=self.namespace)<EOL>logger.debug("<STR_LIT>".format(str(api_response.status)))<EOL>


job_id_list = '<STR_LIT:U+0020>'.join(self.resources.keys())<EOL>cmd = "<STR_LIT>".format(job_id_list)<EOL>retcode, stdout, stderr = super().execute_wait(cmd)<EOL>"""<STR_LIT>"""<EOL>for line in stdout.strip().split('<STR_LIT:\n>'):<EOL><INDENT>parts = line.split()<EOL>job_id = parts[<NUM_LIT:0>]<EOL>status = translate_table.get(parts[<NUM_LIT:1>], '<STR_LIT>')<EOL>self.resources[job_id]['<STR_LIT:status>'] = status<EOL><DEDENT>

Update the resource dictionary with job statuses.


self._status()<EOL>return [self.resources[jid]['<STR_LIT:status>'] for jid in job_ids]<EOL>

Get the status of a list of jobs identified by their ids.

        Parameters
        ----------
        job_ids : list of int
            Identifiers of jobs for which the status will be returned.

        Returns
        -------
        List of int
            Status codes for the requested jobs.


logger.debug("<STR_LIT>".format(blocksize))<EOL>if self.provisioned_blocks >= self.max_blocks:<EOL><INDENT>template = "<STR_LIT>"<EOL>logger.warn(template.format(self.label, self.provisioned_blocks, self.max_blocks))<EOL>return None<EOL><DEDENT>blocksize = max(self.nodes_per_block, blocksize)<EOL>job_name = "<STR_LIT>".format(job_name, time.time())<EOL>script_path = "<STR_LIT>".format(self.script_dir, job_name)<EOL>script_path = os.path.abspath(script_path)<EOL>userscript_path = "<STR_LIT>".format(self.script_dir, job_name)<EOL>userscript_path = os.path.abspath(userscript_path)<EOL>self.environment["<STR_LIT>"] = "<STR_LIT>".format(job_name)<EOL>job_config = {}<EOL>job_config["<STR_LIT>"] = job_name<EOL>job_config["<STR_LIT>"] = self.channel.script_dir<EOL>job_config["<STR_LIT>"] = self.project<EOL>job_config["<STR_LIT>"] = self.nodes_per_block<EOL>job_config["<STR_LIT>"] = self.scheduler_options<EOL>job_config["<STR_LIT>"] = self.worker_init<EOL>job_config["<STR_LIT>"] = command<EOL>job_config["<STR_LIT>"] = tasks_per_node<EOL>job_config["<STR_LIT>"] = self.requirements<EOL>job_config["<STR_LIT>"] = '<STR_LIT:U+0020>'.join(['<STR_LIT>'.format(key, value) for key, value in self.environment.items()])<EOL>wrapped_command = self.launcher(command,<EOL>tasks_per_node,<EOL>self.nodes_per_block)<EOL>with open(userscript_path, '<STR_LIT:w>') as f:<EOL><INDENT>f.write(job_config["<STR_LIT>"] + '<STR_LIT:\n>' + wrapped_command)<EOL><DEDENT>user_script_path = self.channel.push_file(userscript_path, self.channel.script_dir)<EOL>the_input_files = [user_script_path] + self.transfer_input_files<EOL>job_config["<STR_LIT>"] = '<STR_LIT:U+002C>'.join(the_input_files)<EOL>job_config["<STR_LIT>"] = os.path.basename(user_script_path)<EOL>self._write_submit_script(template_string, script_path, job_name, job_config)<EOL>channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)<EOL>cmd = "<STR_LIT>".format(channel_script_path)<EOL>retcode, stdout, stderr = super().execute_wait(cmd, <NUM_LIT:30>)<EOL>logger.debug("<STR_LIT>", retcode, stdout.strip(), stderr.strip())<EOL>job_id = []<EOL>if retcode == <NUM_LIT:0>:<EOL><INDENT>for line in stdout.split('<STR_LIT:\n>'):<EOL><INDENT>if re.match('<STR_LIT>', line) is not None:<EOL><INDENT>cluster = line.split("<STR_LIT:U+0020>")[<NUM_LIT:5>]<EOL>processes = [str(x) for x in range(<NUM_LIT:0>, int(line[<NUM_LIT:0>]))]<EOL>job_id += [cluster + process for process in processes]<EOL><DEDENT><DEDENT>self._add_resource(job_id)<EOL><DEDENT>return job_id[<NUM_LIT:0>]<EOL>

Submits the command onto an Local Resource Manager job of blocksize parallel elements.

        example file with the complex case of multiple submits per job:
            Universe =vanilla
            output = out.$(Cluster).$(Process)
            error = err.$(Cluster).$(Process)
            log = log.$(Cluster)
            leave_in_queue = true
            executable = test.sh
            queue 5
            executable = foo
            queue 1

        $ condor_submit test.sub
        Submitting job(s)......
        5 job(s) submitted to cluster 118907.
        1 job(s) submitted to cluster 118908.

        Parameters
        ----------
        command : str
            Command to execute
        blocksize : int
            Number of blocks to request.
        job_name : str
            Job name prefix.
        tasks_per_node : int
            command invocations to be launched per node
        Returns
        -------
        None or str
            None if at capacity and cannot provision more; otherwise the identifier for the job.


job_id_list = '<STR_LIT:U+0020>'.join(job_ids)<EOL>cmd = "<STR_LIT>".format(job_id_list)<EOL>logger.debug("<STR_LIT>".format(cmd))<EOL>retcode, stdout, stderr = self.channel.execute_wait(cmd, <NUM_LIT:30>)<EOL>rets = None<EOL>if retcode == <NUM_LIT:0>:<EOL><INDENT>for jid in job_ids:<EOL><INDENT>self.resources[jid]['<STR_LIT:status>'] = '<STR_LIT>'<EOL><DEDENT>rets = [True for i in job_ids]<EOL><DEDENT>else:<EOL><INDENT>rets = [False for i in job_ids]<EOL><DEDENT>return rets<EOL>

Cancels the jobs specified by a list of job IDs.

        Parameters
        ----------
        job_ids : list of str
            The job IDs to cancel.

        Returns
        -------
        list of bool
            Each entry in the list will be True if the job is cancelled succesfully, otherwise False.


logger.debug("<STR_LIT>".format(job_ids))<EOL>for job_id in self.resources:<EOL><INDENT>if self.resources[job_id]['<STR_LIT>']:<EOL><INDENT>poll_code = self.resources[job_id]['<STR_LIT>'].poll()<EOL>if self.resources[job_id]['<STR_LIT:status>'] in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>continue<EOL><DEDENT>if poll_code is None:<EOL><INDENT>self.resources[job_id]['<STR_LIT:status>'] = '<STR_LIT>'<EOL><DEDENT>elif poll_code == <NUM_LIT:0>:<EOL><INDENT>self.resources[job_id]['<STR_LIT:status>'] = '<STR_LIT>'<EOL><DEDENT>elif poll_code != <NUM_LIT:0>:<EOL><INDENT>self.resources[job_id]['<STR_LIT:status>'] = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>logger.error("<STR_LIT>")<EOL><DEDENT><DEDENT>elif self.resources[job_id]['<STR_LIT>']:<EOL><INDENT>retcode, stdout, stderr = self.channel.execute_wait('<STR_LIT>',<EOL>self.cmd_timeout)<EOL>for line in stdout.split('<STR_LIT:\n>'):<EOL><INDENT>if line.startswith("<STR_LIT>"):<EOL><INDENT>status = line.split("<STR_LIT>")[<NUM_LIT:1>].strip()<EOL>if status == "<STR_LIT:0>":<EOL><INDENT>self.resources[job_id]['<STR_LIT:status>'] = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>self.resources[job_id]['<STR_LIT:status>'] = '<STR_LIT>'<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return [self.resources[jid]['<STR_LIT:status>'] for jid in job_ids]<EOL>

Get the status of a list of jobs identified by their ids.

        Args:
            - job_ids (List of ids) : List of identifiers for the jobs

        Returns:
            - List of status codes.


def _write_submit_script(self, script_string, script_filename):

try:<EOL><INDENT>with open(script_filename, '<STR_LIT:w>') as f:<EOL><INDENT>f.write(script_string)<EOL><DEDENT><DEDENT>except KeyError as e:<EOL><INDENT>logger.error("<STR_LIT>", e)<EOL>raise (SchedulerMissingArgs(e.args, self.label))<EOL><DEDENT>except IOError as e:<EOL><INDENT>logger.error("<STR_LIT>", script_filename)<EOL>raise (ScriptPathError(script_filename, e))<EOL><DEDENT>return True<EOL>

Load the template string with config values and write the generated submit script to
a submit script file.

Args:
      - template_string (string) : The template string to be used for the writing submit script
      - script_filename (string) : Name of the submit script

Returns:
      - True: on success

Raises:
      SchedulerMissingArgs : If template is missing args
      ScriptPathError : Unable to write submit script out


job_name = "<STR_LIT>".format(job_name, time.time())<EOL>script_path = "<STR_LIT>".format(self.script_dir, job_name)<EOL>script_path = os.path.abspath(script_path)<EOL>wrap_command = self.worker_init + '<STR_LIT:\n>' + self.launcher(command, tasks_per_node, self.nodes_per_block)<EOL>self._write_submit_script(wrap_command, script_path)<EOL>job_id = None<EOL>proc = None<EOL>remote_pid = None<EOL>if (self.move_files is None and not isinstance(self.channel, LocalChannel)) or (self.move_files):<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>script_path = self.channel.push_file(script_path, self.channel.script_dir)<EOL><DEDENT>if not isinstance(self.channel, LocalChannel):<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>cmd = '<STR_LIT>'.format(script_path)<EOL>retcode, stdout, stderr = self.channel.execute_wait(cmd, self.cmd_timeout)<EOL>for line in stdout.split('<STR_LIT:\n>'):<EOL><INDENT>if line.startswith("<STR_LIT>"):<EOL><INDENT>remote_pid = line.split("<STR_LIT>")[<NUM_LIT:1>].strip()<EOL>job_id = remote_pid<EOL><DEDENT><DEDENT>if job_id is None:<EOL><INDENT>logger.warning("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>job_id, proc = self.channel.execute_no_wait('<STR_LIT>'.format(script_path), self.cmd_timeout)<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.debug("<STR_LIT>".format(self.channel, e))<EOL>raise<EOL><DEDENT><DEDENT>self.resources[job_id] = {'<STR_LIT>': job_id, '<STR_LIT:status>': '<STR_LIT>',<EOL>'<STR_LIT>': blocksize,<EOL>'<STR_LIT>': remote_pid,<EOL>'<STR_LIT>': proc}<EOL>return job_id<EOL>

Submits the command onto an Local Resource Manager job of blocksize parallel elements.
        Submit returns an ID that corresponds to the task that was just submitted.

        If tasks_per_node <  1:
             1/tasks_per_node is provisioned

        If tasks_per_node == 1:
             A single node is provisioned

        If tasks_per_node >  1 :
             tasks_per_node * blocksize number of nodes are provisioned.

        Args:
             - command  :(String) Commandline invocation to be made on the remote side.
             - blocksize   :(float) - Not really used for local
             - tasks_per_node (int) : command invocations to be launched per node

        Kwargs:
             - job_name (String): Name for job, must be unique

        Returns:
             - None: At capacity, cannot provision more
             - job_id: (string) Identifier for the job


for job in job_ids:<EOL><INDENT>logger.debug("<STR_LIT>".format(job))<EOL>if self.resources[job]['<STR_LIT>']:<EOL><INDENT>proc = self.resources[job]['<STR_LIT>']<EOL>os.killpg(os.getpgid(proc.pid), signal.SIGTERM)<EOL>self.resources[job]['<STR_LIT:status>'] = '<STR_LIT>'<EOL><DEDENT>elif self.resources[job]['<STR_LIT>']:<EOL><INDENT>cmd = "<STR_LIT>".format(self.resources[job]['<STR_LIT>'])<EOL>retcode, stdout, stderr = self.channel.execute_wait(cmd, self.cmd_timeout)<EOL>if retcode != <NUM_LIT:0>:<EOL><INDENT>logger.warning("<STR_LIT>".format(self.resources[job]['<STR_LIT>'],<EOL>self.label))<EOL><DEDENT><DEDENT><DEDENT>rets = [True for i in job_ids]<EOL>return rets<EOL>

Cancels the jobs specified by a list of job ids

        Args:
        job_ids : [<job_id> ...]

        Returns :
        [True/False...] : If the cancel operation fails the entire list will be False.


wrapped_cmd = self.launcher(command,<EOL>tasks_per_node,<EOL><NUM_LIT:1>)<EOL>instance, name = self.create_instance(command=wrapped_cmd)<EOL>self.provisioned_blocks += <NUM_LIT:1><EOL>self.resources[name] = {"<STR_LIT>": name, "<STR_LIT:status>": translate_table[instance['<STR_LIT:status>']]}<EOL>return name<EOL>

The submit method takes the command string to be executed upon
        instantiation of a resource most often to start a pilot.

        Args :
             - command (str) : The bash command string to be executed.
             - blocksize (int) : Blocksize to be requested
             - tasks_per_node (int) : command invocations to be launched per node

        KWargs:
             - job_name (str) : Human friendly name to be assigned to the job request

        Returns:
             - A job identifier, this could be an integer, string etc

        Raises:
             - ExecutionProviderException or its subclasses


statuses = []<EOL>for job_id in job_ids:<EOL><INDENT>instance = self.client.instances().get(instance=job_id, project=self.project_id, zone=self.zone).execute()<EOL>self.resources[job_id]['<STR_LIT:status>'] = translate_table[instance['<STR_LIT:status>']]<EOL>statuses.append(translate_table[instance['<STR_LIT:status>']])<EOL><DEDENT>return statuses<EOL>

Get the status of a list of jobs identified by the job identifiers
        returned from the submit request.

        Args:
             - job_ids (list) : A list of job identifiers

        Returns:
             - A list of status from ['PENDING', 'RUNNING', 'CANCELLED', 'COMPLETED',
               'FAILED', 'TIMEOUT'] corresponding to each job_id in the job_ids list.

        Raises:
             - ExecutionProviderException or its subclasses


statuses = []<EOL>for job_id in job_ids:<EOL><INDENT>try:<EOL><INDENT>self.delete_instance(job_id)<EOL>statuses.append(True)<EOL>self.provisioned_blocks -= <NUM_LIT:1><EOL><DEDENT>except Exception:<EOL><INDENT>statuses.append(False)<EOL><DEDENT><DEDENT>return statuses<EOL>


return True<EOL>

Scaling is enabled

        Returns:
              - Status (Bool)


return self.provisioned_blocks<EOL>

Returns the number of currently provisioned blocks.


job_id_list = '<STR_LIT:U+0020>'.join(self.resources.keys())<EOL>jobs_missing = list(self.resources.keys())<EOL>retcode, stdout, stderr = super().execute_wait("<STR_LIT>".format(job_id_list))<EOL>for line in stdout.split('<STR_LIT:\n>'):<EOL><INDENT>parts = line.split()<EOL>if not parts or parts[<NUM_LIT:0>].upper().startswith('<STR_LIT>') or parts[<NUM_LIT:0>].startswith('<STR_LIT>'):<EOL><INDENT>continue<EOL><DEDENT>job_id = parts[<NUM_LIT:0>]<EOL>status = translate_table.get(parts[<NUM_LIT:4>], '<STR_LIT>')<EOL>self.resources[job_id]['<STR_LIT:status>'] = status<EOL>jobs_missing.remove(job_id)<EOL><DEDENT>for missing_job in jobs_missing:<EOL><INDENT>if self.resources[missing_job]['<STR_LIT:status>'] in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>self.resources[missing_job]['<STR_LIT:status>'] = translate_table['<STR_LIT:E>']<EOL><DEDENT><DEDENT>

Internal: Do not call. Returns the status list for a list of job_ids

        Args:
              self

        Returns:
              [status...] : Status list of all jobs


if self.provisioned_blocks >= self.max_blocks:<EOL><INDENT>logger.warn("<STR_LIT>", self.label)<EOL>return None<EOL><DEDENT>if blocksize < self.nodes_per_block:<EOL><INDENT>blocksize = self.nodes_per_block<EOL><DEDENT>job_name = "<STR_LIT>".format(job_name, time.time())<EOL>script_path = "<STR_LIT>".format(self.script_dir, job_name)<EOL>script_path = os.path.abspath(script_path)<EOL>logger.debug("<STR_LIT>", blocksize, self.nodes_per_block,<EOL>tasks_per_node)<EOL>job_config = {}<EOL>job_config["<STR_LIT>"] = self.channel.script_dir<EOL>job_config["<STR_LIT>"] = self.nodes_per_block<EOL>job_config["<STR_LIT>"] = self.nodes_per_block * tasks_per_node<EOL>job_config["<STR_LIT>"] = self.nodes_per_block<EOL>job_config["<STR_LIT>"] = tasks_per_node<EOL>job_config["<STR_LIT>"] = self.walltime<EOL>job_config["<STR_LIT>"] = self.scheduler_options<EOL>job_config["<STR_LIT>"] = self.worker_init<EOL>job_config["<STR_LIT>"] = command<EOL>job_config["<STR_LIT>"] = self.launcher(command,<EOL>tasks_per_node,<EOL>self.nodes_per_block)<EOL>logger.debug("<STR_LIT>")<EOL>self._write_submit_script(template_string, script_path, job_name, job_config)<EOL>channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)<EOL>submit_options = '<STR_LIT>'<EOL>if self.queue is not None:<EOL><INDENT>submit_options = '<STR_LIT>'.format(submit_options, self.queue)<EOL><DEDENT>if self.account is not None:<EOL><INDENT>submit_options = '<STR_LIT>'.format(submit_options, self.account)<EOL><DEDENT>launch_cmd = "<STR_LIT>".format(submit_options, channel_script_path)<EOL>retcode, stdout, stderr = super().execute_wait(launch_cmd)<EOL>job_id = None<EOL>if retcode == <NUM_LIT:0>:<EOL><INDENT>for line in stdout.split('<STR_LIT:\n>'):<EOL><INDENT>if line.strip():<EOL><INDENT>job_id = line.strip()<EOL>self.resources[job_id] = {'<STR_LIT>': job_id, '<STR_LIT:status>': '<STR_LIT>', '<STR_LIT>': blocksize}<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>message = "<STR_LIT>".format(launch_cmd, retcode)<EOL>if (stdout is not None) and (stderr is not None):<EOL><INDENT>message += "<STR_LIT>".format(stderr.strip(), stdout.strip())<EOL><DEDENT>logger.error(message)<EOL><DEDENT>return job_id<EOL>

Submits the command onto an Local Resource Manager job of blocksize parallel elements.
        Submit returns an ID that corresponds to the task that was just submitted.

        If tasks_per_node <  1 : ! This is illegal. tasks_per_node should be integer

        If tasks_per_node == 1:
             A single node is provisioned

        If tasks_per_node >  1 :
             tasks_per_node * blocksize number of nodes are provisioned.

        Args:
             - command  :(String) Commandline invocation to be made on the remote side.
             - blocksize   :(float)
             - tasks_per_node (int) : command invocations to be launched per node

        Kwargs:
             - job_name (String): Name for job, must be unique

        Returns:
             - None: At capacity, cannot provision more
             - job_id: (string) Identifier for the job


job_id_list = '<STR_LIT:U+0020>'.join(job_ids)<EOL>retcode, stdout, stderr = super().execute_wait("<STR_LIT>".format(job_id_list))<EOL>rets = None<EOL>if retcode == <NUM_LIT:0>:<EOL><INDENT>for jid in job_ids:<EOL><INDENT>self.resources[jid]['<STR_LIT:status>'] = translate_table['<STR_LIT:E>']  <EOL><DEDENT>rets = [True for i in job_ids]<EOL><DEDENT>else:<EOL><INDENT>rets = [False for i in job_ids]<EOL><DEDENT>return rets<EOL>


def scale_out(self, blocks=<NUM_LIT:1>, block_size=<NUM_LIT:1>):

self.config['<STR_LIT>'.format(self.pool)]['<STR_LIT>']<EOL>count = <NUM_LIT:0><EOL>if blocks == <NUM_LIT:1>:<EOL><INDENT>block_id = len(self.blocks)<EOL>self.blocks[block_id] = []<EOL>for instance_id in range(<NUM_LIT:0>, block_size):<EOL><INDENT>instances = self.server_manager.create(<EOL>'<STR_LIT>'.format(block_id, instance_id),  <EOL>self.client.images.get('<STR_LIT>'),  <EOL>self.client.flavors.list()[<NUM_LIT:0>],<EOL>min_count=<NUM_LIT:1>,<EOL>max_count=<NUM_LIT:1>,<EOL>userdata=setup_script.format(engine_config=self.engine_config),<EOL>key_name='<STR_LIT>',<EOL>security_groups=['<STR_LIT>'],<EOL>nics=[{<EOL>"<STR_LIT>": '<STR_LIT>',<EOL>"<STR_LIT>": '<STR_LIT>',<EOL>"<STR_LIT>": '<STR_LIT>'<EOL>}])<EOL>self.blocks[block_id].extend([instances])<EOL>count += <NUM_LIT:1><EOL><DEDENT><DEDENT>return count<EOL>


def scale_in(self, blocks=<NUM_LIT:0>, machines=<NUM_LIT:0>, strategy=None):

count = <NUM_LIT:0><EOL>instances = self.client.servers.list()<EOL>for instance in instances[<NUM_LIT:0>:machines]:<EOL><INDENT>print("<STR_LIT>", instance)<EOL>instance.delete()<EOL>count += <NUM_LIT:1><EOL><DEDENT>return count<EOL>


@abstractmethod<EOL><INDENT>def submit(self, command, blocksize, tasks_per_node, job_name="<STR_LIT>"):<DEDENT>

pass<EOL>

The submit method takes the command string to be executed upon
        instantiation of a resource most often to start a pilot (such as IPP engine
        or even Swift-T engines).

        Args :
             - command (str) : The bash command string to be executed
             - blocksize (int) : Blocksize to be requested
             - tasks_per_node (int) : command invocations to be launched per node

        KWargs:
             - job_name (str) : Human friendly name to be assigned to the job request

        Returns:
             - A job identifier, this could be an integer, string etc

        Raises:
             - ExecutionProviderException or its subclasses


pass<EOL>


pass<EOL>

Cancels the resources identified by the job_ids provided by the user.

        Args:
             - job_ids (list): A list of job identifiers

        Returns:
             - A list of status from cancelling the job which can be True, False

        Raises:
             - ExecutionProviderException or its subclasses


pass<EOL>

The callers of ParslExecutors need to differentiate between Executors
        and Executors wrapped in a resource provider

        Returns:
              - Status (Bool)


def _write_submit_script(self, template, script_filename, job_name, configs):

try:<EOL><INDENT>submit_script = Template(template).substitute(jobname=job_name, **configs)<EOL>with open(script_filename, '<STR_LIT:w>') as f:<EOL><INDENT>f.write(submit_script)<EOL><DEDENT><DEDENT>except KeyError as e:<EOL><INDENT>logger.error("<STR_LIT>", e)<EOL>raise (SchedulerMissingArgs(e.args, self.sitename))<EOL><DEDENT>except IOError as e:<EOL><INDENT>logger.error("<STR_LIT>", script_filename)<EOL>raise (ScriptPathError(script_filename, e))<EOL><DEDENT>except Exception as e:<EOL><INDENT>print("<STR_LIT>", template)<EOL>print("<STR_LIT>", job_name)<EOL>print("<STR_LIT>", configs)<EOL>logger.error("<STR_LIT>", e)<EOL>raise (e)<EOL><DEDENT>return True<EOL>

Generate submit script and write it to a file.

        Args:
              - template (string) : The template string to be used for the writing submit script
              - script_filename (string) : Name of the submit script
              - job_name (string) : job name
              - configs (dict) : configs that get pushed into the template

        Returns:
              - True: on success

        Raises:
              SchedulerMissingArgs : If template is missing args
              ScriptPathError : Unable to write submit script out


raise NotImplementedError<EOL>

The submit method takes the command string to be executed upon
        instantiation of a resource most often to start a pilot (such as IPP engine
        or even Swift-T engines).

        Args :
             - cmd_string (str) : The bash command string to be executed
             - blocksize (int) : Blocksize to be requested
             - tasks_per_node (int) : command invocations to be launched per node

        KWargs:
             - job_name (str) : Human friendly name to be assigned to the job request

        Returns:
             - A job identifier, this could be an integer, string etc

        Raises:
             - ExecutionProviderExceptions or its subclasses


if job_ids:<EOL><INDENT>self._status()<EOL><DEDENT>return [self.resources[jid]['<STR_LIT:status>'] for jid in job_ids]<EOL>


raise NotImplementedError<EOL>


return self._scaling_enabled<EOL>


return self.provisioned_blocks<EOL>

Returns the currently provisioned blocks.
        This may need to return more information in the futures :
        { minsize, maxsize, current_requested }


jobs_missing = list(self.resources.keys())<EOL>retcode, stdout, stderr = super().execute_wait("<STR_LIT>")<EOL>if retcode != <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>for line in stdout.split('<STR_LIT:\n>'):<EOL><INDENT>if line.startswith('<STR_LIT:=>'):<EOL><INDENT>continue<EOL><DEDENT>parts = line.upper().split()<EOL>if parts and parts[<NUM_LIT:0>] != '<STR_LIT>':<EOL><INDENT>job_id = parts[<NUM_LIT:0>]<EOL>if job_id not in self.resources:<EOL><INDENT>continue<EOL><DEDENT>status = translate_table.get(parts[<NUM_LIT:4>], '<STR_LIT>')<EOL>self.resources[job_id]['<STR_LIT:status>'] = status<EOL>jobs_missing.remove(job_id)<EOL><DEDENT><DEDENT>for missing_job in jobs_missing:<EOL><INDENT>if self.resources[missing_job]['<STR_LIT:status>'] in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>self.resources[missing_job]['<STR_LIT:status>'] = translate_table['<STR_LIT>']<EOL><DEDENT><DEDENT>


if self.provisioned_blocks >= self.max_blocks:<EOL><INDENT>logger.warn("<STR_LIT>", self.label)<EOL>return None<EOL><DEDENT>if blocksize < self.nodes_per_block:<EOL><INDENT>blocksize = self.nodes_per_block<EOL><DEDENT>account_opt = '<STR_LIT>'.format(self.account) if self.account is not None else '<STR_LIT>'<EOL>job_name = "<STR_LIT>".format(job_name, time.time())<EOL>script_path = "<STR_LIT>".format(self.script_dir, job_name)<EOL>script_path = os.path.abspath(script_path)<EOL>job_config = {}<EOL>job_config["<STR_LIT>"] = self.scheduler_options<EOL>job_config["<STR_LIT>"] = self.worker_init<EOL>logger.debug("<STR_LIT>",<EOL>blocksize, self.nodes_per_block, tasks_per_node)<EOL>job_config["<STR_LIT>"] = self.launcher(command, tasks_per_node, self.nodes_per_block)<EOL>queue_opt = '<STR_LIT>'.format(self.queue) if self.queue is not None else '<STR_LIT>'<EOL>logger.debug("<STR_LIT>")<EOL>self._write_submit_script(template_string, script_path, job_name, job_config)<EOL>channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)<EOL>command = '<STR_LIT>'.format(<EOL>self.nodes_per_block, queue_opt, wtime_to_minutes(self.walltime), account_opt, channel_script_path)<EOL>logger.debug("<STR_LIT>".format(command))<EOL>retcode, stdout, stderr = super().execute_wait(command)<EOL>if retcode != <NUM_LIT:0>:<EOL><INDENT>logger.error("<STR_LIT>".format(command))<EOL>logger.error("<STR_LIT>".format(stdout, stderr))<EOL><DEDENT>logger.debug("<STR_LIT>", retcode, stdout.strip(), stderr.strip())<EOL>job_id = None<EOL>if retcode == <NUM_LIT:0>:<EOL><INDENT>job_id = stdout.strip()<EOL>self.resources[job_id] = {'<STR_LIT>': job_id, '<STR_LIT:status>': '<STR_LIT>', '<STR_LIT>': blocksize}<EOL><DEDENT>else:<EOL><INDENT>logger.error("<STR_LIT>".format(stderr))<EOL>raise (ScaleOutFailed(self.__class__, "<STR_LIT>"))<EOL><DEDENT>logger.debug("<STR_LIT>".format(job_id))<EOL>return job_id<EOL>


job_id_list = '<STR_LIT:U+0020>'.join(job_ids)<EOL>retcode, stdout, stderr = super().execute_wait("<STR_LIT>".format(job_id_list))<EOL>rets = None<EOL>if retcode == <NUM_LIT:0>:<EOL><INDENT>for jid in job_ids:<EOL><INDENT>self.resources[jid]['<STR_LIT:status>'] = translate_table['<STR_LIT>']  <EOL><DEDENT>rets = [True for i in job_ids]<EOL><DEDENT>else:<EOL><INDENT>rets = [False for i in job_ids]<EOL><DEDENT>return rets<EOL>


return self.executor.submit(*args, **kwargs)<EOL>

Submits work to the thread pool.

        This method is simply pass through and behaves like a submit call as described
        here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_


raise NotImplementedError<EOL>

Scales out the number of active workers by 1.

        This method is notImplemented for threads and will raise the error if called.

        Raises:
             NotImplemented exception


raise NotImplementedError<EOL>

Scale in the number of active blocks by specified amount.

        This method is not implemented for threads and will raise the error if called.

        Raises:
             NotImplemented exception


x = self.executor.shutdown(wait=block)<EOL>logger.debug("<STR_LIT>")<EOL>return x<EOL>

Shutdown the ThreadPool.

        Kwargs:
            - block (Bool): To block for confirmations or not


if self.mode == "<STR_LIT>":<EOL><INDENT>return<EOL><DEDENT>if self.ipython_dir != '<STR_LIT>':<EOL><INDENT>self.ipython_dir = os.path.abspath(os.path.expanduser(self.ipython_dir))<EOL><DEDENT>if self.log:<EOL><INDENT>stdout = open(os.path.join(self.ipython_dir, "<STR_LIT>".format(self.profile)), '<STR_LIT:w>')<EOL>stderr = open(os.path.join(self.ipython_dir, "<STR_LIT>".format(self.profile)), '<STR_LIT:w>')<EOL><DEDENT>else:<EOL><INDENT>stdout = open(os.devnull, '<STR_LIT:w>')<EOL>stderr = open(os.devnull, '<STR_LIT:w>')<EOL><DEDENT>try:<EOL><INDENT>opts = [<EOL>'<STR_LIT>',<EOL>'<STR_LIT>' if self.ipython_dir == '<STR_LIT>' else '<STR_LIT>'.format(self.ipython_dir),<EOL>self.interfaces if self.interfaces is not None else '<STR_LIT>',<EOL>'<STR_LIT>' if self.profile == '<STR_LIT:default>' else '<STR_LIT>'.format(self.profile),<EOL>'<STR_LIT>' if self.reuse else '<STR_LIT>',<EOL>'<STR_LIT>'.format(self.public_ip) if self.public_ip else '<STR_LIT>',<EOL>'<STR_LIT>'.format(self.port) if self.port is not None else '<STR_LIT>'<EOL>]<EOL>if self.port_range is not None:<EOL><INDENT>opts += [<EOL>'<STR_LIT>'.format(self.hb_ping, self.hb_pong),<EOL>'<STR_LIT>'.format(self.control_client, self.control_engine),<EOL>'<STR_LIT>'.format(self.mux_client, self.mux_engine),<EOL>'<STR_LIT>'.format(self.task_client, self.task_engine)<EOL>]<EOL><DEDENT>logger.debug("<STR_LIT>".format('<STR_LIT:U+0020>'.join([str(x) for x in opts])))<EOL>self.proc = subprocess.Popen(opts, stdout=stdout, stderr=stderr, preexec_fn=os.setsid)<EOL><DEDENT>except FileNotFoundError:<EOL><INDENT>msg = "<STR_LIT>"<EOL>logger.error(msg)<EOL>raise ControllerError(msg)<EOL><DEDENT>except Exception as e:<EOL><INDENT>msg = "<STR_LIT>".format(e)<EOL>logger.error(msg)<EOL>raise ControllerError(msg)<EOL><DEDENT>


return os.path.join(self.ipython_dir,<EOL>'<STR_LIT>'.format(self.profile),<EOL>'<STR_LIT>')<EOL>

Specify path to the ipcontroller-engine.json file.

        This file is stored in in the ipython_dir/profile folders.

        Returns :
              - str, File path to engine file


Specify path to the ipcontroller-client.json file.

        This file is stored in in the ipython_dir/profile folders.

        Returns :
              - str, File path to client file


if self.reuse:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return<EOL><DEDENT>if self.mode == "<STR_LIT>":<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return<EOL><DEDENT>try:<EOL><INDENT>pgid = os.getpgid(self.proc.pid)<EOL>os.killpg(pgid, signal.SIGTERM)<EOL>time.sleep(<NUM_LIT>)<EOL>os.killpg(pgid, signal.SIGKILL)<EOL>try:<EOL><INDENT>self.proc.wait(timeout=<NUM_LIT:1>)<EOL>x = self.proc.returncode<EOL>if x == <NUM_LIT:0>:<EOL><INDENT>logger.debug("<STR_LIT>".format(x))<EOL><DEDENT>else:<EOL><INDENT>logger.error("<STR_LIT>".format(x))<EOL><DEDENT><DEDENT>except subprocess.TimeoutExpired:<EOL><INDENT>logger.warn("<STR_LIT>".format(self.proc.pid))<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>logger.warn("<STR_LIT>".format(self.proc.pid, e))<EOL><DEDENT>

Terminate the controller process and its child processes.

        Args:
              - None


def __init__(self, ip_address, port_range):

self.context = zmq.Context()<EOL>self.zmq_socket = self.context.socket(zmq.DEALER)<EOL>self.zmq_socket.set_hwm(<NUM_LIT:0>)<EOL>self.port = self.zmq_socket.bind_to_random_port("<STR_LIT>".format(ip_address),<EOL>min_port=port_range[<NUM_LIT:0>],<EOL>max_port=port_range[<NUM_LIT:1>])<EOL>self.poller = zmq.Poller()<EOL>self.poller.register(self.zmq_socket, zmq.POLLOUT)<EOL>


task_id_bytes = task_id.to_bytes(<NUM_LIT:4>, "<STR_LIT>")<EOL>message = [b"<STR_LIT>", task_id_bytes] + buffer<EOL>self.zmq_socket.send_multipart(message)<EOL>logger.debug("<STR_LIT>".format(task_id))<EOL>


self.context = zmq.Context()<EOL>self.zmq_socket = self.context.socket(zmq.DEALER)<EOL>self.zmq_socket.set_hwm(<NUM_LIT:0>)<EOL>self.port = self.zmq_socket.bind_to_random_port(<EOL>"<STR_LIT>".format(ip_address),<EOL>min_port=port_range[<NUM_LIT:0>],<EOL>max_port=port_range[<NUM_LIT:1>])<EOL>


def execute_task(f, args, kwargs, user_ns):

fname = getattr(f, '<STR_LIT>', '<STR_LIT:f>')<EOL>prefix = "<STR_LIT>"<EOL>fname = prefix + "<STR_LIT:f>"<EOL>argname = prefix + "<STR_LIT:args>"<EOL>kwargname = prefix + "<STR_LIT>"<EOL>resultname = prefix + "<STR_LIT:result>"<EOL>user_ns.update({fname: f,<EOL>argname: args,<EOL>kwargname: kwargs,<EOL>resultname: resultname})<EOL>code = "<STR_LIT>".format(resultname, fname,<EOL>argname, kwargname)<EOL>try:<EOL><INDENT>exec(code, user_ns, user_ns)<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.warning("<STR_LIT>".format(e))<EOL>raise e<EOL><DEDENT>else:<EOL><INDENT>return user_ns.get(resultname)<EOL><DEDENT>

Deserialize the buffer and execute the task.

# Returns the result or exception.


def start_file_logger(filename, rank, name='<STR_LIT>', level=logging.DEBUG, format_string=None):

try:<EOL><INDENT>os.makedirs(os.path.dirname(filename), <NUM_LIT>, True)<EOL><DEDENT>except Exception as e:<EOL><INDENT>print("<STR_LIT>".format(e))<EOL><DEDENT>if format_string is None:<EOL><INDENT>format_string = "<STR_LIT>".format(<EOL>rank)<EOL><DEDENT>global logger<EOL>logger = logging.getLogger(name)<EOL>logger.setLevel(logging.DEBUG)<EOL>handler = logging.FileHandler(filename)<EOL>handler.setLevel(level)<EOL>formatter = logging.Formatter(format_string, datefmt='<STR_LIT>')<EOL>handler.setFormatter(formatter)<EOL>logger.addHandler(handler)<EOL>

Add a stream log handler.

    Args:
        - filename (string): Name of the file to write logs to
        - name (string): Logger name
        - level (logging.LEVEL): Set the logging level.
        - format_string (string): Set the format string

    Returns:
       -  None


def worker(worker_id, task_url, debug=True, logdir="<STR_LIT>", uid="<STR_LIT:1>"):

start_file_logger('<STR_LIT>'.format(logdir, uid, worker_id),<EOL><NUM_LIT:0>,<EOL>level=logging.DEBUG if debug is True else logging.INFO)<EOL>logger.info("<STR_LIT>".format(worker_id))<EOL>task_ids_received = []<EOL>message_q = zmq_pipes.WorkerMessages(task_url)<EOL>while True:<EOL><INDENT>print("<STR_LIT>")<EOL>task_id, buf = message_q.get()<EOL>task_ids_received.append(task_id)<EOL>user_ns = locals()<EOL>user_ns.update({'<STR_LIT>': __builtins__})<EOL>f, args, kwargs = unpack_apply_message(buf, user_ns, copy=False)<EOL>logger.debug("<STR_LIT>".format(worker_id, task_id))<EOL>result = execute_task(f, args, kwargs, user_ns)<EOL>logger.debug("<STR_LIT>".format(worker_id, task_id))<EOL>reply = {"<STR_LIT:result>": result, "<STR_LIT>": worker_id}<EOL>message_q.put(task_id, serialize_object(reply))<EOL>logger.debug("<STR_LIT>")<EOL><DEDENT>

TODO: docstring

    TODO : Cleanup debug, logdir and uid to function correctly


def start_file_logger(filename, name='<STR_LIT>', level=logging.DEBUG, format_string=None):

if format_string is None:<EOL><INDENT>format_string = "<STR_LIT>"<EOL><DEDENT>global logger<EOL>logger = logging.getLogger(name)<EOL>logger.setLevel(level)<EOL>handler = logging.FileHandler(filename)<EOL>handler.setLevel(level)<EOL>formatter = logging.Formatter(format_string, datefmt='<STR_LIT>')<EOL>handler.setFormatter(formatter)<EOL>logger.addHandler(handler)<EOL>

Add a stream log handler.

    Parameters
    ---------

    filename: string
        Name of the file to write logs to. Required.
    name: string
        Logger name. Default="parsl.executors.interchange"
    level: logging.LEVEL
        Set the logging level. Default=logging.DEBUG
        - format_string (string): Set the format string
    format_string: string
        Format string to use.

    Returns
    -------
        None.


<EOL>ic = Interchange(*args, **kwargs)<EOL>comm_q.put(ic.worker_port)<EOL>ic.start()<EOL>logger.debug("<STR_LIT>")<EOL>

Start the interchange process

    The executor is expected to call this function. The args, kwargs match that of the Interchange.__init__


logger.info("<STR_LIT>")<EOL>while True:<EOL><INDENT>socks = dict(self.poller.poll(<NUM_LIT:1>))<EOL>if socks.get(self.task_incoming) == zmq.POLLIN:<EOL><INDENT>message = self.task_incoming.recv_multipart()<EOL>logger.debug("<STR_LIT>")<EOL>self.worker_messages.send_multipart(message)<EOL>logger.debug("<STR_LIT>")<EOL><DEDENT>if socks.get(self.worker_messages) == zmq.POLLIN:<EOL><INDENT>message = self.worker_messages.recv_multipart()<EOL>logger.debug("<STR_LIT>")<EOL>self.result_outgoing.send_multipart(message[<NUM_LIT:1>:])<EOL>logger.debug("<STR_LIT>")<EOL><DEDENT><DEDENT>


self.outgoing_q = zmq_pipes.TasksOutgoing(<EOL>"<STR_LIT:127.0.0.1>", self.interchange_port_range)<EOL>self.incoming_q = zmq_pipes.ResultsIncoming(<EOL>"<STR_LIT:127.0.0.1>", self.interchange_port_range)<EOL>self.is_alive = True<EOL>self._queue_management_thread = None<EOL>self._start_queue_management_thread()<EOL>self._start_local_queue_process()<EOL>logger.debug("<STR_LIT>"<EOL>.format(self._queue_management_thread))<EOL>if self.provider:<EOL><INDENT>l_cmd = self.launch_cmd.format(  <EOL>task_url=self.worker_task_url,<EOL>workers_per_node=self.workers_per_node,<EOL>logdir="<STR_LIT>".format(self.run_dir, self.label))<EOL>self.launch_cmd = l_cmd<EOL>logger.debug("<STR_LIT>".format(self.launch_cmd))<EOL>self._scaling_enabled = self.provider.scaling_enabled<EOL>logger.debug(<EOL>"<STR_LIT>", self.provider)<EOL>if hasattr(self.provider, '<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>for i in range(self.provider.init_blocks):<EOL><INDENT>block = self.provider.submit(<EOL>self.launch_cmd, <NUM_LIT:1>, self.workers_per_node)<EOL>logger.debug("<STR_LIT>".format(i, block))<EOL>if not block:<EOL><INDENT>raise(ScalingFailed(self.provider.label,<EOL>"<STR_LIT>"))<EOL><DEDENT>self.blocks.extend([block])<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>logger.error("<STR_LIT>".format(e))<EOL>raise e<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>self._scaling_enabled = False<EOL>logger.debug("<STR_LIT>")<EOL><DEDENT>

Create the Interchange process and connect to it.


comm_q = Queue(maxsize=<NUM_LIT:10>)<EOL>self.queue_proc = Process(target=interchange.starter,<EOL>args=(comm_q,),<EOL>kwargs={"<STR_LIT>": (self.outgoing_q.port,<EOL>self.incoming_q.port),<EOL>"<STR_LIT>": self.worker_port,<EOL>"<STR_LIT>": self.worker_port_range<EOL>})<EOL>self.queue_proc.start()<EOL>try:<EOL><INDENT>worker_port = comm_q.get(block=True, timeout=<NUM_LIT>)<EOL>logger.debug(<EOL>"<STR_LIT>".format(worker_port))<EOL><DEDENT>except queue.Empty:<EOL><INDENT>logger.error(<EOL>"<STR_LIT>")<EOL>raise Exception("<STR_LIT>")<EOL><DEDENT>self.worker_task_url = "<STR_LIT>".format(<EOL>self.address, worker_port)<EOL>


if self._queue_management_thread is None:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>self._queue_management_thread = threading.Thread(<EOL>target=self._queue_management_worker)<EOL>self._queue_management_thread.daemon = True<EOL>self._queue_management_thread.start()<EOL>logger.debug("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL><DEDENT>


logger.debug("<STR_LIT>")<EOL>while True:<EOL><INDENT>task_id, buf = self.incoming_q.get()  <EOL>msg = deserialize_object(buf)[<NUM_LIT:0>]<EOL>task_fut = self.tasks[task_id]<EOL>logger.debug("<STR_LIT>".format(task_id))<EOL>if "<STR_LIT:result>" in msg:<EOL><INDENT>task_fut.set_result(msg["<STR_LIT:result>"])<EOL><DEDENT>elif "<STR_LIT>" in msg:<EOL><INDENT>pass<EOL><DEDENT>elif '<STR_LIT>' in msg:<EOL><INDENT>logger.warning("<STR_LIT>")<EOL>try:<EOL><INDENT>s, _ = deserialize_object(msg['<STR_LIT>'])<EOL>exception = ValueError("<STR_LIT>".format(s))<EOL>task_fut.set_exception(exception)<EOL><DEDENT>except Exception as e:<EOL><INDENT>task_fut.set_exception(<EOL>DeserializationError("<STR_LIT>".format(e)))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise BadMessage(<EOL>"<STR_LIT>")<EOL><DEDENT>if not self.is_alive:<EOL><INDENT>break<EOL><DEDENT><DEDENT>logger.info("<STR_LIT>")<EOL>


def submit(self, func, *args, **kwargs):

self._task_counter += <NUM_LIT:1><EOL>task_id = self._task_counter<EOL>logger.debug(<EOL>"<STR_LIT>".format(func, args))<EOL>self.tasks[task_id] = Future()<EOL>fn_buf = pack_apply_message(func, args, kwargs,<EOL>buffer_threshold=<NUM_LIT> * <NUM_LIT>,<EOL>item_threshold=<NUM_LIT>)<EOL>self.outgoing_q.put(task_id, fn_buf)<EOL>return self.tasks[task_id]<EOL>


r = []<EOL>for i in range(blocks):<EOL><INDENT>if self.provider:<EOL><INDENT>block = self.provider.submit(<EOL>self.launch_cmd, <NUM_LIT:1>, self.workers_per_node)<EOL>logger.debug("<STR_LIT>".format(i, block))<EOL>if not block:<EOL><INDENT>raise(ScalingFailed(self.provider.label,<EOL>"<STR_LIT>"))<EOL><DEDENT>self.blocks.extend([block])<EOL><DEDENT>else:<EOL><INDENT>logger.error("<STR_LIT>")<EOL>r = None<EOL><DEDENT><DEDENT>return r<EOL>

Scales out the number of active workers by the number of blocks specified.

        Parameters
        ----------

        blocks : int
             # of blocks to scale out. Default=1

        Raises:
             NotImplementedError


to_kill = self.blocks[:blocks]<EOL>if self.provider:<EOL><INDENT>r = self.provider.cancel(to_kill)<EOL><DEDENT>return r<EOL>

Scale in the number of active blocks by specified amount.

        The scale in method here is very rude. It doesn't give the workers
        the opportunity to finish current tasks or cleanup. This is tracked
        in issue #530

        Raises:
             NotImplementedError


status = []<EOL>if self.provider:<EOL><INDENT>status = self.provider.status(self.blocks)<EOL><DEDENT>return status<EOL>


def shutdown(self, hub=True, targets='<STR_LIT:all>', block=False):

logger.warning("<STR_LIT>")<EOL>self.queue_proc.terminate()<EOL>logger.warning("<STR_LIT>")<EOL>return True<EOL>

Shutdown the executor, including all workers and controllers.

        This is not implemented.

        Kwargs:
            - hub (Bool): Whether the hub should be shutdown, Default:True,
            - targets (list of ints| 'all'): List of block id's to kill, Default:'all'
            - block (Bool): To block for confirmations or not

        Raises:
             NotImplementedError


@abstractmethod<EOL><INDENT>def start(self, *args, **kwargs):<DEDENT>

pass<EOL>

Start the executor.

        Any spin-up operations (for example: starting thread pools) should be performed here.


@abstractmethod<EOL><INDENT>def submit(self, *args, **kwargs):<DEDENT>

pass<EOL>

Submit.

        We haven't yet decided on what the args to this can be,
        whether it should just be func, args, kwargs or be the partially evaluated
        fn


@abstractmethod<EOL><INDENT>def scale_out(self, *args, **kwargs):<DEDENT>

pass<EOL>

Scale out method.

        We should have the scale out method simply take resource object
        which will have the scaling methods, scale_out itself should be a coroutine, since
        scaling tasks can be slow.


pass<EOL>

Scale in method.

        Cause the executor to reduce the number of blocks by count.

        We should have the scale in method simply take resource object
        which will have the scaling methods, scale_in itself should be a coroutine, since
        scaling tasks can be slow.


@abstractmethod<EOL><INDENT>def shutdown(self, *args, **kwargs):<DEDENT>

pass<EOL>

Shutdown the executor.

        This includes all attached resources such as workers and controllers.


pass<EOL>

Specify if scaling is enabled.

        The callers of ParslExecutors need to differentiate between Executors
        and Executors wrapped in a resource provider


return self._run_dir<EOL>


logger.debug("<STR_LIT>")<EOL>def execute_task(bufs):<EOL><INDENT>"""<STR_LIT>"""<EOL>user_ns = locals()<EOL>user_ns.update({'<STR_LIT>': __builtins__})<EOL>f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False)<EOL>fname = getattr(f, '<STR_LIT>', '<STR_LIT:f>')<EOL>prefix = "<STR_LIT>"<EOL>fname = prefix + "<STR_LIT:f>"<EOL>argname = prefix + "<STR_LIT:args>"<EOL>kwargname = prefix + "<STR_LIT>"<EOL>resultname = prefix + "<STR_LIT:result>"<EOL>user_ns.update({fname: f,<EOL>argname: args,<EOL>kwargname: kwargs,<EOL>resultname: resultname})<EOL>code = "<STR_LIT>".format(resultname, fname,<EOL>argname, kwargname)<EOL>try:<EOL><INDENT>logger.debug("<STR_LIT>".format(code))<EOL>exec(code, user_ns, user_ns)<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.warning("<STR_LIT>".format(e))<EOL>raise e<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>".format(user_ns.get(resultname)))<EOL>return user_ns.get(resultname)<EOL><DEDENT><DEDENT>while True:<EOL><INDENT>try:<EOL><INDENT>msg = incoming_q.get(block=True, timeout=<NUM_LIT:10>)<EOL><DEDENT>except queue.Empty:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL><DEDENT>except IOError as e:<EOL><INDENT>logger.debug("<STR_LIT>".format(e))<EOL>try:<EOL><INDENT>outgoing_q.put(None)<EOL><DEDENT>except Exception:<EOL><INDENT>pass<EOL><DEDENT>break<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.debug("<STR_LIT>".format(e))<EOL><DEDENT>else:<EOL><INDENT>if not msg:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>outgoing_q.put(None)<EOL>break<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>".format(msg["<STR_LIT>"]))<EOL>try:<EOL><INDENT>response_obj = execute_task(msg['<STR_LIT>'])<EOL>response = {"<STR_LIT>": msg["<STR_LIT>"],<EOL>"<STR_LIT:result>": serialize_object(response_obj)}<EOL>logger.debug("<STR_LIT>".format(<EOL>deserialize_object(response["<STR_LIT:result>"])))<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.debug("<STR_LIT>".format(e))<EOL>response = {"<STR_LIT>": msg["<STR_LIT>"],<EOL>"<STR_LIT>": serialize_object(e)}<EOL><DEDENT>outgoing_q.put(response)<EOL><DEDENT><DEDENT><DEDENT>logger.debug("<STR_LIT>")<EOL>

This is a function that mocks the Swift-T side.

    It listens on the the incoming_q for tasks and posts returns on the outgoing_q.

    Args:
         - incoming_q (Queue object) : The queue to listen on
         - outgoing_q (Queue object) : Queue to post results on

    The messages posted on the incoming_q will be of the form :

    .. code:: python

       {
          "task_id" : <uuid.uuid4 string>,
          "buffer"  : serialized buffer containing the fn, args and kwargs
       }

    If ``None`` is received, the runner will exit.

    Response messages should be of the form:

    .. code:: python

       {
          "task_id" : <uuid.uuid4 string>,
          "result"  : serialized buffer containing result
          "exception" : serialized exception object
       }

    On exiting the runner will post ``None`` to the outgoing_q


def __init__(self, label='<STR_LIT>', storage_access=None, working_dir=None, managed=True):

logger.debug("<STR_LIT>")<EOL>self.label = label<EOL>self.storage_access = storage_access if storage_access is not None else []<EOL>if len(self.storage_access) > <NUM_LIT:1>:<EOL><INDENT>raise ConfigurationError('<STR_LIT>')<EOL><DEDENT>self.working_dir = working_dir<EOL>self.managed = managed<EOL>

Initialize the thread pool.

        Trying to implement the emews model.


while True:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>try:<EOL><INDENT>msg = self.incoming_q.get(block=True, timeout=<NUM_LIT:1>)<EOL><DEDENT>except queue.Empty:<EOL><INDENT>pass<EOL><DEDENT>except IOError as e:<EOL><INDENT>logger.debug("<STR_LIT>".format(e.errno, e))<EOL>return<EOL><DEDENT>except Exception as e:<EOL><INDENT>logger.debug("<STR_LIT>".format(e))<EOL><DEDENT>else:<EOL><INDENT>if msg is None:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>return<EOL><DEDENT>else:<EOL><INDENT>logger.debug("<STR_LIT>".format(msg))<EOL>task_fut = self.tasks[msg['<STR_LIT>']]<EOL>if '<STR_LIT:result>' in msg:<EOL><INDENT>result, _ = deserialize_object(msg['<STR_LIT:result>'])<EOL>task_fut.set_result(result)<EOL><DEDENT>elif '<STR_LIT>' in msg:<EOL><INDENT>exception, _ = deserialize_object(msg['<STR_LIT>'])<EOL>task_fut.set_exception(exception)<EOL><DEDENT><DEDENT><DEDENT>if not self.is_alive:<EOL><INDENT>break<EOL><DEDENT><DEDENT>

Listen to the queue for task status messages and handle them.

        Depending on the message, tasks will be updated with results, exceptions,
        or updates. It expects the following messages:

        .. code:: python

            {
               "task_id" : <task_id>
               "result"  : serialized result object, if task succeeded
               ... more tags could be added later
            }

            {
               "task_id" : <task_id>
               "exception" : serialized exception object, on failure
            }

        We do not support these yet, but they could be added easily.

        .. code:: python

            {
               "task_id" : <task_id>
               "cpu_stat" : <>
               "mem_stat" : <>
               "io_stat"  : <>
               "started"  : tstamp
            }

        The `None` message is a die request.


q.put(None)<EOL>


logging.debug("<STR_LIT>", "<STR_LIT:*>" * <NUM_LIT>)<EOL>if self._queue_management_thread is None:<EOL><INDENT>logging.debug("<STR_LIT>")<EOL>self._queue_management_thread = threading.Thread(target=self._queue_management_worker)<EOL>self._queue_management_thread.daemon = True<EOL>self._queue_management_thread.start()<EOL><DEDENT>else:<EOL><INDENT>logging.debug("<STR_LIT>")<EOL><DEDENT>

Method to start the management thread as a daemon.

        Checks if a thread already exists, then starts it.
        Could be used later as a restart if the management thread dies.


self.is_alive = False<EOL>logging.debug("<STR_LIT>")<EOL>self.incoming_q.put(None)  <EOL>self._queue_management_thread.join()  <EOL>logging.debug("<STR_LIT>")<EOL>self.worker.join()<EOL>return True<EOL>

Shutdown method, to kill the threads and workers.


task_id = uuid.uuid4()<EOL>logger.debug("<STR_LIT>".format(func, args))<EOL>self.tasks[task_id] = Future()<EOL>fn_buf = pack_apply_message(func, args, kwargs,<EOL>buffer_threshold=<NUM_LIT> * <NUM_LIT>,<EOL>item_threshold=<NUM_LIT>)<EOL>msg = {"<STR_LIT>": task_id,<EOL>"<STR_LIT>": fn_buf}<EOL>self.outgoing_q.put(msg)<EOL>return self.tasks[task_id]<EOL>

Submits work to the the outgoing_q.

        The outgoing_q is an external process listens on this
        queue for new work. This method is simply pass through and behaves like a
        submit call as described here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_

        Args:
            - func (callable) : Callable function
            - *args (list) : List of arbitrary positional arguments.

        Kwargs:
            - **kwargs (dict) : A dictionary of arbitrary keyword args for func.

        Returns:
              Future


raise NotImplementedError<EOL>

Scales out the number of active workers by 1.

        This method is not implemented for threads and will raise the error if called.
        This would be nice to have, and can be done

        Raises:
             NotImplementedError


raise NotImplementedError<EOL>

Scale in the number of active blocks by specified amount.

        This method is not implemented for turbine and will raise an error if called.

        Raises:
             NotImplementedError


def compose_launch_cmd(self, filepath, engine_dir, container_image):

self.engine_file = os.path.expanduser(filepath)<EOL>uid = str(uuid.uuid4())<EOL>engine_json = None<EOL>try:<EOL><INDENT>with open(self.engine_file, '<STR_LIT:r>') as f:<EOL><INDENT>engine_json = f.read()<EOL><DEDENT><DEDENT>except OSError as e:<EOL><INDENT>logger.error("<STR_LIT>", self.engine_file)<EOL>raise e<EOL><DEDENT>return

Reads the json contents from filepath and uses that to compose the engine launch command.

        Args:
            filepath: Path to the engine file
            engine_dir: CWD for the engines


def compose_containerized_launch_cmd(self, filepath, engine_dir, container_image):

Reads the json contents from filepath and uses that to compose the engine launch command.

        Notes: Add this to the ipengine launch for debug logs :
                          --log-to-file --debug
        Args:
            filepath (str): Path to the engine file
            engine_dir (str): CWD for the engines .
            container_image (str): The container to be used to launch workers


return self.lb_view.apply_async(*args, **kwargs)<EOL>

Submits work to the thread pool.

        This method is simply pass through and behaves like a submit call as described
        here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_

        Returns:
              Future


r = []<EOL>for i in range(blocks):<EOL><INDENT>if self.provider:<EOL><INDENT>block = self.provider.submit(self.launch_cmd, <NUM_LIT:1>, self.workers_per_node)<EOL>logger.debug("<STR_LIT>".format(i, block))<EOL>if not block:<EOL><INDENT>raise(ScalingFailed(self.provider.label,<EOL>"<STR_LIT>"))<EOL><DEDENT>self.engines.extend([block])<EOL>r.extend([block])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>logger.error("<STR_LIT>")<EOL>r = None<EOL><DEDENT>return r<EOL>

Scales out the number of active workers by 1.

        This method is notImplemented for threads and will raise the error if called.

        Parameters:
            blocks : int
               Number of blocks to be provisioned.


status = dict(zip(self.engines, self.provider.status(self.engines)))<EOL>to_kill = [engine for engine in status if status[engine] == "<STR_LIT>"][:blocks]<EOL>if self.provider:<EOL><INDENT>r = self.provider.cancel(to_kill)<EOL><DEDENT>else:<EOL><INDENT>logger.error("<STR_LIT>")<EOL>r = None<EOL><DEDENT>return r<EOL>

Scale in the number of active blocks by the specified number.


if self.provider:<EOL><INDENT>status = self.provider.status(self.engines)<EOL><DEDENT>else:<EOL><INDENT>status = []<EOL><DEDENT>return status<EOL>

Returns the status of the executor via probing the execution providers.


if self.controller:<EOL><INDENT>logger.debug("<STR_LIT>")<EOL>self.controller.close()<EOL><DEDENT>logger.debug("<STR_LIT>")<EOL>return True<EOL>

Shutdown the executor, including all workers and controllers.

        The interface documentation for IPP is `here <http://ipyparallel.readthedocs.io/en/latest/api/ipyparallel.html#ipyparallel.Client.shutdown>`_

        Kwargs:
            - hub (Bool): Whether the hub should be shutdown, Default:True,
            - targets (list of ints| 'all'): List of engine id's to kill, Default:'all'
            - block (Bool): To block for confirmations or not

        Raises:
             NotImplementedError


if isinstance(buf, memoryview):<EOL><INDENT>if PY3:<EOL><INDENT>return buf.nbytes<EOL><DEDENT>else:<EOL><INDENT>size = buf.itemsize<EOL>for dim in buf.shape:<EOL><INDENT>size *= dim<EOL><DEDENT>return size<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return len(buf)<EOL><DEDENT>

Return byte-size of a memoryview or buffer.


buffers = []<EOL>if isinstance(obj, CannedObject) and obj.buffers:<EOL><INDENT>for i, buf in enumerate(obj.buffers):<EOL><INDENT>nbytes = _nbytes(buf)<EOL>if nbytes > threshold:<EOL><INDENT>obj.buffers[i] = None<EOL>buffers.append(buf)<EOL><DEDENT>elif isinstance(buf, memoryview):<EOL><INDENT>obj.buffers[i] = buf.tobytes()<EOL><DEDENT>elif isinstance(buf, buffer):<EOL><INDENT>obj.buffers[i] = bytes(buf)<EOL><DEDENT><DEDENT><DEDENT>return buffers<EOL>

Extract buffers larger than a certain threshold.


if isinstance(obj, CannedObject) and obj.buffers:<EOL><INDENT>for i, buf in enumerate(obj.buffers):<EOL><INDENT>if buf is None:<EOL><INDENT>obj.buffers[i] = buffers.pop(<NUM_LIT:0>)<EOL><DEDENT><DEDENT><DEDENT>


def serialize_object(obj, buffer_threshold=MAX_BYTES, item_threshold=MAX_ITEMS):

buffers = []<EOL>if istype(obj, sequence_types) and len(obj) < item_threshold:<EOL><INDENT>cobj = can_sequence(obj)<EOL>for c in cobj:<EOL><INDENT>buffers.extend(_extract_buffers(c, buffer_threshold))<EOL><DEDENT><DEDENT>elif istype(obj, dict) and len(obj) < item_threshold:<EOL><INDENT>cobj = {}<EOL>for k in sorted(obj):<EOL><INDENT>c = can(obj[k])<EOL>buffers.extend(_extract_buffers(c, buffer_threshold))<EOL>cobj[k] = c<EOL><DEDENT><DEDENT>else:<EOL><INDENT>cobj = can(obj)<EOL>buffers.extend(_extract_buffers(cobj, buffer_threshold))<EOL><DEDENT>buffers.insert(<NUM_LIT:0>, pickle.dumps(cobj, PICKLE_PROTOCOL))<EOL>return buffers<EOL>

Serialize an object into a list of sendable buffers.

    Parameters
    ----------

    obj : object
        The object to be serialized
    buffer_threshold : int
        The threshold (in bytes) for pulling out data buffers
        to avoid pickling them.
    item_threshold : int
        The maximum number of items over which canning will iterate.
        Containers (lists, dicts) larger than this will be pickled without
        introspection.

    Returns
    -------
    [bufs] : list of buffers representing the serialized object.


bufs = list(buffers)<EOL>pobj = buffer_to_bytes_py2(bufs.pop(<NUM_LIT:0>))<EOL>canned = pickle.loads(pobj)<EOL>if istype(canned, sequence_types) and len(canned) < MAX_ITEMS:<EOL><INDENT>for c in canned:<EOL><INDENT>_restore_buffers(c, bufs)<EOL><DEDENT>newobj = uncan_sequence(canned, g)<EOL><DEDENT>elif istype(canned, dict) and len(canned) < MAX_ITEMS:<EOL><INDENT>newobj = {}<EOL>for k in sorted(canned):<EOL><INDENT>c = canned[k]<EOL>_restore_buffers(c, bufs)<EOL>newobj[k] = uncan(c, g)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>_restore_buffers(canned, bufs)<EOL>newobj = uncan(canned, g)<EOL><DEDENT>return newobj, bufs<EOL>

Reconstruct an object serialized by serialize_object from data buffers.

    Parameters
    ----------

    bufs : list of buffers/bytes

    g : globals to be used when uncanning

    Returns
    -------

    (newobj, bufs) : unpacked object, and the list of remaining unused buffers.


def pack_apply_message(f, args, kwargs, buffer_threshold=MAX_BYTES, item_threshold=MAX_ITEMS):

arg_bufs = list(chain.from_iterable(<EOL>serialize_object(arg, buffer_threshold, item_threshold) for arg in args))<EOL>kw_keys = sorted(kwargs.keys())<EOL>kwarg_bufs = list(chain.from_iterable(<EOL>serialize_object(kwargs[key], buffer_threshold, item_threshold) for key in kw_keys))<EOL>info = dict(nargs=len(args), narg_bufs=len(arg_bufs), kw_keys=kw_keys)<EOL>msg = [pickle.dumps(can(f), PICKLE_PROTOCOL)]<EOL>msg.append(pickle.dumps(info, PICKLE_PROTOCOL))<EOL>msg.extend(arg_bufs)<EOL>msg.extend(kwarg_bufs)<EOL>return msg<EOL>

Pack up a function, args, and kwargs to be sent over the wire.

    Each element of args/kwargs will be canned for special treatment,
    but inspection will not go any deeper than that.

    Any object whose data is larger than `threshold`  will not have their data copied
    (only numpy arrays and bytes/buffers support zero-copy)

    Message will be a list of bytes/buffers of the format:

    [ cf, pinfo, <arg_bufs>, <kwarg_bufs> ]

    With length at least two + len(args) + len(kwargs)


def unpack_apply_message(bufs, g=None, copy=True):

bufs = list(bufs)  <EOL>assert len(bufs) >= <NUM_LIT:2>, "<STR_LIT>"<EOL>pf = buffer_to_bytes_py2(bufs.pop(<NUM_LIT:0>))<EOL>f = uncan(pickle.loads(pf), g)<EOL>pinfo = buffer_to_bytes_py2(bufs.pop(<NUM_LIT:0>))<EOL>info = pickle.loads(pinfo)<EOL>arg_bufs, kwarg_bufs = bufs[:info['<STR_LIT>']], bufs[info['<STR_LIT>']:]<EOL>args = []<EOL>for i in range(info['<STR_LIT>']):<EOL><INDENT>arg, arg_bufs = deserialize_object(arg_bufs, g)<EOL>args.append(arg)<EOL><DEDENT>args = tuple(args)<EOL>assert not arg_bufs, "<STR_LIT>"<EOL>kwargs = {}<EOL>for key in info['<STR_LIT>']:<EOL><INDENT>kwarg, kwarg_bufs = deserialize_object(kwarg_bufs, g)<EOL>kwargs[key] = kwarg<EOL><DEDENT>assert not kwarg_bufs, "<STR_LIT>"<EOL>return f, args, kwargs<EOL>

Unpack f,args,kwargs from buffers packed by pack_apply_message().

    Returns: original f,args,kwargs


def inner():<EOL><INDENT>return a<EOL><DEDENT>return type(py3compat.get_closure(inner)[<NUM_LIT:0>])<EOL>

The type of a closure cell doesn't seem to be importable, so just create one.


<EOL>if isinstance(f, FunctionType):<EOL><INDENT>mainmod = __import__('<STR_LIT:__main__>')<EOL>f = FunctionType(f.__code__, mainmod.__dict__,<EOL>f.__name__, f.__defaults__,<EOL>)<EOL><DEDENT>f.__module__ = '<STR_LIT:__main__>'<EOL>return f<EOL>

Decorator for making functions appear as interactively defined.

    This results in the function being linked to the user_ns as globals()
    instead of the module globals().


import dill<EOL>from . import serialize<EOL>serialize.pickle = dill<EOL>can_map.pop(FunctionType, None)<EOL>

Use dill to expand serialization support.

    Adds support for object methods and closures to serialization.


import cloudpickle<EOL>from . import serialize<EOL>serialize.pickle = cloudpickle<EOL>can_map.pop(FunctionType, None)<EOL>

Use cloudpickle to expand serialization support.

    Adds support for object methods and closures to serialization.


from . import serialize<EOL>serialize.pickle = serialize._stdlib_pickle<EOL>can_map[FunctionType] = _original_can_map[FunctionType]<EOL>

Revert to using stdlib pickle.

    Reverts custom serialization enabled by use_dill|cloudpickle.


<EOL>for key, value in list(mapping.items()):<EOL><INDENT>if isinstance(key, string_types):<EOL><INDENT>try:<EOL><INDENT>cls = import_item(key)<EOL><DEDENT>except Exception:<EOL><INDENT>if original and key not in original:<EOL><INDENT>print("<STR_LIT>", key, exc_info=True)<EOL><DEDENT>mapping.pop(key)<EOL><DEDENT>else:<EOL><INDENT>mapping[cls] = mapping.pop(key)<EOL><DEDENT><DEDENT><DEDENT>

Import any string-keys in a type mapping.


if isinstance(check, tuple):<EOL><INDENT>for cls in check:<EOL><INDENT>if type(obj) is cls:<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL><DEDENT>else:<EOL><INDENT>return type(obj) is check<EOL><DEDENT>

Like isinstance(obj, check), but strict.

    This won't catch subclasses.


import_needed = False<EOL>for cls, canner in iteritems(can_map):<EOL><INDENT>if isinstance(cls, string_types):<EOL><INDENT>import_needed = True<EOL>break<EOL><DEDENT>elif istype(obj, cls):<EOL><INDENT>return canner(obj)<EOL><DEDENT><DEDENT>if import_needed:<EOL><INDENT>_import_mapping(can_map, _original_can_map)<EOL>return can(obj)<EOL><DEDENT>return obj<EOL>


if istype(obj, dict):<EOL><INDENT>newobj = {}<EOL>for k, v in iteritems(obj):<EOL><INDENT>newobj[k] = can(v)<EOL><DEDENT>return newobj<EOL><DEDENT>else:<EOL><INDENT>return obj<EOL><DEDENT>


if istype(obj, sequence_types):<EOL><INDENT>t = type(obj)<EOL>return t([can(i) for i in obj])<EOL><DEDENT>else:<EOL><INDENT>return obj<EOL><DEDENT>


import_needed = False<EOL>for cls, uncanner in iteritems(uncan_map):<EOL><INDENT>if isinstance(cls, string_types):<EOL><INDENT>import_needed = True<EOL>break<EOL><DEDENT>elif isinstance(obj, cls):<EOL><INDENT>return uncanner(obj, g)<EOL><DEDENT><DEDENT>if import_needed:<EOL><INDENT>_import_mapping(uncan_map, _original_uncan_map)<EOL>return uncan(obj, g)<EOL><DEDENT>return obj<EOL>


def __init__(self, obj, keys=[], hook=None):

self.keys = keys<EOL>self.obj = copy.copy(obj)<EOL>self.hook = can(hook)<EOL>for key in keys:<EOL><INDENT>setattr(self.obj, key, can(getattr(obj, key)))<EOL><DEDENT>self.buffers = []<EOL>

Can an object for safe pickling.

        Parameters
        ==========

        obj:
            The object to be canned
        keys: list (optional)
            list of attribute names that will be explicitly canned / uncanned
        hook: callable (optional)
            An optional extra callable,
            which can do additional processing of the uncanned object.

        Large data may be offloaded into the buffers list,
        used for zero-copy transfers.


def __init__(self, ip_address, port_range):

self.context = zmq.Context()<EOL>self.zmq_socket = self.context.socket(zmq.REQ)<EOL>self.port = self.zmq_socket.bind_to_random_port("<STR_LIT>".format(ip_address),<EOL>min_port=port_range[<NUM_LIT:0>],<EOL>max_port=port_range[<NUM_LIT:1>])<EOL>

Parameters
----------

ip_address: str
   IP address of the client (where Parsl runs)
port_range: tuple(int, int)
   Port range for the comms between client and interchange