def validate_submit_args_or_fail(job_descriptor, provider_name, input_providers,<EOL>output_providers, logging_providers):

job_resources = job_descriptor.job_resources<EOL>job_params = job_descriptor.job_params<EOL>task_descriptors = job_descriptor.task_descriptors<EOL>_validate_providers([job_resources.logging], '<STR_LIT>', logging_providers,<EOL>provider_name)<EOL>_validate_providers(job_params['<STR_LIT>'], '<STR_LIT:input>', input_providers,<EOL>provider_name)<EOL>_validate_providers(job_params['<STR_LIT>'], '<STR_LIT>', output_providers,<EOL>provider_name)<EOL>for task_descriptor in task_descriptors:<EOL><INDENT>_validate_providers(task_descriptor.task_params['<STR_LIT>'], '<STR_LIT:input>',<EOL>input_providers, provider_name)<EOL>_validate_providers(task_descriptor.task_params['<STR_LIT>'], '<STR_LIT>',<EOL>output_providers, provider_name)<EOL><DEDENT>

Validate that arguments passed to submit_job have valid file providers.

    This utility function takes resources and task data args from `submit_job`
    in the base provider. This function will fail with a value error if any of the
    parameters are not valid. See the following example;

    >>> job_resources = type('', (object,),
    ...    {"logging": job_model.LoggingParam('gs://logtemp', job_model.P_GCS)})()
    >>> job_params={'inputs': set(), 'outputs': set(), 'mounts': set()}
    >>> task_descriptors = [
    ...     job_model.TaskDescriptor(None, {
    ...       'inputs': {
    ...           job_model.FileParam('IN', uri='gs://in/*',
    ...                               file_provider=job_model.P_GCS)},
    ...       'outputs': set()}, None),
    ...     job_model.TaskDescriptor(None, {
    ...       'inputs': set(),
    ...       'outputs': {
    ...           job_model.FileParam('OUT', uri='gs://out/*',
    ...                               file_provider=job_model.P_GCS)}}, None)]
    ...
    >>> validate_submit_args_or_fail(job_model.JobDescriptor(None, job_params,
    ...                              job_resources, task_descriptors),
    ...                              provider_name='MYPROVIDER',
    ...                              input_providers=[job_model.P_GCS],
    ...                              output_providers=[job_model.P_GCS],
    ...                              logging_providers=[job_model.P_GCS])
    ...
    >>> validate_submit_args_or_fail(job_model.JobDescriptor(None, job_params,
    ...                              job_resources, task_descriptors),
    ...                              provider_name='MYPROVIDER',
    ...                              input_providers=[job_model.P_GCS],
    ...                              output_providers=[job_model.P_LOCAL],
    ...                              logging_providers=[job_model.P_GCS])
    Traceback (most recent call last):
         ...
    ValueError: Unsupported output path (gs://out/*) for provider 'MYPROVIDER'.

    Args:
      job_descriptor: instance of job_model.JobDescriptor.
      provider_name: (str) the name of the execution provider.
      input_providers: (string collection) whitelist of file providers for input.
      output_providers: (string collection) whitelist of providers for output.
      logging_providers: (string collection) whitelist of providers for logging.

    Raises:
      ValueError: if any file providers do not match the whitelists.


if not logging_uri:<EOL><INDENT>return job_model.LoggingParam(None, None)<EOL><DEDENT>recursive = not logging_uri.endswith('<STR_LIT>')<EOL>oututil = util_class('<STR_LIT>')<EOL>_, uri, provider = oututil.parse_uri(logging_uri, recursive)<EOL>if '<STR_LIT:*>' in uri.basename:<EOL><INDENT>raise ValueError('<STR_LIT>' % uri)<EOL><DEDENT>return job_model.LoggingParam(uri, provider)<EOL>

Convenience function simplifies construction of the logging uri.


<EOL>raw_uri = directory_fmt(raw_uri)<EOL>_, docker_path = _gcs_uri_rewriter(raw_uri)<EOL>docker_uri = os.path.join(self._relative_path, docker_path)<EOL>return docker_uri<EOL>

Return a valid docker_path for a GCS bucket.


if credentials is None:<EOL><INDENT>credentials = oauth2client.client.GoogleCredentials.get_application_default(<EOL>)<EOL><DEDENT>return discovery.build('<STR_LIT>', '<STR_LIT>', credentials=credentials)<EOL>

Get a storage client using the provided credentials or defaults.


if file_path.startswith('<STR_LIT>'):<EOL><INDENT>return _load_file_from_gcs(file_path, credentials)<EOL><DEDENT>else:<EOL><INDENT>return open(file_path, '<STR_LIT:r>')<EOL><DEDENT>

Load a file from either local or gcs.

    Args:
      file_path: The target file path, which should have the prefix 'gs://' if
                 to be loaded from gcs.
      credentials: Optional credential to be used to load the file from gcs.

    Returns:
      A python File object if loading file from local or a StringIO object if
      loading from gcs.


printer = _Printer(fileobj)<EOL>previous_stdout = sys.stdout<EOL>sys.stdout = printer<EOL>try:<EOL><INDENT>yield printer<EOL><DEDENT>finally:<EOL><INDENT>sys.stdout = previous_stdout<EOL><DEDENT>

Sys.out replacer, by default with stderr.

    Use it like this:
    with replace_print_with(fileobj):
      print "hello"  # writes to the file
    print "done"  # prints to stdout

    Args:
      fileobj: a file object to replace stdout.

    Yields:
      The printer.


gcs_service = _get_storage_service(credentials)<EOL>bucket_name, object_name = gcs_file_path[len('<STR_LIT>'):].split('<STR_LIT:/>', <NUM_LIT:1>)<EOL>request = gcs_service.objects().get_media(<EOL>bucket=bucket_name, object=object_name)<EOL>file_handle = io.BytesIO()<EOL>downloader = MediaIoBaseDownload(file_handle, request, chunksize=<NUM_LIT> * <NUM_LIT>)<EOL>done = False<EOL>while not done:<EOL><INDENT>_, done = _downloader_next_chunk(downloader)<EOL><DEDENT>filevalue = file_handle.getvalue()<EOL>if not isinstance(filevalue, six.string_types):<EOL><INDENT>filevalue = filevalue.decode()<EOL><DEDENT>return six.StringIO(filevalue)<EOL>

Load context from a text file in gcs.

    Args:
      gcs_file_path: The target file path; should have the 'gs://' prefix.
      credentials: Optional credential to be used to load the file from gcs.

    Returns:
      The content of the text file as a string.


<EOL>if self.priority == other.priority:<EOL><INDENT>localkeys = self.next_value.keys()<EOL>otherkeys = other.next_value.keys()<EOL>shared_keys = sorted(set(localkeys).intersection(otherkeys))<EOL>for key in shared_keys:<EOL><INDENT>if self.next_value[key] != other.next_value[key]:<EOL><INDENT>return self.next_value[key] < other.next_value[key]<EOL><DEDENT><DEDENT><DEDENT>return self.priority < other.priority<EOL>

Allow order testing via '<' operator, needed by priority queue.


for task_desc in task_descriptors:<EOL><INDENT>for param in [<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>'<EOL>]:<EOL><INDENT>if not task_desc.task_params.get(param):<EOL><INDENT>task_desc.task_params[param] = set()<EOL><DEDENT><DEDENT><DEDENT>

For each task, ensure that each task param entry is not None.


try:<EOL><INDENT>job = yaml.full_load(yaml_string)<EOL><DEDENT>except AttributeError:<EOL><INDENT>job = yaml.load(yaml_string)<EOL><DEDENT>dsub_version = job.get('<STR_LIT>')<EOL>if not dsub_version:<EOL><INDENT>return cls._from_yaml_v0(job)<EOL><DEDENT>job_metadata = {}<EOL>for key in [<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>'<EOL>]:<EOL><INDENT>if job.get(key) is not None:<EOL><INDENT>job_metadata[key] = job.get(key)<EOL><DEDENT><DEDENT>job_metadata['<STR_LIT>'] = dsub_util.replace_timezone(<EOL>job.get('<STR_LIT>'), pytz.utc)<EOL>job_resources = Resources(logging=job.get('<STR_LIT>'))<EOL>job_params = {}<EOL>job_params['<STR_LIT>'] = cls._label_params_from_dict(job.get('<STR_LIT>', {}))<EOL>job_params['<STR_LIT>'] = cls._env_params_from_dict(job.get('<STR_LIT>', {}))<EOL>job_params['<STR_LIT>'] = cls._input_file_params_from_dict(<EOL>job.get('<STR_LIT>', {}), False)<EOL>job_params['<STR_LIT>'] = cls._input_file_params_from_dict(<EOL>job.get('<STR_LIT>', {}), True)<EOL>job_params['<STR_LIT>'] = cls._output_file_params_from_dict(<EOL>job.get('<STR_LIT>', {}), False)<EOL>job_params['<STR_LIT>'] = cls._output_file_params_from_dict(<EOL>job.get('<STR_LIT>', {}), True)<EOL>job_params['<STR_LIT>'] = cls._mount_params_from_dict(job.get('<STR_LIT>', {}))<EOL>task_descriptors = []<EOL>for task in job.get('<STR_LIT>', []):<EOL><INDENT>task_metadata = {'<STR_LIT>': task.get('<STR_LIT>')}<EOL>create_time = task.get('<STR_LIT>')<EOL>if create_time:<EOL><INDENT>task_metadata['<STR_LIT>'] = dsub_util.replace_timezone(<EOL>create_time, pytz.utc)<EOL><DEDENT>if task.get('<STR_LIT>') is not None:<EOL><INDENT>task_metadata['<STR_LIT>'] = task.get('<STR_LIT>')<EOL><DEDENT>task_params = {}<EOL>task_params['<STR_LIT>'] = cls._label_params_from_dict(<EOL>task.get('<STR_LIT>', {}))<EOL>task_params['<STR_LIT>'] = cls._env_params_from_dict(task.get('<STR_LIT>', {}))<EOL>task_params['<STR_LIT>'] = cls._input_file_params_from_dict(<EOL>task.get('<STR_LIT>', {}), False)<EOL>task_params['<STR_LIT>'] = cls._input_file_params_from_dict(<EOL>task.get('<STR_LIT>', {}), True)<EOL>task_params['<STR_LIT>'] = cls._output_file_params_from_dict(<EOL>task.get('<STR_LIT>', {}), False)<EOL>task_params['<STR_LIT>'] = cls._output_file_params_from_dict(<EOL>task.get('<STR_LIT>', {}), True)<EOL>task_resources = Resources(logging_path=task.get('<STR_LIT>'))<EOL>task_descriptors.append(<EOL>TaskDescriptor(task_metadata, task_params, task_resources))<EOL><DEDENT>return JobDescriptor(job_metadata, job_params, job_resources,<EOL>task_descriptors)<EOL>

Populate and return a JobDescriptor from a YAML string.


<EOL>if task_id is not None:<EOL><INDENT>if task_id.startswith('<STR_LIT>'):<EOL><INDENT>return int(task_id[len('<STR_LIT>'):])<EOL><DEDENT>else:<EOL><INDENT>return int(task_id)<EOL><DEDENT><DEDENT>

Converts a task-id to the numeric task-id.

    Args:
      task_id: task-id in either task-n or n format

    Returns:
      n


return yaml.dump(self.to_serializable(), default_flow_style=False)<EOL>

Return a YAML string representing the job and task data.

        A provider's internal representation of a dsub task typically does not map
        1-1 to the dsub representation. For example, the Google Genomics Pipeline
        does not natively support "input-recursive" or "output-recursive", so the
        google provider cannot easily reconstruct the user inputs from the
        pipeline's associated Operation object.

        All providers are likely to need a way to reliably serialize job and task-
        related information, either for dstat or for any type of "retry" mechanism
        we might want to build.

        Returns:
          YAML string


task_metadata = self.task_metadata<EOL>task_params = self.task_params<EOL>task_resources = self.task_resources<EOL>task_id = None<EOL>if task_metadata.get('<STR_LIT>') is not None:<EOL><INDENT>task_id = str(task_metadata.get('<STR_LIT>'))<EOL><DEDENT>task = {'<STR_LIT>': task_id}<EOL>task['<STR_LIT>'] = task_metadata.get('<STR_LIT>')<EOL>task['<STR_LIT>'] = task_metadata.get('<STR_LIT>')<EOL>if task_resources.logging_path:<EOL><INDENT>task['<STR_LIT>'] = str(task_resources.logging_path.uri)<EOL><DEDENT>task['<STR_LIT>'] = {var.name: var.value for var in task_params['<STR_LIT>']}<EOL>task['<STR_LIT>'] = {var.name: var.value for var in task_params['<STR_LIT>']}<EOL>task['<STR_LIT>'] = {<EOL>var.name: var.value<EOL>for var in task_params['<STR_LIT>']<EOL>if not var.recursive<EOL>}<EOL>task['<STR_LIT>'] = {<EOL>var.name: var.value<EOL>for var in task_params['<STR_LIT>']<EOL>if var.recursive<EOL>}<EOL>task['<STR_LIT>'] = {<EOL>var.name: var.value<EOL>for var in task_params['<STR_LIT>']<EOL>if not var.recursive<EOL>}<EOL>task['<STR_LIT>'] = {<EOL>var.name: var.value<EOL>for var in task_params['<STR_LIT>']<EOL>if var.recursive<EOL>}<EOL>return _remove_empty_items(task, ['<STR_LIT>'])<EOL>

Return a dict populated for serialization (as YAML/JSON).


@classmethod<EOL><INDENT>def _validate_label(cls, name, value):<DEDENT>

<EOL>cls._check_label_name(name)<EOL>cls._check_label_value(value)<EOL>if not cls._allow_reserved_keys and name in RESERVED_LABELS:<EOL><INDENT>raise ValueError('<STR_LIT>' %<EOL>(name, list(RESERVED_LABELS)))<EOL><DEDENT>

Raise ValueError if the label is invalid.


filtered_outs = [<EOL>var for var in outputs<EOL>if var.recursive and var.file_provider == job_model.P_GCS<EOL>]<EOL>return '<STR_LIT:\n>'.join([<EOL>'<STR_LIT>'.format(var.name,<EOL>source.rstrip('<STR_LIT:/>'),<EOL>var.docker_path.rstrip('<STR_LIT:/>'))<EOL>for var in filtered_outs<EOL>])<EOL>

Return a multi-line string with export statements for the variables.

    Arguments:
      source: Folder with the data.
              For example /mnt/data
      outputs: a list of OutputFileParam

    Returns:
      a multi-line string with a shell script that sets environment variables
      corresponding to the outputs.


export_input_dirs = '<STR_LIT:\n>'.join([<EOL>'<STR_LIT>'.format(var.name, destination.rstrip('<STR_LIT:/>'),<EOL>var.docker_path.rstrip('<STR_LIT:/>'))<EOL>for var in inputs<EOL>if var.recursive and var.docker_path<EOL>])<EOL>return export_input_dirs<EOL>

Return a multi-line string with export statements for the variables.

    Arguments:
      destination: Folder where the data will be put.
                   For example /mnt/data
      inputs: a list of InputFileParam

    Returns:
      a multi-line string with a shell script that sets environment variables
      corresponding to the inputs.


return '<STR_LIT:\n>'.join([<EOL>'<STR_LIT>'.format(var.name, source.rstrip('<STR_LIT:/>'),<EOL>var.docker_path.rstrip('<STR_LIT:/>')) for var in mounts<EOL>])<EOL>

Return a multi-line string with export statements for the variables.

    Arguments:
      source: Folder with the data. For example /mnt/data
      mounts: a list of MountParam

    Returns:
      a multi-line string with a shell script that sets environment variables
      corresponding to the mounts.


Kills the operations associated with the specified job or job.task.

        Args:
          user_ids: List of user ids who "own" the job(s) to cancel.
          job_ids: List of job_ids to cancel.
          task_ids: List of task-ids to cancel.
          labels: List of LabelParam, each must match the job(s) to be canceled.
          create_time_min: a timezone-aware datetime value for the earliest create
                           time of a task, inclusive.
          create_time_max: a timezone-aware datetime value for the most recent
                           create time of a task, inclusive.

        Returns:
          A list of tasks canceled and a list of error messages.


if date is None:<EOL><INDENT>return None<EOL><DEDENT>epoch = dsub_util.replace_timezone(datetime.utcfromtimestamp(<NUM_LIT:0>), pytz.utc)<EOL>return (date - epoch).total_seconds()<EOL>

Convert the integer UTC time value into a local datetime.


@classmethod<EOL><INDENT>def build_pipeline_args(cls, project, script, job_params, task_params,<EOL>reserved_labels, preemptible, logging_uri, scopes,<EOL>keep_alive):<DEDENT>

<EOL>inputs = {}<EOL>inputs.update({SCRIPT_VARNAME: script})<EOL>inputs.update({<EOL>var.name: var.value<EOL>for var in job_params['<STR_LIT>'] | task_params['<STR_LIT>']<EOL>if var.value<EOL>})<EOL>inputs.update({<EOL>var.name: var.uri<EOL>for var in job_params['<STR_LIT>'] | task_params['<STR_LIT>']<EOL>if not var.recursive and var.value<EOL>})<EOL>outputs = {}<EOL>for var in job_params['<STR_LIT>'] | task_params['<STR_LIT>']:<EOL><INDENT>if var.recursive or not var.value:<EOL><INDENT>continue<EOL><DEDENT>if '<STR_LIT:*>' in var.uri.basename:<EOL><INDENT>outputs[var.name] = var.uri.path<EOL><DEDENT>else:<EOL><INDENT>outputs[var.name] = var.uri<EOL><DEDENT><DEDENT>labels = {}<EOL>labels.update({<EOL>label.name: label.value if label.value else '<STR_LIT>'<EOL>for label in (reserved_labels | job_params['<STR_LIT>']<EOL>| task_params['<STR_LIT>'])<EOL>})<EOL>args = {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': project,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': preemptible,<EOL>},<EOL>'<STR_LIT>': inputs,<EOL>'<STR_LIT>': outputs,<EOL>'<STR_LIT>': labels,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:email>': '<STR_LIT:default>',<EOL>'<STR_LIT>': scopes,<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': logging_uri<EOL>},<EOL>}<EOL>}<EOL>if keep_alive:<EOL><INDENT>args['<STR_LIT>'][<EOL>'<STR_LIT>'] = '<STR_LIT>' % keep_alive<EOL><DEDENT>return args<EOL>

Builds pipeline args for execution.

        Args:
          project: string name of project.
          script: Body of the script to execute.
          job_params: dictionary of values for labels, envs, inputs, and outputs
              for this job.
          task_params: dictionary of values for labels, envs, inputs, and outputs
              for this task.
          reserved_labels: dictionary of reserved labels (e.g. task-id,
              task-attempt)
          preemptible: use a preemptible VM for the job
          logging_uri: path for job logging output.
          scopes: list of scope.
          keep_alive: Seconds to keep VM alive on failure

        Returns:
          A nested dictionary with one entry under the key pipelineArgs containing
          the pipeline arguments.


@classmethod<EOL><INDENT>def list(cls, service, ops_filter, page_size=<NUM_LIT:0>):<DEDENT>

page_token = None<EOL>more_operations = True<EOL>documented_default_page_size = <NUM_LIT><EOL>documented_max_page_size = <NUM_LIT><EOL>if not page_size:<EOL><INDENT>page_size = documented_default_page_size<EOL><DEDENT>page_size = min(page_size, documented_max_page_size)<EOL>while more_operations:<EOL><INDENT>api = service.operations().list(<EOL>name='<STR_LIT>',<EOL>filter=ops_filter,<EOL>pageToken=page_token,<EOL>pageSize=page_size)<EOL>response = google_base.Api.execute(api)<EOL>ops = response.get('<STR_LIT>', [])<EOL>for op in ops:<EOL><INDENT>if cls.is_dsub_operation(op):<EOL><INDENT>yield GoogleOperation(op)<EOL><DEDENT><DEDENT>page_token = response.get('<STR_LIT>')<EOL>more_operations = bool(page_token)<EOL><DEDENT>

Gets the list of operations for the specified filter.

        Args:
          service: Google Genomics API service object
          ops_filter: string filter of operations to return
          page_size: the number of operations to requested on each list operation to
            the pipelines API (if 0 or None, the API default is used)

        Yields:
          Operations matching the filter criteria.


@classmethod<EOL><INDENT>def build_pipeline(cls, project, zones, min_cores, min_ram, disk_size,<EOL>boot_disk_size, preemptible, accelerator_type,<EOL>accelerator_count, image, script_name, envs, inputs,<EOL>outputs, pipeline_name):<DEDENT>

if min_cores is None:<EOL><INDENT>min_cores = job_model.DEFAULT_MIN_CORES<EOL><DEDENT>if min_ram is None:<EOL><INDENT>min_ram = job_model.DEFAULT_MIN_RAM<EOL><DEDENT>if disk_size is None:<EOL><INDENT>disk_size = job_model.DEFAULT_DISK_SIZE<EOL><DEDENT>if boot_disk_size is None:<EOL><INDENT>boot_disk_size = job_model.DEFAULT_BOOT_DISK_SIZE<EOL><DEDENT>if preemptible is None:<EOL><INDENT>preemptible = job_model.DEFAULT_PREEMPTIBLE<EOL><DEDENT>docker_command = cls._build_pipeline_docker_command(script_name, inputs,<EOL>outputs, envs)<EOL>input_envs = [{<EOL>'<STR_LIT:name>': SCRIPT_VARNAME<EOL>}] + [{<EOL>'<STR_LIT:name>': env.name<EOL>} for env in envs if env.value]<EOL>input_files = [<EOL>cls._build_pipeline_input_file_param(var.name, var.docker_path)<EOL>for var in inputs<EOL>if not var.recursive and var.value<EOL>]<EOL>output_files = [<EOL>cls._build_pipeline_file_param(var.name, var.docker_path)<EOL>for var in outputs<EOL>if not var.recursive and var.value<EOL>]<EOL>return {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': project,<EOL>'<STR_LIT:name>': pipeline_name,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': min_cores,<EOL>'<STR_LIT>': min_ram,<EOL>'<STR_LIT>': boot_disk_size,<EOL>'<STR_LIT>': preemptible,<EOL>'<STR_LIT>': google_base.get_zones(zones),<EOL>'<STR_LIT>': accelerator_type,<EOL>'<STR_LIT>': accelerator_count,<EOL>'<STR_LIT>': [{<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT>': True,<EOL>'<STR_LIT>': disk_size,<EOL>'<STR_LIT>': providers_util.DATA_MOUNT_POINT,<EOL>}],<EOL>},<EOL>'<STR_LIT>': input_envs + input_files,<EOL>'<STR_LIT>': output_files,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': image,<EOL>'<STR_LIT>': docker_command,<EOL>}<EOL>}<EOL>}<EOL>

Builds a pipeline configuration for execution.

        Args:
          project: string name of project.
          zones: list of zone names for jobs to be run at.
          min_cores: int number of CPU cores required per job.
          min_ram: int GB of RAM required per job.
          disk_size: int GB of disk to attach under /mnt/data.
          boot_disk_size: int GB of disk for boot.
          preemptible: use a preemptible VM for the job
          accelerator_type: string GCE defined accelerator type.
          accelerator_count: int number of accelerators of the specified type to
            attach.
          image: string Docker image name in which to run.
          script_name: file name of the script to run.
          envs: list of EnvParam objects specifying environment variables to set
            within each job.
          inputs: list of FileParam objects specifying input variables to set
            within each job.
          outputs: list of FileParam objects specifying output variables to set
            within each job.
          pipeline_name: string name of pipeline.

        Returns:
          A nested dictionary with one entry under the key ephemeralPipeline
          containing the pipeline configuration.


if not self._op['<STR_LIT>']:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>if '<STR_LIT:error>' not in self._op:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>if self._op['<STR_LIT:error>'].get('<STR_LIT:code>', <NUM_LIT:0>) == <NUM_LIT:1>:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>return '<STR_LIT>'<EOL>

Returns the status of this operation.

        ie. RUNNING, SUCCESS, CANCELED or FAILURE.

        Returns:
          A printable status string


if '<STR_LIT:error>' in self._op:<EOL><INDENT>if '<STR_LIT>' in self._op['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>job_id = self._op['<STR_LIT>']['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>job_id = self._op['<STR_LIT>']['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>return '<STR_LIT>' % (job_id,<EOL>self._op['<STR_LIT:error>']['<STR_LIT:code>'],<EOL>self._op['<STR_LIT:error>']['<STR_LIT:message>'])<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>

Returns an error message if the operation failed for any reason.

        Failure as defined here means; ended for any reason other than 'success'.
        This means that a successful cancelation will also create an error message
        here.

        Returns:
          string, string will be empty if job did not error.


raise NotImplementedError()<EOL>

Kills the operations associated with the specified job or job.task.

        Some providers may provide only a "cancel" operation, which terminates the
        task but does not truly "delete" it from the "task list".

        Args:
          user_ids: a set of user ids who "own" the job(s) to delete.
          job_ids: a set of job ids to delete.
          task_ids: a set of task ids to delete.
          labels: a set of LabelParam, each must match the job(s) to be cancelled.
          create_time_min: a timezone-aware datetime value for the earliest create
                           time of a task, inclusive.
          create_time_max: a timezone-aware datetime value for the most recent
                           create time of a task, inclusive.

        Returns:
          (list of tasks canceled,
           for each task that couldn't be canceled, the error message).

          Only tasks that were running are included in the return value.


@abc.abstractmethod<EOL><INDENT>def submit_job(self, job_descriptor, skip_if_output_present):<DEDENT>

raise NotImplementedError()<EOL>

Submit the job to be executed.

        Args:
          job_descriptor (job_model.JobDescriptor): parameters needed to launch all
          job tasks
          skip_if_output_present: (boolean) if true, skip tasks whose output
            is present (see --skip flag for more explanation).

        Returns:
          A dictionary containing the 'user-id', 'job-id', and 'task-id' list.
          For jobs that are not task array jobs, the task-id list should be empty.
          If all tasks were skipped, then the job-id is dsub_lib.NO_JOB.

        Raises:
          ValueError: submit_job may validate any of the parameters and raise
            a value error if any parameter (or specific combination of parameters)
            is not supported by the provider.


<EOL>m = re.match(r'<STR_LIT>',<EOL>rfc3339_utc_string)<EOL>if not m:<EOL><INDENT>return None<EOL><DEDENT>groups = m.groups()<EOL>if len(groups[<NUM_LIT:6>]) not in (<NUM_LIT:0>, <NUM_LIT:3>, <NUM_LIT:6>, <NUM_LIT:9>):<EOL><INDENT>return None<EOL><DEDENT>g = [int(val) for val in groups[:<NUM_LIT:6>]]<EOL>fraction = groups[<NUM_LIT:6>]<EOL>if not fraction:<EOL><INDENT>micros = <NUM_LIT:0><EOL><DEDENT>elif len(fraction) == <NUM_LIT:3>:<EOL><INDENT>micros = int(fraction) * <NUM_LIT:1000><EOL><DEDENT>elif len(fraction) == <NUM_LIT:6>:<EOL><INDENT>micros = int(fraction)<EOL><DEDENT>elif len(fraction) == <NUM_LIT:9>:<EOL><INDENT>micros = int(round(int(fraction) / <NUM_LIT:1000>))<EOL><DEDENT>else:<EOL><INDENT>assert False, '<STR_LIT>'.len(fraction)<EOL><DEDENT>try:<EOL><INDENT>return datetime(g[<NUM_LIT:0>], g[<NUM_LIT:1>], g[<NUM_LIT:2>], g[<NUM_LIT:3>], g[<NUM_LIT:4>], g[<NUM_LIT:5>], micros, tzinfo=pytz.utc)<EOL><DEDENT>except ValueError as e:<EOL><INDENT>assert False, '<STR_LIT>'.format(<EOL>rfc3339_utc_string, e)<EOL><DEDENT>

Converts a datestamp from RFC3339 UTC to a datetime.

    Args:
      rfc3339_utc_string: a datetime string in RFC3339 UTC "Zulu" format

    Returns:
      A datetime.


def _cancel_batch(batch_fn, cancel_fn, ops):

<EOL>canceled = []<EOL>failed = []<EOL>def handle_cancel_response(request_id, response, exception):<EOL><INDENT>"""<STR_LIT>"""<EOL>del response  <EOL>if exception:<EOL><INDENT>msg = '<STR_LIT>' % (exception.resp.status, exception.resp.reason)<EOL>if exception.resp.status == FAILED_PRECONDITION_CODE:<EOL><INDENT>detail = json.loads(exception.content)<EOL>status = detail.get('<STR_LIT:error>', {}).get('<STR_LIT:status>')<EOL>if status == FAILED_PRECONDITION_STATUS:<EOL><INDENT>msg = '<STR_LIT>'<EOL><DEDENT><DEDENT>failed.append({'<STR_LIT:name>': request_id, '<STR_LIT>': msg})<EOL><DEDENT>else:<EOL><INDENT>canceled.append({'<STR_LIT:name>': request_id})<EOL><DEDENT>return<EOL><DEDENT>batch = batch_fn(callback=handle_cancel_response)<EOL>ops_by_name = {}<EOL>for op in ops:<EOL><INDENT>op_name = op.get_field('<STR_LIT>')<EOL>ops_by_name[op_name] = op<EOL>batch.add(cancel_fn(name=op_name, body={}), request_id=op_name)<EOL><DEDENT>batch.execute()<EOL>canceled_ops = [ops_by_name[op['<STR_LIT:name>']] for op in canceled]<EOL>error_messages = []<EOL>for fail in failed:<EOL><INDENT>op = ops_by_name[fail['<STR_LIT:name>']]<EOL>error_messages.append("<STR_LIT>" %<EOL>(get_operation_full_job_id(op), fail['<STR_LIT>']))<EOL><DEDENT>return canceled_ops, error_messages<EOL>

Cancel a batch of operations.

    Args:
      batch_fn: API-specific batch function.
      cancel_fn: API-specific cancel function.
      ops: A list of operations to cancel.

    Returns:
      A list of operations canceled and a list of error messages.


if not credentials:<EOL><INDENT>credentials = oauth2client.client.GoogleCredentials.get_application_default(<EOL>)<EOL><DEDENT>return apiclient.discovery.build(<EOL>api_name, api_version, credentials=credentials)<EOL>

Configures genomics API client.

    Args:
      api_name: Name of the Google API (for example: "genomics")
      api_version: Version of the API (for example: "v2alpha1")
      credentials: Credentials to be used for the gcloud API calls.

    Returns:
      A configured Google Genomics API client with appropriate credentials.


print(msg, file=sys.stderr)<EOL>

Utility routine to emit messages to stderr.


<EOL>canceled_ops = []<EOL>error_messages = []<EOL>max_batch = <NUM_LIT><EOL>total_ops = len(ops)<EOL>for first_op in range(<NUM_LIT:0>, total_ops, max_batch):<EOL><INDENT>batch_canceled, batch_messages = _cancel_batch(<EOL>batch_fn, cancel_fn, ops[first_op:first_op + max_batch])<EOL>canceled_ops.extend(batch_canceled)<EOL>error_messages.extend(batch_messages)<EOL><DEDENT>return canceled_ops, error_messages<EOL>

Cancel operations.

    Args:
      batch_fn: API-specific batch function.
      cancel_fn: API-specific cancel function.
      ops: A list of operations to cancel.

    Returns:
      A list of operations canceled and a list of error messages.


def _get_mount_actions(self, mounts, mnt_datadisk):

actions_to_add = []<EOL>for mount in mounts:<EOL><INDENT>bucket = mount.value[len('<STR_LIT>'):]<EOL>mount_path = mount.docker_path<EOL>actions_to_add.extend([<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>'.format(bucket),<EOL>flags=['<STR_LIT>', '<STR_LIT>'],<EOL>image_uri=_GCSFUSE_IMAGE,<EOL>mounts=[mnt_datadisk],<EOL>commands=[<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', bucket,<EOL>os.path.join(providers_util.DATA_MOUNT_POINT, mount_path)<EOL>]),<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>'.format(bucket),<EOL>flags=['<STR_LIT>'],<EOL>image_uri=_GCSFUSE_IMAGE,<EOL>mounts=[mnt_datadisk],<EOL>commands=[<EOL>'<STR_LIT>',<EOL>os.path.join(providers_util.DATA_MOUNT_POINT, mount_path)<EOL>])<EOL>])<EOL><DEDENT>return actions_to_add<EOL>

Returns a list of two actions per gcs bucket to mount.


def _build_user_environment(self, envs, inputs, outputs, mounts):

envs = {env.name: env.value for env in envs}<EOL>envs.update(providers_util.get_file_environment_variables(inputs))<EOL>envs.update(providers_util.get_file_environment_variables(outputs))<EOL>envs.update(providers_util.get_file_environment_variables(mounts))<EOL>return envs<EOL>

Returns a dictionary of for the user container environment.


def prepare_job_metadata(self, script, job_name, user_id, create_time):

return google_base.prepare_job_metadata(script, job_name, user_id,<EOL>create_time)<EOL>

Returns a dictionary of metadata fields for the job.


job_metadata = task_view.job_metadata<EOL>job_params = task_view.job_params<EOL>job_resources = task_view.job_resources<EOL>task_metadata = task_view.task_descriptors[<NUM_LIT:0>].task_metadata<EOL>task_params = task_view.task_descriptors[<NUM_LIT:0>].task_params<EOL>task_resources = task_view.task_descriptors[<NUM_LIT:0>].task_resources<EOL>mnt_datadisk = google_v2_pipelines.build_mount(<EOL>disk=_DATA_DISK_NAME,<EOL>path=providers_util.DATA_MOUNT_POINT,<EOL>read_only=False)<EOL>scopes = job_resources.scopes or google_base.DEFAULT_SCOPES<EOL>labels = {<EOL>label.name: label.value if label.value else '<STR_LIT>' for label in<EOL>google_base.build_pipeline_labels(job_metadata, task_metadata)<EOL>| job_params['<STR_LIT>'] | task_params['<STR_LIT>']<EOL>}<EOL>script = task_view.job_metadata['<STR_LIT>']<EOL>user_project = task_view.job_metadata['<STR_LIT>'] or '<STR_LIT>'<EOL>envs = job_params['<STR_LIT>'] | task_params['<STR_LIT>']<EOL>inputs = job_params['<STR_LIT>'] | task_params['<STR_LIT>']<EOL>outputs = job_params['<STR_LIT>'] | task_params['<STR_LIT>']<EOL>mounts = job_params['<STR_LIT>']<EOL>gcs_mounts = param_util.get_gcs_mounts(mounts)<EOL>persistent_disk_mount_params = param_util.get_persistent_disk_mounts(mounts)<EOL>persistent_disks = [<EOL>google_v2_pipelines.build_disk(<EOL>name=disk.name.replace('<STR_LIT:_>', '<STR_LIT:->'),  <EOL>size_gb=disk.disk_size or job_model.DEFAULT_MOUNTED_DISK_SIZE,<EOL>source_image=disk.value,<EOL>disk_type=disk.disk_type or job_model.DEFAULT_DISK_TYPE)<EOL>for disk in persistent_disk_mount_params<EOL>]<EOL>persistent_disk_mounts = [<EOL>google_v2_pipelines.build_mount(<EOL>disk=persistent_disk.get('<STR_LIT:name>'),<EOL>path=os.path.join(providers_util.DATA_MOUNT_POINT,<EOL>persistent_disk_mount_param.docker_path),<EOL>read_only=True)<EOL>for persistent_disk, persistent_disk_mount_param in zip(<EOL>persistent_disks, persistent_disk_mount_params)<EOL>]<EOL>optional_actions = <NUM_LIT:0><EOL>if job_resources.ssh:<EOL><INDENT>optional_actions += <NUM_LIT:1><EOL><DEDENT>mount_actions = self._get_mount_actions(gcs_mounts, mnt_datadisk)<EOL>optional_actions += len(mount_actions)<EOL>user_action = <NUM_LIT:4> + optional_actions<EOL>final_logging_action = <NUM_LIT:6> + optional_actions<EOL>logging_cmd = _LOGGING_CMD.format(<EOL>log_cp_fn=_GSUTIL_CP_FN,<EOL>log_cp_cmd=_LOG_CP_CMD.format(<EOL>user_action=user_action, logging_action='<STR_LIT>'))<EOL>continuous_logging_cmd = _CONTINUOUS_LOGGING_CMD.format(<EOL>log_msg_fn=_LOG_MSG_FN,<EOL>log_cp_fn=_GSUTIL_CP_FN,<EOL>log_cp_cmd=_LOG_CP_CMD.format(<EOL>user_action=user_action,<EOL>logging_action='<STR_LIT>'),<EOL>final_logging_action=final_logging_action,<EOL>log_interval=job_resources.log_interval or '<STR_LIT>')<EOL>logging_env = self._get_logging_env(task_resources.logging_path.uri,<EOL>user_project)<EOL>script_path = os.path.join(providers_util.SCRIPT_DIR, script.name)<EOL>prepare_command = _PREPARE_CMD.format(<EOL>log_msg_fn=_LOG_MSG_FN,<EOL>mk_runtime_dirs=_MK_RUNTIME_DIRS_CMD,<EOL>script_var=_SCRIPT_VARNAME,<EOL>python_decode_script=_PYTHON_DECODE_SCRIPT,<EOL>script_path=script_path,<EOL>mk_io_dirs=_MK_IO_DIRS)<EOL>prepare_env = self._get_prepare_env(script, task_view, inputs, outputs,<EOL>mounts)<EOL>localization_env = self._get_localization_env(inputs, user_project)<EOL>user_environment = self._build_user_environment(envs, inputs, outputs,<EOL>mounts)<EOL>delocalization_env = self._get_delocalization_env(outputs, user_project)<EOL>actions = []<EOL>actions.append(<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>',<EOL>flags='<STR_LIT>',<EOL>image_uri=_CLOUD_SDK_IMAGE,<EOL>environment=logging_env,<EOL>entrypoint='<STR_LIT>',<EOL>commands=['<STR_LIT:-c>', continuous_logging_cmd]))<EOL>if job_resources.ssh:<EOL><INDENT>actions.append(<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>',<EOL>image_uri=_SSH_IMAGE,<EOL>mounts=[mnt_datadisk],<EOL>entrypoint='<STR_LIT>',<EOL>port_mappings={_DEFAULT_SSH_PORT: _DEFAULT_SSH_PORT},<EOL>flags='<STR_LIT>'))<EOL><DEDENT>actions.append(<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>',<EOL>image_uri=_PYTHON_IMAGE,<EOL>mounts=[mnt_datadisk],<EOL>environment=prepare_env,<EOL>entrypoint='<STR_LIT>',<EOL>commands=['<STR_LIT:-c>', prepare_command]),)<EOL>actions.extend(mount_actions)<EOL>actions.extend([<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>',<EOL>image_uri=_CLOUD_SDK_IMAGE,<EOL>mounts=[mnt_datadisk],<EOL>environment=localization_env,<EOL>entrypoint='<STR_LIT>',<EOL>commands=[<EOL>'<STR_LIT:-c>',<EOL>_LOCALIZATION_CMD.format(<EOL>log_msg_fn=_LOG_MSG_FN,<EOL>recursive_cp_fn=_GSUTIL_RSYNC_FN,<EOL>cp_fn=_GSUTIL_CP_FN,<EOL>cp_loop=_LOCALIZATION_LOOP)<EOL>]),<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>',<EOL>image_uri=job_resources.image,<EOL>mounts=[mnt_datadisk] + persistent_disk_mounts,<EOL>environment=user_environment,<EOL>entrypoint='<STR_LIT>',<EOL>commands=[<EOL>'<STR_LIT>', '<STR_LIT:-c>',<EOL>_USER_CMD.format(<EOL>tmp_dir=providers_util.TMP_DIR,<EOL>working_dir=providers_util.WORKING_DIR,<EOL>user_script=script_path)<EOL>]),<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>',<EOL>image_uri=_CLOUD_SDK_IMAGE,<EOL>mounts=[mnt_datadisk],<EOL>environment=delocalization_env,<EOL>entrypoint='<STR_LIT>',<EOL>commands=[<EOL>'<STR_LIT:-c>',<EOL>_LOCALIZATION_CMD.format(<EOL>log_msg_fn=_LOG_MSG_FN,<EOL>recursive_cp_fn=_GSUTIL_RSYNC_FN,<EOL>cp_fn=_GSUTIL_CP_FN,<EOL>cp_loop=_DELOCALIZATION_LOOP)<EOL>]),<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>',<EOL>flags='<STR_LIT>',<EOL>image_uri=_CLOUD_SDK_IMAGE,<EOL>environment=logging_env,<EOL>entrypoint='<STR_LIT>',<EOL>commands=['<STR_LIT:-c>', logging_cmd]),<EOL>])<EOL>assert len(actions) - <NUM_LIT:2> == user_action<EOL>assert len(actions) == final_logging_action<EOL>disks = [<EOL>google_v2_pipelines.build_disk(<EOL>_DATA_DISK_NAME,<EOL>job_resources.disk_size,<EOL>source_image=None,<EOL>disk_type=job_resources.disk_type or job_model.DEFAULT_DISK_TYPE)<EOL>]<EOL>disks.extend(persistent_disks)<EOL>network = google_v2_pipelines.build_network(<EOL>job_resources.network, job_resources.subnetwork,<EOL>job_resources.use_private_address)<EOL>if job_resources.machine_type:<EOL><INDENT>machine_type = job_resources.machine_type<EOL><DEDENT>elif job_resources.min_cores or job_resources.min_ram:<EOL><INDENT>machine_type = GoogleV2CustomMachine.build_machine_type(<EOL>job_resources.min_cores, job_resources.min_ram)<EOL><DEDENT>else:<EOL><INDENT>machine_type = job_model.DEFAULT_MACHINE_TYPE<EOL><DEDENT>accelerators = None<EOL>if job_resources.accelerator_type:<EOL><INDENT>accelerators = [<EOL>google_v2_pipelines.build_accelerator(job_resources.accelerator_type,<EOL>job_resources.accelerator_count)<EOL>]<EOL><DEDENT>service_account = google_v2_pipelines.build_service_account(<EOL>job_resources.service_account or '<STR_LIT:default>', scopes)<EOL>resources = google_v2_pipelines.build_resources(<EOL>self._project,<EOL>job_resources.regions,<EOL>google_base.get_zones(job_resources.zones),<EOL>google_v2_pipelines.build_machine(<EOL>network=network,<EOL>machine_type=machine_type,<EOL>preemptible=job_resources.preemptible,<EOL>service_account=service_account,<EOL>boot_disk_size_gb=job_resources.boot_disk_size,<EOL>disks=disks,<EOL>accelerators=accelerators,<EOL>nvidia_driver_version=job_resources.nvidia_driver_version,<EOL>labels=labels,<EOL>cpu_platform=job_resources.cpu_platform),<EOL>)<EOL>pipeline = google_v2_pipelines.build_pipeline(actions, resources, None,<EOL>job_resources.timeout)<EOL>return {'<STR_LIT>': pipeline, '<STR_LIT>': labels}<EOL>

Returns a Pipeline objects for the task.


if cores == <NUM_LIT:1> or cores % <NUM_LIT:2> == <NUM_LIT:0>:<EOL><INDENT>return cores<EOL><DEDENT>else:<EOL><INDENT>return cores + <NUM_LIT:1><EOL><DEDENT>


<EOL>ops_filter = self._build_query_filter(<EOL>statuses, user_ids, job_ids, job_names, task_ids, task_attempts, labels,<EOL>create_time_min, create_time_max)<EOL>page_token = None<EOL>tasks_yielded = <NUM_LIT:0><EOL>while True:<EOL><INDENT>max_to_fetch = None<EOL>if max_tasks:<EOL><INDENT>max_to_fetch = max_tasks - tasks_yielded<EOL><DEDENT>ops, page_token = self._operations_list(ops_filter, max_to_fetch,<EOL>page_size, page_token)<EOL>for op in ops:<EOL><INDENT>yield op<EOL>tasks_yielded += <NUM_LIT:1><EOL><DEDENT>assert (max_tasks >= tasks_yielded or not max_tasks)<EOL>if not page_token or <NUM_LIT:0> < max_tasks <= tasks_yielded:<EOL><INDENT>break<EOL><DEDENT><DEDENT>

Yields operations based on the input criteria.

        If any of the filters are empty or {'*'}, then no filtering is performed on
        that field. Filtering by both a job id list and job name list is
        unsupported.

        Args:
          statuses: {'*'}, or a list of job status strings to return. Valid
            status strings are 'RUNNING', 'SUCCESS', 'FAILURE', or 'CANCELED'.
          user_ids: a list of ids for the user(s) who launched the job.
          job_ids: a list of job ids to return.
          job_names: a list of job names to return.
          task_ids: a list of specific tasks within the specified job(s) to return.
          task_attempts: a list of specific attempts within the specified tasks(s)
            to return.
          labels: a list of LabelParam with user-added labels. All labels must
                  match the task being fetched.
          create_time_min: a timezone-aware datetime value for the earliest create
                           time of a task, inclusive.
          create_time_max: a timezone-aware datetime value for the most recent
                           create time of a task, inclusive.
          max_tasks: the maximum number of job tasks to return or 0 for no limit.
          page_size: the page size to use for each query to the pipelins API.

        Raises:
          ValueError: if both a job id list and a job name list are provided

        Yeilds:
          Genomics API Operations objects.


@classmethod<EOL><INDENT>def build_machine_type(cls, min_cores, min_ram):<DEDENT>

min_cores = min_cores or job_model.DEFAULT_MIN_CORES<EOL>min_ram = min_ram or job_model.DEFAULT_MIN_RAM<EOL>min_ram *= GoogleV2CustomMachine._MB_PER_GB<EOL>cores = cls._validate_cores(min_cores)<EOL>ram = cls._validate_ram(min_ram)<EOL>memory_to_cpu_ratio = ram / cores<EOL>if memory_to_cpu_ratio < GoogleV2CustomMachine._MIN_MEMORY_PER_CPU:<EOL><INDENT>adjusted_ram = GoogleV2CustomMachine._MIN_MEMORY_PER_CPU * cores<EOL>ram = cls._validate_ram(adjusted_ram)<EOL><DEDENT>elif memory_to_cpu_ratio > GoogleV2CustomMachine._MAX_MEMORY_PER_CPU:<EOL><INDENT>adjusted_cores = math.ceil(<EOL>ram / GoogleV2CustomMachine._MAX_MEMORY_PER_CPU)<EOL>cores = cls._validate_cores(adjusted_cores)<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT>return '<STR_LIT>'.format(int(cores), int(ram))<EOL>


def _get_localization_env(self, inputs, user_project):

<EOL>non_empty_inputs = [var for var in inputs if var.value]<EOL>env = {'<STR_LIT>': str(len(non_empty_inputs))}<EOL>for idx, var in enumerate(non_empty_inputs):<EOL><INDENT>env['<STR_LIT>'.format(idx)] = var.name<EOL>env['<STR_LIT>'.format(idx)] = str(int(var.recursive))<EOL>env['<STR_LIT>'.format(idx)] = var.value<EOL>dst = os.path.join(providers_util.DATA_MOUNT_POINT, var.docker_path)<EOL>path, filename = os.path.split(dst)<EOL>if '<STR_LIT:*>' in filename:<EOL><INDENT>dst = '<STR_LIT>'.format(path)<EOL><DEDENT>env['<STR_LIT>'.format(idx)] = dst<EOL><DEDENT>env['<STR_LIT>'] = user_project<EOL>return env<EOL>

Return a dict with variables for the 'localization' action.


<EOL>user_image = google_v2_operations.get_action_image(self._op,<EOL>_ACTION_USER_COMMAND)<EOL>need_ok = google_v2_operations.is_success(self._op)<EOL>events = {}<EOL>for event in google_v2_operations.get_events(self._op):<EOL><INDENT>if self._filter(event):<EOL><INDENT>continue<EOL><DEDENT>mapped, match = self._map(event)<EOL>name = mapped['<STR_LIT:name>']<EOL>if name == '<STR_LIT>':<EOL><INDENT>if not need_ok or '<STR_LIT>' in events:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>if name == '<STR_LIT>':<EOL><INDENT>if match.group(<NUM_LIT:1>) != user_image:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>events[name] = mapped<EOL><DEDENT>return sorted(events.values(), key=operator.itemgetter('<STR_LIT>'))<EOL>

Filter the granular v2 events down to events of interest.

        Filter through the large number of granular events returned by the
        pipelines API, and extract only those that are interesting to a user. This
        is implemented by filtering out events which are known to be uninteresting
        (i.e. the default actions run for every job) and by explicitly matching
        specific events which are interesting and mapping those to v1 style naming.

        Events which are not whitelisted or blacklisted will still be output,
        meaning any events which are added in the future won't be masked.
        We don't want to suppress display of events that we don't recognize.
        They may be important.

        Returns:
          A list of maps containing the normalized, filtered events.


def _get_logging_env(self, logging_uri, user_project):

if not logging_uri.endswith('<STR_LIT>'):<EOL><INDENT>raise ValueError('<STR_LIT>'.format(logging_uri))<EOL><DEDENT>logging_prefix = logging_uri[:-len('<STR_LIT>')]<EOL>return {<EOL>'<STR_LIT>': '<STR_LIT>'.format(logging_prefix),<EOL>'<STR_LIT>': '<STR_LIT>'.format(logging_prefix),<EOL>'<STR_LIT>': '<STR_LIT>'.format(logging_prefix),<EOL>'<STR_LIT>': user_project,<EOL>}<EOL>

Returns the environment for actions that copy logging files.


msg = None<EOL>action = None<EOL>if not google_v2_operations.is_done(self._op):<EOL><INDENT>last_event = google_v2_operations.get_last_event(self._op)<EOL>if last_event:<EOL><INDENT>msg = last_event['<STR_LIT:description>']<EOL>action_id = last_event.get('<STR_LIT>', {}).get('<STR_LIT>')<EOL>if action_id:<EOL><INDENT>action = google_v2_operations.get_action_by_id(self._op, action_id)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>msg = '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>failed_events = google_v2_operations.get_failed_events(self._op)<EOL>if failed_events:<EOL><INDENT>failed_event = failed_events[-<NUM_LIT:1>]<EOL>msg = failed_event.get('<STR_LIT>', {}).get('<STR_LIT>')<EOL>action_id = failed_event.get('<STR_LIT>', {}).get('<STR_LIT>')<EOL>if action_id:<EOL><INDENT>action = google_v2_operations.get_action_by_id(self._op, action_id)<EOL><DEDENT><DEDENT>if not msg:<EOL><INDENT>error = google_v2_operations.get_error(self._op)<EOL>if error:<EOL><INDENT>msg = error['<STR_LIT:message>']<EOL><DEDENT>else:<EOL><INDENT>msg = '<STR_LIT>'<EOL><DEDENT><DEDENT><DEDENT>return msg, action<EOL>

Returns the most relevant status string and failed action.

        This string is meant for display only.

        Returns:
          A printable status string and name of failed action (if any).


return self._raw._asdict()<EOL>

Return a provider-specific representation of task data.

        Returns:
          string of task data from the provider.


def _task_directory(self, job_id, task_id, task_attempt):

dir_name = '<STR_LIT>' if task_id is None else str(task_id)<EOL>if task_attempt:<EOL><INDENT>dir_name = '<STR_LIT>' % (dir_name, task_attempt)<EOL><DEDENT>return self._provider_root() + '<STR_LIT:/>' + job_id + '<STR_LIT:/>' + dir_name<EOL>

The local dir for staging files for that particular task.


if '<STR_LIT:.>' in task_dir:<EOL><INDENT>return task_dir.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>return task_dir, None<EOL><DEDENT>

Return task_id and task_attempt from dir_name.


def _localize_inputs_recursive_command(self, task_dir, inputs):

data_dir = os.path.join(task_dir, _DATA_SUBDIR)<EOL>provider_commands = [<EOL>providers_util.build_recursive_localize_command(data_dir, inputs,<EOL>file_provider)<EOL>for file_provider in _SUPPORTED_INPUT_PROVIDERS<EOL>]<EOL>return '<STR_LIT:\n>'.join(provider_commands)<EOL>

Returns a command that will stage recursive inputs.


def _datetime_in_range(self, dt, dt_min=None, dt_max=None):

<EOL>dt = dt.replace(microsecond=<NUM_LIT:0>)<EOL>if dt_min:<EOL><INDENT>dt_min = dt_min.replace(microsecond=<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>dt_min = dsub_util.replace_timezone(datetime.datetime.min, pytz.utc)<EOL><DEDENT>if dt_max:<EOL><INDENT>dt_max = dt_max.replace(microsecond=<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>dt_max = dsub_util.replace_timezone(datetime.datetime.max, pytz.utc)<EOL><DEDENT>return dt_min <= dt <= dt_max<EOL>

Determine if the provided time is within the range, inclusive.


def _delocalize_outputs_commands(self, task_dir, outputs, user_project):

commands = []<EOL>for o in outputs:<EOL><INDENT>if o.recursive or not o.value:<EOL><INDENT>continue<EOL><DEDENT>dest_path = o.uri.path<EOL>local_path = task_dir + '<STR_LIT:/>' + _DATA_SUBDIR + '<STR_LIT:/>' + o.docker_path<EOL>if o.file_provider == job_model.P_LOCAL:<EOL><INDENT>commands.append('<STR_LIT>' % dest_path)<EOL><DEDENT>if o.file_provider in [job_model.P_LOCAL, job_model.P_GCS]:<EOL><INDENT>if user_project:<EOL><INDENT>command = '<STR_LIT>' % (user_project, local_path,<EOL>dest_path)<EOL><DEDENT>else:<EOL><INDENT>command = '<STR_LIT>' % (local_path, dest_path)<EOL><DEDENT>commands.append(command)<EOL><DEDENT><DEDENT>return '<STR_LIT:\n>'.join(commands)<EOL>


<EOL>tasks.sort(key=_task_sort_function, reverse=True)<EOL>


return {<EOL>'<STR_LIT>': project,<EOL>'<STR_LIT>': regions,<EOL>'<STR_LIT>': zones,<EOL>'<STR_LIT>': virtual_machine,<EOL>}<EOL>

Build a Resources object for a Pipeline request.

    Args:
      project (str): Cloud project for the Pipeline to run in.
      regions (List[str]): List of regions for the pipeline to run in.
      zones (List[str]): List of zones for the pipeline to run in.
      virtual_machine(str): Virtual machine type string.

    Returns:
      An object representing a Resource.


def build_pipeline(actions, resources, environment, timeout):

return {<EOL>'<STR_LIT>': actions,<EOL>'<STR_LIT>': resources,<EOL>'<STR_LIT>': environment,<EOL>'<STR_LIT>': timeout,<EOL>}<EOL>

Build an Pipeline argument for a Pipeline request.

    Args:
      actions (List): A list of actions to execute.
      resources (dict): An object indicating pipeline resources.
      environment (dict[str,str]): The environment to pass into the container.
      timeout (str): A duration in seconds with up to nine fractional digits,
        terminated by 's'.

    Returns:
      An object representing a Pipelines Resource.


def parse_args(parser, provider_required_args, argv):

<EOL>epilog = '<STR_LIT>'<EOL>for provider in provider_required_args:<EOL><INDENT>epilog += '<STR_LIT>' % (provider, provider_required_args[provider])<EOL><DEDENT>parser.epilog = epilog<EOL>args = parser.parse_args(argv)<EOL>for arg in provider_required_args[args.provider]:<EOL><INDENT>if not args.__getattribute__(arg):<EOL><INDENT>parser.error('<STR_LIT>' % arg)<EOL><DEDENT><DEDENT>return args<EOL>

Add provider required arguments epilog message, parse, and validate.


provider = getattr(args, '<STR_LIT>', '<STR_LIT>')<EOL>if provider == '<STR_LIT>':<EOL><INDENT>return google.GoogleJobProvider(<EOL>getattr(args, '<STR_LIT>', False),<EOL>getattr(args, '<STR_LIT>', False), args.project)<EOL><DEDENT>elif provider == '<STR_LIT>':<EOL><INDENT>return google_v2.GoogleV2JobProvider(<EOL>getattr(args, '<STR_LIT>', False), getattr(args, '<STR_LIT>', False),<EOL>args.project)<EOL><DEDENT>elif provider == '<STR_LIT>':<EOL><INDENT>return local.LocalJobProvider(resources)<EOL><DEDENT>elif provider == '<STR_LIT>':<EOL><INDENT>return test_fails.FailsJobProvider()<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' + provider)<EOL><DEDENT>

Returns a provider for job submission requests.


parser = argparse.ArgumentParser(prog=prog, formatter_class=DsubHelpFormatter)<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>default='<STR_LIT>',<EOL>choices=['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'],<EOL>help="""<STR_LIT>""",<EOL>metavar='<STR_LIT>')<EOL>return parser<EOL>

Create an argument parser, adding in the list of providers.


<EOL>return get_dstat_provider_args(provider_type, project)<EOL>

A string with the arguments to point ddel to the same provider+project.


statuses = None if statuses == {'<STR_LIT:*>'} else statuses<EOL>user_ids = None if user_ids == {'<STR_LIT:*>'} else user_ids<EOL>job_ids = None if job_ids == {'<STR_LIT:*>'} else job_ids<EOL>job_names = None if job_names == {'<STR_LIT:*>'} else job_names<EOL>task_ids = None if task_ids == {'<STR_LIT:*>'} else task_ids<EOL>task_attempts = None if task_attempts == {'<STR_LIT:*>'} else task_attempts<EOL>if labels or create_time_min or create_time_max:<EOL><INDENT>raise NotImplementedError(<EOL>'<STR_LIT>')<EOL><DEDENT>operations = [<EOL>x for x in self._operations<EOL>if ((not statuses or x.get_field('<STR_LIT:status>', (None, None))[<NUM_LIT:0>] in statuses<EOL>) and (not user_ids or x.get_field('<STR_LIT:user>', None) in user_ids) and<EOL>(not job_ids or x.get_field('<STR_LIT>', None) in job_ids) and<EOL>(not job_names or x.get_field('<STR_LIT>', None) in job_names) and<EOL>(not task_ids or x.get_field('<STR_LIT>', None) in task_ids) and<EOL>(not task_attempts or<EOL>x.get_field('<STR_LIT>', None) in task_attempts))<EOL>]<EOL>if max_tasks > <NUM_LIT:0>:<EOL><INDENT>operations = operations[:max_tasks]<EOL><DEDENT>return operations<EOL>

Return a list of operations. See base.py for additional detail.


self._operations = [StubTask(o) for o in ops]<EOL>

Set the state of the fictional world.

        Args:
         ops: a list of dict, each representing an operation.

        Operations can have the following fields:
           - status: tuple (string,date)
           - user: string
           - job-id: string
           - job-name: string
           - task-id: string
           - task-attempt: integer
           - labels: list<dict>
           - status-message: string
           - error-messages : list of string


return is_done(op) and ('<STR_LIT:error>' not in op)<EOL>

Return whether the operation has completed successfully.


action = _get_action_by_name(op, name)<EOL>if action:<EOL><INDENT>return action.get('<STR_LIT>')<EOL><DEDENT>


last_update = get_end_time(op)<EOL>if not last_update:<EOL><INDENT>last_event = get_last_event(op)<EOL>if last_event:<EOL><INDENT>last_update = last_event['<STR_LIT>']<EOL><DEDENT><DEDENT>if not last_update:<EOL><INDENT>last_update = get_create_time(op)<EOL><DEDENT>return last_update<EOL>

Return the most recent timestamp in the operation.


return '<STR_LIT>'.format(comparator, create_time)<EOL>

Return a valid createTime filter for operations.list().


return get_metadata_type(<EOL>op) == '<STR_LIT>'<EOL>

Check that an operation is a genomics pipeline run.

    An operation is a Genomics Pipeline run if the request metadata's @type
    is "type.googleapis.com/google.genomics.v2alpha1.Metadata".

    Args:
      op: a pipelines operation.

    Returns:
      Boolean, true if the operation is a RunPipelineRequest.


return '<STR_LIT>'.format(label_key, label_value)<EOL>

Return a valid label filter for operations.list().


return op.get('<STR_LIT>', {}).get('<STR_LIT>', {})<EOL>


actions = get_actions(op)<EOL>for action in actions:<EOL><INDENT>if action.get('<STR_LIT:name>') == name:<EOL><INDENT>return action<EOL><DEDENT><DEDENT>

Return the value for the specified action.


return op.get('<STR_LIT>', {}).get('<STR_LIT>')<EOL>

Return the start time string of the operation.


return get_labels(op).get(name)<EOL>

Return the value for the specified label.


Return the end time string of the operation.


if not is_pipeline(op):<EOL><INDENT>return False<EOL><DEDENT>for name in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>if not get_label(op, name):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>

Determine if a pipelines operation is a dsub request.

    We don't have a rigorous way to identify an operation as being submitted
    by dsub. Our best option is to check for certain fields that have always
    been part of dsub operations.

    - labels: job-id, job-name, and user-id have always existed. The dsub-version
              label has always existed for the google-v2 provider.

    Args:
      op: a pipelines operation.

    Returns:
      Boolean, true if the pipeline run was generated by dsub.


events = get_events(op)<EOL>if events:<EOL><INDENT>return [<EOL>e for e in events if int(e.get('<STR_LIT>', {}).get('<STR_LIT>', <NUM_LIT:0>)) != <NUM_LIT:0><EOL>]<EOL><DEDENT>return None<EOL>

Return the events (if any) with a non-zero exitStatus.


return [t.raw_task_data() for t in tasklist]<EOL>

Convert returned operations to raw operations.


if seconds < <NUM_LIT:0>:<EOL><INDENT>raise IOError("<STR_LIT>")<EOL><DEDENT>target = self._now + seconds<EOL>if target < self._next:<EOL><INDENT>self._now = target<EOL>return<EOL><DEDENT>self._now = target<EOL>for step in self._chronology:<EOL><INDENT>self._next += step<EOL>if self._next > target:<EOL><INDENT>return<EOL><DEDENT><DEDENT>raise BaseException("<STR_LIT>")<EOL>

Sleep for this many fictitious seconds.

        The world will update accordingly.

        Args:
          seconds: how many seconds to pretend sleep for.

        Raises:
          IOError: for negative delays (matches time.sleep).
          BaseException: at the end of times, to detect infinite loop bugs
        and the like.


self._chronology = chronology<EOL>self._now = <NUM_LIT:0><EOL>self._next = six.advance_iterator(self._chronology)<EOL>

The chronology is a generator that updates the world.

        Yield how many seconds of fake time elapse. Think of each yield as a
        "sleep". We read the very first value immediately, so that the world can be
        initialized in the chronology.

        Args:
          chronology: a generator that updates the state of the world and then
          sleeps via yielding.


def expand_tsv_fields(newenv, tmpl_file, tsv_file):

with open(tmpl_file, '<STR_LIT:r>') as f:<EOL><INDENT>input_lines = f.readlines()<EOL><DEDENT>with open(tsv_file, '<STR_LIT:w>') as f:<EOL><INDENT>f.write(input_lines[<NUM_LIT:0>])<EOL>for line in input_lines[<NUM_LIT:1>:]:<EOL><INDENT>curr = []<EOL>for field in line.split('<STR_LIT:\t>'):<EOL><INDENT>cell = subprocess.check_output(<EOL>'<STR_LIT>' % field, env=newenv, shell=True)<EOL>curr.append(cell)<EOL><DEDENT>f.write('<STR_LIT:\t>'.join(curr) + '<STR_LIT:\n>')<EOL><DEDENT><DEDENT>

Write a TSV file, expanding environment variables in the template.


if sys.version_info[<NUM_LIT:0>] < <NUM_LIT:3>:<EOL><INDENT>return stdoutbytes<EOL><DEDENT>encoding = sys.stdout.encoding if sys.stdout.encoding else '<STR_LIT:utf-8>'<EOL>return stdoutbytes.decode(encoding)<EOL>

Convert stdout to a string in python 2 or 3.


if value.startswith('<STR_LIT:">') and value.endswith('<STR_LIT:">'):<EOL><INDENT>return value[<NUM_LIT:1>:-<NUM_LIT:1>].replace('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>return value<EOL>

Remove surrounding single or double quotes.


loggers = list(Logger.manager.loggerDict.items())<EOL>loggers.append(("<STR_LIT:root>", getLogger()))<EOL>level_filter = LevelFilter(levels)<EOL>for logger_name, logger in loggers:<EOL><INDENT>for handler in getattr(logger, "<STR_LIT>", []):<EOL><INDENT>handler.addFilter(level_filter)<EOL><DEDENT><DEDENT>

see --quiet flag help for what this does


<EOL>sys.stdout.write(line)<EOL>sys.stdout.flush()<EOL>


def execute(self, arg_str='<STR_LIT>', **kwargs):

cmd = "<STR_LIT>".format(self.cmd_prefix, self.script, arg_str)<EOL>expected_ret_code = kwargs.pop('<STR_LIT:code>', <NUM_LIT:0>)<EOL>environ = self.environ<EOL>for k in list(kwargs.keys()):<EOL><INDENT>if k.isupper():<EOL><INDENT>environ[k] = kwargs.pop(k)<EOL><DEDENT><DEDENT>kwargs.setdefault("<STR_LIT>", subprocess.STDOUT)<EOL>kwargs["<STR_LIT>"] = True<EOL>kwargs["<STR_LIT>"] = subprocess.PIPE<EOL>kwargs["<STR_LIT>"] = self.cwd<EOL>kwargs["<STR_LIT>"] = environ<EOL>process = None<EOL>self.buf = deque(maxlen=self.bufsize)<EOL>try:<EOL><INDENT>process = subprocess.Popen(<EOL>cmd,<EOL>**kwargs<EOL>)<EOL>for line in iter(process.stdout.readline, b"<STR_LIT>"):<EOL><INDENT>line = line.decode(self.encoding)<EOL>self.buf.append(line.rstrip())<EOL>yield line<EOL><DEDENT>process.wait()<EOL>if process.returncode != expected_ret_code:<EOL><INDENT>if process.returncode > <NUM_LIT:0>:<EOL><INDENT>raise RuntimeError("<STR_LIT>".format(<EOL>cmd,<EOL>process.returncode,<EOL>self.output<EOL>))<EOL><DEDENT><DEDENT><DEDENT>except subprocess.CalledProcessError as e:<EOL><INDENT>if e.returncode != expected_ret_code:<EOL><INDENT>raise RuntimeError("<STR_LIT>".format(<EOL>cmd,<EOL>e.returncode,<EOL>self.output<EOL>))<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>if process:<EOL><INDENT>process.stdout.close()<EOL><DEDENT><DEDENT>

runs the passed in arguments and returns an iterator on the output of
        running command


exc_info = kwargs.pop("<STR_LIT>", False)<EOL>stderr.warning(str(format_msg).format(*args, **kwargs), exc_info=exc_info)<EOL>


ret = []<EOL>prefix = kwargs.get('<STR_LIT>', '<STR_LIT>')<EOL>buf_count = kwargs.get('<STR_LIT>', <NUM_LIT:2>)<EOL>if len(columns) == <NUM_LIT:1>:<EOL><INDENT>columns = list(columns[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>columns = list(zip(*columns))<EOL><DEDENT>headers = kwargs.get("<STR_LIT>", [])<EOL>if headers:<EOL><INDENT>columns.insert(<NUM_LIT:0>, headers)<EOL><DEDENT>widths = kwargs.get("<STR_LIT>", [])<EOL>row_counts = Counter()<EOL>for i in range(len(widths)):<EOL><INDENT>row_counts[i] = int(widths[i])<EOL><DEDENT>width = int(kwargs.get("<STR_LIT:width>", <NUM_LIT:0>))<EOL>for row in columns:<EOL><INDENT>for i, c in enumerate(row):<EOL><INDENT>if isinstance(c, basestring):<EOL><INDENT>cl = len(c)<EOL><DEDENT>else:<EOL><INDENT>cl = len(str(c))<EOL><DEDENT>if cl > row_counts[i]:<EOL><INDENT>row_counts[i] = cl<EOL><DEDENT><DEDENT><DEDENT>width = int(kwargs.get("<STR_LIT:width>", <NUM_LIT:0>))<EOL>if width:<EOL><INDENT>for i in row_counts:<EOL><INDENT>if row_counts[i] < width:<EOL><INDENT>row_counts[i] = width<EOL><DEDENT><DEDENT><DEDENT>def colstr(c):<EOL><INDENT>if isinstance(c, basestring): return c<EOL>return str(c)<EOL><DEDENT>def rowstr(row, prefix, row_counts):<EOL><INDENT>row_format = prefix<EOL>cols = list(map(colstr, row))<EOL>for i in range(len(row_counts)):<EOL><INDENT>c = cols[i]<EOL>if re.match(r"<STR_LIT>", c):<EOL><INDENT>if i == <NUM_LIT:0>:<EOL><INDENT>row_format += "<STR_LIT>" + str(row_counts[i]) + "<STR_LIT:}>"<EOL><DEDENT>else:<EOL><INDENT>row_format += "<STR_LIT>" + str(row_counts[i] + buf_count) + "<STR_LIT:}>"<EOL><DEDENT><DEDENT>else:<EOL><INDENT>row_format += "<STR_LIT>" + str(row_counts[i] + buf_count) + "<STR_LIT:}>"<EOL><DEDENT><DEDENT>return row_format.format(*cols)<EOL><DEDENT>for row in columns:<EOL><INDENT>ret.append(rowstr(row, prefix, row_counts))<EOL><DEDENT>out(os.linesep.join(ret))<EOL>

format columned data so we can easily print it out on a console, this just takes
columns of data and it will format it into properly aligned columns, it's not
fancy, but it works for most type of strings that I need it for, like server name
lists.

other formatting options:
    http://stackoverflow.com/a/8234511/5006

other packages that probably do this way better:
    https://stackoverflow.com/a/26937531/5006

:Example:
    >>> echo.table([(1, 2), (3, 4), (5, 6), (7, 8), (9, 0)])
    1  2
    3  4
    5  6
    7  8
    9  0
    >>> echo.table([1, 3, 5, 7, 9], [2, 4, 6, 8, 0])
    1  2
    3  4
    5  6
    7  8
    9  0

:param *columns: can either be a list of rows or multiple lists representing each
    column in the table
:param **kwargs: dict
    prefix -- string -- what you want before each row (eg, a tab)
    buf_count -- integer -- how many spaces between longest col value and its neighbor
    headers -- list -- the headers you want, must match column count
    widths -- list -- the widths of each column you want to use, this doesn't have
        to match column count, so you can do something like [0, 5] to set the
        width of the second column
    width -- int -- similar to widths except it will set this value for all columns


bullets(*lines, numbers=False)<EOL>


<EOL>istdout.info(c)<EOL>

print one or more characters without a newline at the end

    example --
        for x in range(1000):
            echo.ch(".")

    c -- string -- the chars that will be output


sep = kwargs.get("<STR_LIT>", "<STR_LIT:*>")<EOL>count = kwargs.get("<STR_LIT:width>", globals()["<STR_LIT>"])<EOL>out(sep * count)<EOL>if lines:<EOL><INDENT>out(sep)<EOL>for line in lines:<EOL><INDENT>out("<STR_LIT>".format(sep, line))<EOL><DEDENT>out(sep)<EOL>out(sep * count)<EOL><DEDENT>

prints a banner

    sep -- string -- the character that will be on the line on the top and bottom
        and before any of the lines, defaults to *
    count -- integer -- the line width, defaults to 80


def increment(itr, n=<NUM_LIT:1>, format_msg="<STR_LIT>"):

for i, v in enumerate(itr, n):<EOL><INDENT>with prefix(format_msg, i):<EOL><INDENT>yield v<EOL><DEDENT><DEDENT>

Similar to enumerate but will set format_msg.format(n) into the prefix on
    each iteration

    :Example:
        for v in increment(["foo", "bar"]):
            echo.out(v) # 1. foo\n2. bar

    :param itr: iterator, any iterator you want to set a numeric prefix on on every
        iteration
    :param n: integer, the starting integer for the numeric prefix
    :param format_msg: string, this will basically do: format_msg.format(n) so there
        should only be one set of curly brackets
    :returns: yield generator


if not width: width = globals()["<STR_LIT>"]<EOL>bar("<STR_LIT:_>", width=width)<EOL>blank()<EOL>

similar to the html horizontal rule in html


stderr.exception(e)<EOL>

print an exception message to stderr (this does not honor quiet)


for x in range(count):<EOL><INDENT>out("<STR_LIT>")<EOL><DEDENT>


bullets(*lines, numbers=True)<EOL>


def out(format_msg="<STR_LIT>", *args, **kwargs):

logmethod = kwargs.get("<STR_LIT>", stdout.info)<EOL>if format_msg != "<STR_LIT>":<EOL><INDENT>if Prefix.has():<EOL><INDENT>if isinstance(format_msg, basestring):<EOL><INDENT>format_msg = Prefix.get() + format_msg<EOL><DEDENT>else:<EOL><INDENT>format_msg = Prefix.get() + str(format_msg)<EOL><DEDENT><DEDENT>if isinstance(format_msg, basestring):<EOL><INDENT>if args or kwargs:<EOL><INDENT>s = format_msg.format(*args, **kwargs)<EOL><DEDENT>else:<EOL><INDENT>s = format_msg<EOL><DEDENT>logmethod(s)<EOL><INDENT>width = globals()["<STR_LIT:width>"]<EOL>s = textwrap.fill(s, width=width)<EOL>stdout.info(s)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>logmethod(str(format_msg))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>logmethod("<STR_LIT>")<EOL><DEDENT>

print format_msg to stdout, taking into account --quiet setting


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._body = inspect.getsource(self.module)<EOL><DEDENT>return self._body<EOL>


<EOL>if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>if "<STR_LIT:__main__>" in sys.modules:<EOL><INDENT>mod = sys.modules["<STR_LIT:__main__>"]<EOL>path = self.normalize_path(mod.__file__)<EOL>if os.path.splitext(path) == os.path.splitext(self.path):<EOL><INDENT>self._module = mod<EOL><DEDENT>else:<EOL><INDENT>self._module = imp.load_source('<STR_LIT>', self.path)<EOL><DEDENT><DEDENT><DEDENT>return self._module<EOL>

load the module so we can actually run the script's function


ret = False<EOL>ast_tree = ast.parse(self.body, self.path)<EOL>calls = self._find_calls(ast_tree, __name__, "<STR_LIT>")<EOL>for call in calls:<EOL><INDENT>if re.search("<STR_LIT>".format(re.escape(call)), self.body):<EOL><INDENT>ret = True<EOL>break<EOL><DEDENT><DEDENT>return ret<EOL>

return True if this script can be run from the command line


parser = self.parser<EOL>args, kwargs = parser.parse_callback_args(raw_args)<EOL>callback = kwargs.pop("<STR_LIT>")<EOL>if parser.has_injected_quiet():<EOL><INDENT>levels = kwargs.pop("<STR_LIT>", "<STR_LIT>")<EOL>logging.inject_quiet(levels)<EOL><DEDENT>try:<EOL><INDENT>ret_code = callback(*args, **kwargs)<EOL>ret_code = int(ret_code) if ret_code else <NUM_LIT:0><EOL><DEDENT>except ArgError as e:<EOL><INDENT>echo.err("<STR_LIT>", parser.prog, str(e))<EOL>ret_code = <NUM_LIT:2><EOL><DEDENT>return ret_code<EOL>

parse and import the script, and then run the script's main function


def _find_calls(self, ast_tree, called_module, called_func):

s = set()<EOL>s.add("<STR_LIT>".format(called_module, called_func))<EOL>if hasattr(ast_tree, '<STR_LIT:body>'):<EOL><INDENT>if isinstance(ast_tree.body, collections.Iterable):<EOL><INDENT>for ast_body in ast_tree.body:<EOL><INDENT>s.update(self._find_calls(ast_body, called_module, called_func))<EOL><DEDENT><DEDENT><DEDENT>elif hasattr(ast_tree, '<STR_LIT>'):<EOL><INDENT>if hasattr(ast_tree, '<STR_LIT>'):<EOL><INDENT>if ast_tree.module == called_module:<EOL><INDENT>for ast_name in ast_tree.names:<EOL><INDENT>if ast_name.name == called_func:<EOL><INDENT>s.add(unicode(ast_name.asname if ast_name.asname is not None else ast_name.name))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for ast_name in ast_tree.names:<EOL><INDENT>if hasattr(ast_name, '<STR_LIT:name>') and (ast_name.name == called_module):<EOL><INDENT>call = "<STR_LIT>".format(<EOL>ast_name.asname if ast_name.asname is not None else ast_name.name,<EOL>called_func<EOL>)<EOL>s.add(call)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return s<EOL>

scan the abstract source tree looking for possible ways to call the called_module
and called_func

borrowed from pout

ast_tree -- _ast.* instance -- the internal ast object that is being checked, returned from compile()
    with ast.PyCF_ONLY_AST flag
called_module -- string -- we are checking the ast for imports of this module
called_func -- string -- we are checking the ast for aliases of this function
return -- set -- the list of possible calls the ast_tree could make to call the called_func


kwargs = {}<EOL>if isinstance(na, (type, types.FunctionType)):<EOL><INDENT>kwargs['<STR_LIT:type>'] = na<EOL>kwargs['<STR_LIT>'] = True<EOL>kwargs["<STR_LIT:default>"] = argparse.SUPPRESS<EOL><DEDENT>elif isinstance(na, bool):<EOL><INDENT>kwargs['<STR_LIT:action>'] = '<STR_LIT>' if na else '<STR_LIT:store_true>'<EOL>kwargs['<STR_LIT>'] = False<EOL><DEDENT>elif isinstance(na, (int, float, str)):<EOL><INDENT>kwargs['<STR_LIT:type>'] = type(na)<EOL>kwargs['<STR_LIT:default>'] = na<EOL>kwargs['<STR_LIT>'] = False<EOL><DEDENT>elif isinstance(na, (list, set)):<EOL><INDENT>na = list(na)<EOL>kwargs['<STR_LIT:action>'] = '<STR_LIT>'<EOL>kwargs['<STR_LIT>'] = True<EOL>if len(na) > <NUM_LIT:0>:<EOL><INDENT>if isinstance(na[<NUM_LIT:0>], type):<EOL><INDENT>kwargs['<STR_LIT:type>'] = na[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>kwargs['<STR_LIT:action>'] = '<STR_LIT:store>'<EOL>l = set()<EOL>ltype = None<EOL>for elt in na:<EOL><INDENT>vtype = type(elt)<EOL>l.add(elt)<EOL>if ltype is None:<EOL><INDENT>ltype = vtype<EOL><DEDENT>else:<EOL><INDENT>if ltype is not vtype:<EOL><INDENT>ltype = str<EOL><DEDENT><DEDENT><DEDENT>kwargs['<STR_LIT>'] = l<EOL>kwargs['<STR_LIT:type>'] = ltype<EOL><DEDENT><DEDENT><DEDENT>self.parser_kwargs.update(kwargs)<EOL>

this is used for introspection from the main() method when there is an
        argument with a default value, this figures out how to set up the ArgParse
        arguments


def _fill_text(self, text, width, indent):

lines = []<EOL>for line in text.splitlines(False):<EOL><INDENT>if line:<EOL><INDENT>lines.extend(textwrap.wrap(<EOL>line.strip(),<EOL>width,<EOL>initial_indent=indent,<EOL>subsequent_indent=indent<EOL>))<EOL><DEDENT>else:<EOL><INDENT>lines.append(line)<EOL><DEDENT><DEDENT>text = "<STR_LIT:\n>".join(lines)<EOL>return text<EOL>

Overridden to not get rid of newlines

        https://github.com/python/cpython/blob/2.7/Lib/argparse.py#L620


def make_buffer_get_request(self, expected_status=<NUM_LIT:200>, expected_count=None, **get_params):

response = self.client.get(<EOL>reverse('<STR_LIT>'),<EOL>get_params,<EOL>format='<STR_LIT>'<EOL>)<EOL>self.assertEqual(response.status_code, expected_status)<EOL>if expected_status == <NUM_LIT:200>:<EOL><INDENT>t_id = get_params.get("<STR_LIT>")<EOL>if expected_count is None:<EOL><INDENT>expected_count = Buffer.objects.filter(transfer_session_id=t_id).count()<EOL><DEDENT>data = json.loads(response.content.decode())<EOL>if isinstance(data, dict) and "<STR_LIT>" in data:<EOL><INDENT>data = data["<STR_LIT>"]<EOL><DEDENT>model_uuids = [d["<STR_LIT>"] for d in data]<EOL>Buffer.objects.filter(transfer_session_id=t_id, model_uuid__in=model_uuids).delete()<EOL>errors = validate_and_create_buffer_data(data, TransferSession.objects.get(id=t_id))<EOL>self.assertFalse(errors)<EOL>self.assertEqual(expected_count, Buffer.objects.filter(transfer_session_id=t_id, model_uuid__in=model_uuids).count())<EOL>self.assertEqual(expected_count, len(data))<EOL>for record in data:<EOL><INDENT>self.assertEqual(<NUM_LIT:3>, len(record["<STR_LIT>"]))<EOL><DEDENT>return data<EOL><DEDENT>

Make a GET request to the buffer endpoint. Warning: Deletes the local buffer instances before validating.


if os.path.isfile(SQLITE_VARIABLE_FILE_CACHE):<EOL><INDENT>return<EOL><DEDENT>conn = sqlite3.connect('<STR_LIT>')<EOL>low = <NUM_LIT:1><EOL>high = <NUM_LIT:1000>  <EOL>conn.execute('<STR_LIT>')<EOL>while low < high - <NUM_LIT:1>:<EOL><INDENT>guess = (low + high) // <NUM_LIT:2><EOL>try:<EOL><INDENT>statement = '<STR_LIT>' % '<STR_LIT:U+002C>'.join(['<STR_LIT:?>' for _ in range(guess)])<EOL>values = [i for i in range(guess)]<EOL>conn.execute(statement, values)<EOL><DEDENT>except sqlite3.DatabaseError as ex:<EOL><INDENT>if '<STR_LIT>' in str(ex):<EOL><INDENT>high = guess<EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>else:<EOL><INDENT>low = guess<EOL><DEDENT><DEDENT>conn.close()<EOL>with open(SQLITE_VARIABLE_FILE_CACHE, '<STR_LIT:w>') as file:<EOL><INDENT>file.write(str(low))<EOL><DEDENT>

SQLite has a limit on the max number of variables allowed for parameter substitution. This limit is usually 999, but
can be compiled to a different number. This function calculates what the max is for the sqlite version running on the device.
We use the calculated value to chunk our SQL bulk insert statements when deserializing from the store to the app layer.


for f in klass_model._meta.concrete_fields:<EOL><INDENT>if f.related_model:<EOL><INDENT>if issubclass(klass_model, f.related_model):<EOL><INDENT>return f.attname<EOL><DEDENT><DEDENT><DEDENT>return None<EOL>

Return whether this model has a self ref FK, and the name for the field


_serialize_into_store(self.profile, filter=filter)<EOL>

Takes data from app layer and serializes the models into the store.


@receiver(post_delete)<EOL>def add_to_deleted_models(sender, instance=None, *args, **kwargs):

if issubclass(sender, SyncableModel):<EOL><INDENT>instance._update_deleted_models()<EOL><DEDENT>

Whenever a model is deleted, we record its ID in a separate model for tracking purposes. During serialization, we will mark
the model as deleted in the store.


with connection.cursor() as cursor:<EOL><INDENT>DBBackend._dequeuing_delete_rmcb_records(cursor, transfersession.id)<EOL>DBBackend._dequeuing_delete_buffered_records(cursor, transfersession.id)<EOL>current_id = InstanceIDModel.get_current_instance_and_increment_counter()<EOL>DBBackend._dequeuing_merge_conflict_buffer(cursor, current_id, transfersession.id)<EOL>DBBackend._dequeuing_merge_conflict_rmcb(cursor, transfersession.id)<EOL>DBBackend._dequeuing_update_rmcs_last_saved_by(cursor, current_id, transfersession.id)<EOL>DBBackend._dequeuing_delete_mc_rmcb(cursor, transfersession.id)<EOL>DBBackend._dequeuing_delete_mc_buffer(cursor, transfersession.id)<EOL>DBBackend._dequeuing_insert_remaining_buffer(cursor, transfersession.id)<EOL>DBBackend._dequeuing_insert_remaining_rmcb(cursor, transfersession.id)<EOL>DBBackend._dequeuing_delete_remaining_rmcb(cursor, transfersession.id)<EOL>DBBackend._dequeuing_delete_remaining_buffer(cursor, transfersession.id)<EOL><DEDENT>if getattr(settings, '<STR_LIT>', True):<EOL><INDENT>_deserialize_from_store(transfersession.sync_session.profile)<EOL><DEDENT>

Takes data from the buffers and merges into the store and record max counters.


if not isinstance(data, int) and not isinstance(data, long):<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>res = []<EOL>while data > <NUM_LIT:0> or not res:<EOL><INDENT>for j in range(<NUM_LIT:5>):<EOL><INDENT>if not j % <NUM_LIT:2>:<EOL><INDENT>res += CONSONANTS[(data & <NUM_LIT>)]<EOL>data >>= <NUM_LIT:4><EOL><DEDENT>else:<EOL><INDENT>res += VOWELS[(data & <NUM_LIT>)]<EOL>data >>= <NUM_LIT:2><EOL><DEDENT><DEDENT>if data > <NUM_LIT:0>:<EOL><INDENT>res += '<STR_LIT:->'<EOL><DEDENT><DEDENT>res.reverse()<EOL>return '<STR_LIT>'.join(res)<EOL>

:params data: integer
:returns: proquint made from input data
:type data: int
:rtype: string


if not isinstance(data, basestring):<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>res = <NUM_LIT:0><EOL>for part in data.split('<STR_LIT:->'):<EOL><INDENT>if len(part) != <NUM_LIT:5>:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>for j in range(<NUM_LIT:5>):<EOL><INDENT>try:<EOL><INDENT>if not j % <NUM_LIT:2>:<EOL><INDENT>res <<= <NUM_LIT:4><EOL>res |= CONSONANTS.index(part[j])<EOL><DEDENT>else:<EOL><INDENT>res <<= <NUM_LIT:2><EOL>res |= VOWELS.index(part[j])<EOL><DEDENT><DEDENT>except ValueError:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(part[j]))<EOL><DEDENT><DEDENT><DEDENT>return res<EOL>

:params data: proquint
:returns: proquint decoded into an integer
:type data: string
:rtype: int


return from_int(int(uuid.uuid4().hex[:<NUM_LIT:8>], <NUM_LIT:16>)).replace('<STR_LIT:->', '<STR_LIT>')<EOL>


import django.apps<EOL>from morango.models import SyncableModel<EOL>from morango.manager import SyncableModelManager<EOL>from morango.query import SyncableModelQuerySet<EOL>model_list = []<EOL>for model_class in django.apps.apps.get_models():<EOL><INDENT>if issubclass(model_class, SyncableModel):<EOL><INDENT>name = model_class.__name__<EOL>if _multiple_self_ref_fk_check(model_class):<EOL><INDENT>raise InvalidMorangoModelConfiguration("<STR_LIT>")<EOL><DEDENT>try:<EOL><INDENT>from mptt import models<EOL>from morango.utils.morango_mptt import MorangoMPTTModel, MorangoMPTTTreeManager, MorangoTreeQuerySet<EOL>if issubclass(model_class, models.MPTTModel):<EOL><INDENT>if not issubclass(model_class, MorangoMPTTModel):<EOL><INDENT>raise InvalidMorangoModelConfiguration("<STR_LIT>".format(name))<EOL><DEDENT>if not isinstance(model_class.objects, MorangoMPTTTreeManager):<EOL><INDENT>raise InvalidMPTTManager("<STR_LIT>".format(name))<EOL><DEDENT>if not isinstance(model_class.objects.none(), MorangoTreeQuerySet):<EOL><INDENT>raise InvalidMPTTQuerySet("<STR_LIT>".format(name))<EOL><DEDENT><DEDENT><DEDENT>except ImportError:<EOL><INDENT>pass<EOL><DEDENT>if not isinstance(model_class.objects, SyncableModelManager):<EOL><INDENT>raise InvalidSyncableManager("<STR_LIT>".format(name))<EOL><DEDENT>if not isinstance(model_class.objects.none(), SyncableModelQuerySet):<EOL><INDENT>raise InvalidSyncableQueryset("<STR_LIT>".format(name))<EOL><DEDENT>if model_class._meta.many_to_many:<EOL><INDENT>raise UnsupportedFieldType("<STR_LIT>")<EOL><DEDENT>if not hasattr(model_class, '<STR_LIT>'):<EOL><INDENT>raise InvalidMorangoModelConfiguration("<STR_LIT>".format(name))<EOL><DEDENT>if not hasattr(model_class, '<STR_LIT>'):<EOL><INDENT>raise InvalidMorangoModelConfiguration("<STR_LIT>".format(name))<EOL><DEDENT>profile = model_class.morango_profile<EOL>_profile_models[profile] = _profile_models.get(profile, [])<EOL>if model_class.morango_model_name is not None:<EOL><INDENT>_insert_model_into_profile_dict(model_class, profile)<EOL><DEDENT><DEDENT><DEDENT>for profile, model_list in iteritems(_profile_models):<EOL><INDENT>syncable_models_dict = OrderedDict()<EOL>for model_class in model_list:<EOL><INDENT>syncable_models_dict[model_class.morango_model_name] = model_class<EOL><DEDENT>_profile_models[profile] = syncable_models_dict<EOL><DEDENT>

Per profile, adds each model to a dictionary mapping the morango model name to its model class.
We sort by ForeignKey dependencies to safely sync data.


<EOL>if self.uuid_input_fields is None:<EOL><INDENT>raise NotImplementedError("""<STR_LIT>""")<EOL><DEDENT>if self.uuid_input_fields == "<STR_LIT>":<EOL><INDENT>return uuid.uuid4().hex<EOL><DEDENT>assert isinstance(self.uuid_input_fields, tuple), "<STR_LIT>"<EOL>hashable_input_vals = []<EOL>for field in self.uuid_input_fields:<EOL><INDENT>new_value = getattr(self, field)<EOL>if new_value:<EOL><INDENT>hashable_input_vals.append(str(new_value))<EOL><DEDENT><DEDENT>hashable_input = "<STR_LIT::>".join(hashable_input_vals)<EOL>if not hashable_input:<EOL><INDENT>return uuid.uuid4().hex<EOL><DEDENT>return sha2_uuid(hashable_input)<EOL>

Should return a 32-digit hex string for a UUID that is calculated as a function of a set of fields from the model.


def _bulk_insert_into_app_models(self, cursor, app_model, fields, db_values, placeholder_list):

<EOL>num_of_rows_able_to_insert = self.SQLITE_MAX_VARIABLE_NUMBER // len(fields)<EOL>num_of_values_able_to_insert = num_of_rows_able_to_insert * len(fields)<EOL>value_chunks = [db_values[x:x + num_of_values_able_to_insert] for x in range(<NUM_LIT:0>, len(db_values), num_of_values_able_to_insert)]<EOL>placeholder_chunks = [placeholder_list[x: x + num_of_rows_able_to_insert] for x in range(<NUM_LIT:0>, len(placeholder_list), num_of_rows_able_to_insert)]<EOL>fields = str(tuple(str(f.attname) for f in fields)).replace("<STR_LIT:'>", '<STR_LIT>')<EOL>for values, params in zip(value_chunks, placeholder_chunks):<EOL><INDENT>placeholder_str = '<STR_LIT:U+002CU+0020>'.join(params).replace("<STR_LIT:'>", '<STR_LIT>')<EOL>insert = """<STR_LIT>""".format(app_model=app_model, fields=fields, placeholder_str=placeholder_str)<EOL>cursor.execute(insert, values)<EOL><DEDENT>

Example query:
`REPLACE INTO model (F1,F2,F3) VALUES (%s, %s, %s), (%s, %s, %s), (%s, %s, %s)`
where values=[1,2,3,4,5,6,7,8,9]


def authenticate_credentials(self, userargs, password, request=None):

credentials = {<EOL>'<STR_LIT:password>': password<EOL>}<EOL>if "<STR_LIT:=>" not in userargs:<EOL><INDENT>credentials[get_user_model().USERNAME_FIELD] = userargs<EOL><DEDENT>else:<EOL><INDENT>for arg in userargs.split("<STR_LIT:&>"):<EOL><INDENT>key, val = arg.split("<STR_LIT:=>")<EOL>credentials[key] = val<EOL><DEDENT><DEDENT>user = authenticate(**credentials)<EOL>if user is None:<EOL><INDENT>raise exceptions.AuthenticationFailed('<STR_LIT>')<EOL><DEDENT>if not user.is_active:<EOL><INDENT>raise exceptions.AuthenticationFailed('<STR_LIT>')<EOL><DEDENT>return (user, None)<EOL>

Authenticate the userargs and password against Django auth backends.
The "userargs" string may be just the username, or a querystring-encoded set of params.


raise NotImplementedError("<STR_LIT>")<EOL>

Should return a string that uniquely defines the model instance or `None` for a random uuid.


Should return a string specifying this model instance's partition, using `self.ID_PLACEHOLDER` in place of its own ID, if needed.


klass_model = _profile_models[self.profile][self.model_name]<EOL>if self.deleted:<EOL><INDENT>if self.hard_deleted:<EOL><INDENT>try:<EOL><INDENT>klass_model.objects.get(id=self.id).delete(hard_delete=True)<EOL><DEDENT>except klass_model.DoesNotExist:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>else:<EOL><INDENT>klass_model.objects.filter(id=self.id).delete()<EOL><DEDENT>return None<EOL><DEDENT>else:<EOL><INDENT>app_model = klass_model.deserialize(json.loads(self.serialized))<EOL>app_model._morango_source_id = self.source_id<EOL>app_model._morango_partition = self.partition<EOL>app_model._morango_dirty_bit = False<EOL>try:<EOL><INDENT>app_model.cached_clean_fields(fk_cache)<EOL>return app_model<EOL><DEDENT>except exceptions.ValidationError as e:<EOL><INDENT>logger.warn("<STR_LIT>".format(model=klass_model.__name__, id=app_model.id, error=e))<EOL>fk_ids = [getattr(app_model, field.attname) for field in app_model._meta.fields if isinstance(field, ForeignKey)]<EOL>for fk_id in fk_ids:<EOL><INDENT>try:<EOL><INDENT>st_model = Store.objects.get(id=fk_id)<EOL>if st_model.deleted:<EOL><INDENT>if st_model.hard_deleted:<EOL><INDENT>app_model._update_hard_deleted_models()<EOL><DEDENT>app_model._update_deleted_models()<EOL>return None<EOL><DEDENT><DEDENT>except Store.DoesNotExist:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>raise e<EOL><DEDENT><DEDENT>

When deserializing a store model, we look at the deleted flags to know if we should delete the app model.
Upon loading the app model in memory we validate the app models fields, if any errors occurs we follow
foreign key relationships to see if the related model has been deleted to propagate that deletion to the target app model.
We return:
None => if the model was deleted successfully
model => if the model validates successfully


@abc.abstractmethod<EOL><INDENT>def _to(self, handle, fs):<DEDENT>

raise NotImplementedError()<EOL>

Save the given FS to the given stream handle.

        Parameters:
            handle (`io.IOBase`): a writable stream in which to write
                the filesystem in an archive.
            fs (`fs.base.FS`): the filesystem to save.


if self.overwrite: <EOL><INDENT>fd, temp = tempfile.mkstemp()<EOL>os.close(fd)<EOL>self._to(temp, fs)<EOL>self.output.seek(self.initial_position)<EOL>with open(temp, '<STR_LIT:rb>') as f:<EOL><INDENT>shutil.copyfileobj(f, self.output)<EOL><DEDENT>os.remove(temp)<EOL><DEDENT>else:<EOL><INDENT>self._to(self.output, fs)<EOL><DEDENT>

Save the given FS, considering ``self.output`` as a stream.

        Parameters:
            fs (`fs.base.FS`): the filesystem to save in the archive.


if self.overwrite: <EOL><INDENT>tmp = '<STR_LIT:.>'.join([self.output, '<STR_LIT>'])<EOL>self._to(tmp, fs)<EOL>shutil.move(tmp, self.output)<EOL><DEDENT>else:<EOL><INDENT>self._to(self.output, fs)<EOL><DEDENT>

Save the given FS, considering ``self.output`` as a filename.

        Parameters:
            fs (`fs.base.FS`): the filesystem to save in the archive.


super(ArchiveReadFS, self).__init__()<EOL>if isinstance(handle, six.binary_type):<EOL><INDENT>handle = fsdecode(fspath(handle))<EOL><DEDENT>if isinstance(handle, six.text_type):<EOL><INDENT>_path = os.path.expanduser(os.path.expandvars(handle))<EOL>_path = os.path.normpath(os.path.abspath(_path))<EOL>if os.path.exists(_path) and os.access(_path, os.R_OK):<EOL><INDENT>self._close_handle = True<EOL>self._handle = open(_path, '<STR_LIT:rb>')<EOL><DEDENT><DEDENT>elif hasattr(handle, '<STR_LIT>'):<EOL><INDENT>if handle.readable() and handle.seekable():<EOL><INDENT>self._close_handle = options.get('<STR_LIT>', True)<EOL>self._handle = handle<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise errors.CreateFailed("<STR_LIT>".format(handle))<EOL><DEDENT>

Create a new archive reader filesystem.

        Parameters:
            handle (`io.IOBase` or `str`): a filename or a readable
                file-like object storing the archive to read

        Keyword Arguments:
            close_handle (`boolean`): if ``True``, close the handle
                when the filesystem is closed. **[default: True]**


@abc.abstractmethod<EOL><INDENT>def compress(self, handle, fs):<DEDENT>

pass<EOL>

Compress ``fs`` and write the resulting archive to ``handle``.


raise NotImplementedError()<EOL>

Return an iterator over the directories in the archive in ``handle``.


pass<EOL>


return open_fs('<STR_LIT>')<EOL>


it = pkg_resources.iter_entry_points('<STR_LIT>')<EOL>entry_point = next((ep for ep in it if archive.endswith(ep.name)), None)<EOL>if entry_point is None:<EOL><INDENT>raise UnsupportedProtocol(<EOL>'<STR_LIT>'.format(archive))<EOL><DEDENT>try:<EOL><INDENT>archive_opener = entry_point.load()<EOL><DEDENT>except pkg_resources.DistributionNotFound as df: <EOL><INDENT>six.raise_from(UnsupportedProtocol(<EOL>'<STR_LIT>'.format(entry_point.name, df.req)), None)<EOL><DEDENT>try:<EOL><INDENT>binfile = None<EOL>archive_fs = None<EOL>fs = open_fs(fs_url)<EOL>if issubclass(archive_opener, base.ArchiveFS):<EOL><INDENT>try:<EOL><INDENT>binfile = fs.openbin(archive, '<STR_LIT>')<EOL><DEDENT>except errors.ResourceNotFound:<EOL><INDENT>binfile = fs.openbin(archive, '<STR_LIT:w>')<EOL><DEDENT>except errors.ResourceReadOnly:<EOL><INDENT>binfile = fs.openbin(archive, '<STR_LIT:r>')<EOL>archive_opener = archive_opener._read_fs_cls<EOL><DEDENT><DEDENT>elif issubclass(archive_opener, base.ArchiveReadFS):<EOL><INDENT>binfile = fs.openbin(archive, '<STR_LIT:r>')<EOL><DEDENT>if not hasattr(binfile, '<STR_LIT:name>'):<EOL><INDENT>binfile.name = basename(archive)<EOL><DEDENT>archive_fs = archive_opener(binfile)<EOL><DEDENT>except Exception:<EOL><INDENT>getattr(archive_fs, '<STR_LIT>', lambda: None)()<EOL>getattr(binfile, '<STR_LIT>', lambda: None)()<EOL>raise<EOL><DEDENT>else:<EOL><INDENT>return archive_fs<EOL><DEDENT>

Open an archive on a filesystem.

    This function tries to mimick the behaviour of `fs.open_fs` as closely
    as possible: it accepts either a FS URL or a filesystem instance, and
    will close all resources it had to open.

    Arguments:
        fs_url (FS or text_type): a FS URL, or a filesystem
            instance, where the archive file is located.
        archive (text_type): the path to the archive file on the
            given filesystem.

    Raises:
        `fs.opener._errors.Unsupported`: when the archive type is not supported
            (either the file extension is unknown or the opener requires unmet
            dependencies).

    Example:
        >>> from fs.archive import open_archive
        >>> with open_archive('mem://', 'test.tar.gz') as archive_fs:
        ...     type(archive_fs)
        <class 'fs.archive.tarfs.TarFS'>

    Hint:
        This function finds the entry points defined in group
        ``fs.archive.open_archive``, using the names of the entry point
        as the registered extension.


def iso_name_increment(name, is_dir=False, max_length=<NUM_LIT:8>):

<EOL>if not is_dir and '<STR_LIT:.>' in name:<EOL><INDENT>name, ext = name.rsplit('<STR_LIT:.>')<EOL>ext = '<STR_LIT>'.format(ext)<EOL><DEDENT>else:<EOL><INDENT>ext = '<STR_LIT>'<EOL><DEDENT>for position, char in reversed(list(enumerate(name))):<EOL><INDENT>if char not in string.digits:<EOL><INDENT>break<EOL><DEDENT><DEDENT>base, tag = name[:position+<NUM_LIT:1>], name[position+<NUM_LIT:1>:]<EOL>tag = str(int(tag or <NUM_LIT:0>) + <NUM_LIT:1>)<EOL>if len(tag) + len(base) > max_length:<EOL><INDENT>base = base[:max_length - len(tag)]<EOL><DEDENT>return '<STR_LIT>'.join([base, tag, ext])<EOL>

Increment an ISO name to avoid name collision.

    Example:
        >>> iso_name_increment('foo.txt')
        'foo1.txt'
        >>> iso_name_increment('bar10')
        'bar11'
        >>> iso_name_increment('bar99', max_length=5)
        'ba100'


def iso_path_slugify(path, path_table, is_dir=False, strict=True):

<EOL>parent, base = split(path)<EOL>slug_parent = path_table[parent]<EOL>if is_dir:<EOL><INDENT>slug_base = iso_name_slugify(base)[:<NUM_LIT:8>]<EOL><DEDENT>else:<EOL><INDENT>name, ext = base.rsplit('<STR_LIT:.>', <NUM_LIT:1>) if '<STR_LIT:.>' in base else (base, '<STR_LIT>')<EOL>slug_base = '<STR_LIT:.>'.join([iso_name_slugify(name)[:<NUM_LIT:8>], ext])<EOL><DEDENT>if strict:<EOL><INDENT>slug_base = slug_base.upper()<EOL><DEDENT>slugs = set(path_table.values())<EOL>path_table[path] = slug = join(slug_parent, slug_base)<EOL>while slug in slugs:<EOL><INDENT>slug_base = iso_name_increment(slug_base, is_dir)<EOL>path_table[path] = slug = join(slug_parent, slug_base)<EOL><DEDENT>return slug<EOL>

Slugify a path, maintaining a map with the previously slugified paths.

    The path table is used to prevent slugified names from collisioning,
    using the `iso_name_increment` function to deduplicate slugs.

    Example:
        >>> path_table = {'/': '/'}
        >>> iso_path_slugify('/ébc.txt', path_table)
        '/_BC.TXT'
        >>> iso_path_slugify('/àbc.txt', path_table)
        '/_BC2.TXT'


seen = set()<EOL>seen_add = seen.add<EOL>if key is None:<EOL><INDENT>for element in filterfalse(seen.__contains__, iterable):<EOL><INDENT>seen_add(element)<EOL>yield element<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for element in iterable:<EOL><INDENT>k = key(element)<EOL>if k not in seen:<EOL><INDENT>seen_add(k)<EOL>yield element<EOL><DEDENT><DEDENT><DEDENT>

Yield unique elements, preserving order.


await self.play()<EOL>

Plays the next track in the queue, if any.


if not self.is_connected:<EOL><INDENT>return<EOL><DEDENT>await self.stop()<EOL>ws = self._lavalink.bot._connection._get_websocket(int(self.guild_id))<EOL>await ws.voice_state(self.guild_id, None)<EOL>

Disconnects from the voice channel, if any.


await self._lavalink.ws.send(op='<STR_LIT>', guildId=self.guild_id, pause=pause)<EOL>self.paused = pause<EOL>


Removes all of the players from the cache.


await self._lavalink.ws.send(op='<STR_LIT>', guildId=self.guild_id)<EOL>self.current = None<EOL>


async def play(self, track_index: int = <NUM_LIT:0>, ignore_shuffle: bool = False):

if self.repeat and self.current:<EOL><INDENT>self.queue.append(self.current)<EOL><DEDENT>self.previous = self.current<EOL>self.current = None<EOL>self.position = <NUM_LIT:0><EOL>self.paused = False<EOL>if not self.queue:<EOL><INDENT>await self.stop()<EOL>await self._lavalink.dispatch_event(QueueEndEvent(self))<EOL><DEDENT>else:<EOL><INDENT>if self.shuffle and not ignore_shuffle:<EOL><INDENT>track = self.queue.pop(randrange(len(self.queue)))<EOL><DEDENT>else:<EOL><INDENT>track = self.queue.pop(min(track_index, len(self.queue) - <NUM_LIT:1>))<EOL><DEDENT>self.current = track<EOL>await self._lavalink.ws.send(op='<STR_LIT>', guildId=self.guild_id, track=track.track)<EOL>await self._lavalink.dispatch_event(TrackStartEvent(self, track))<EOL><DEDENT>

Plays the first track in the queue, if any or plays a track from the specified index in the queue.


found = self.find_all(predicate)<EOL>return found[<NUM_LIT:0>] if found else None<EOL>

Returns the first player in the list based on the given filter predicate. Could be None.


if not issubclass(player, BasePlayer):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>self.lavalink = lavalink<EOL>self._player = player<EOL>self._players = {}<EOL>

Instantiates a Player Manager.

:param lavalink:
    Must be a lavalink.Client object.
:param player:
    Must implement lavalink.BasePlayer.


def add_at(self, index: int, requester: int, track: dict):

self.queue.insert(min(index, len(self.queue) - <NUM_LIT:1>), AudioTrack().build(track, requester))<EOL>

Adds a track at a specific index in the queue.


if guild_id in self._players:<EOL><INDENT>self._players[guild_id].cleanup()<EOL>del self._players[guild_id]<EOL><DEDENT>

Removes a player from the current players.


return item in self._players<EOL>

Returns the presence of a player in the cache.


if not self.channel_id:<EOL><INDENT>return None<EOL><DEDENT>return self._lavalink.bot.get_channel(int(self.channel_id))<EOL>

Returns the voice channel the player is connected to.


return self.connected_channel is not None and self.current is not None<EOL>


def add(self, requester: int, track: dict):

self.queue.append(AudioTrack().build(track, requester))<EOL>


try:<EOL><INDENT>del self._user_data[key]<EOL><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT>

Removes an item from the the stored user data.


if self._lavalink._server_version <= <NUM_LIT:2>:<EOL><INDENT>self.volume = max(min(vol, <NUM_LIT>), <NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>self.volume = max(min(vol, <NUM_LIT:1000>), <NUM_LIT:0>)<EOL><DEDENT>await self._lavalink.ws.send(op='<STR_LIT>', guildId=self.guild_id, volume=self.volume)<EOL>

Sets the player's volume (150% or 1000% limit imposed by lavalink depending on the version).


if guild_id not in self._players:<EOL><INDENT>p = self._player(lavalink=self.lavalink, guild_id=guild_id)<EOL>self._players[guild_id] = p<EOL><DEDENT>return self._players[guild_id]<EOL>

Returns a player from the cache, or creates one if it does not exist.


if self._ws and self._ws.open:<EOL><INDENT>log.debug('<STR_LIT>'.format(str(data)))<EOL>await self._ws.send(json.dumps(data))<EOL><DEDENT>else:<EOL><INDENT>self._queue.append(data)<EOL>log.debug('<STR_LIT>'.format(str(data)))<EOL><DEDENT>


while not self._shutdown:<EOL><INDENT>try:<EOL><INDENT>data = json.loads(await self._ws.recv())<EOL><DEDENT>except websockets.ConnectionClosed as error:<EOL><INDENT>log.warning('<STR_LIT>'.format(str(error)))<EOL>for g in self._lavalink.players._players.copy().keys():<EOL><INDENT>ws = self._lavalink.bot._connection._get_websocket(int(g))<EOL>await ws.voice_state(int(g), None)<EOL><DEDENT>self._lavalink.players.clear()<EOL>if self._shutdown:<EOL><INDENT>break<EOL><DEDENT>if await self._attempt_reconnect():<EOL><INDENT>return<EOL><DEDENT>log.warning('<STR_LIT>')<EOL>break<EOL><DEDENT>op = data.get('<STR_LIT>', None)<EOL>log.debug('<STR_LIT>'.format(str(data)))<EOL>if not op:<EOL><INDENT>return log.debug('<STR_LIT>'.format(str(data)))<EOL><DEDENT>if op == '<STR_LIT>':<EOL><INDENT>log.debug('<STR_LIT>'.format(data['<STR_LIT:type>']))<EOL>player = self._lavalink.players[int(data['<STR_LIT>'])]<EOL>event = None<EOL>if data['<STR_LIT:type>'] == '<STR_LIT>':<EOL><INDENT>event = TrackEndEvent(player, data['<STR_LIT>'], data['<STR_LIT>'])<EOL><DEDENT>elif data['<STR_LIT:type>'] == '<STR_LIT>':<EOL><INDENT>event = TrackExceptionEvent(player, data['<STR_LIT>'], data['<STR_LIT:error>'])<EOL><DEDENT>elif data['<STR_LIT:type>'] == '<STR_LIT>':<EOL><INDENT>event = TrackStuckEvent(player, data['<STR_LIT>'], data['<STR_LIT>'])<EOL><DEDENT>if event:<EOL><INDENT>await self._lavalink.dispatch_event(event)<EOL><DEDENT><DEDENT>elif op == '<STR_LIT>':<EOL><INDENT>await self._lavalink.update_state(data)<EOL><DEDENT>elif op == '<STR_LIT>':<EOL><INDENT>self._lavalink.stats._update(data)<EOL>await self._lavalink.dispatch_event(StatsUpdateEvent(self._lavalink.stats))<EOL><DEDENT><DEDENT>log.debug('<STR_LIT>')<EOL>await self._ws.close()<EOL>

Waits to receive a payload from the Lavalink server and processes it.


hours, remainder = divmod(time / <NUM_LIT:1000>, <NUM_LIT>)<EOL>minutes, seconds = divmod(remainder, <NUM_LIT>)<EOL>return '<STR_LIT>' % (hours, minutes, seconds)<EOL>


if '<STR_LIT>' in self.uri:<EOL><INDENT>return "<STR_LIT>".format(self.identifier)<EOL><DEDENT>return "<STR_LIT>"<EOL>

Returns the video thumbnail. Could be an empty string.


if func in self.hooks:<EOL><INDENT>self.hooks.remove(func)<EOL><DEDENT>

Unregisters a hook. For further explanation, please have a look at ``register_hook``.


log.debug('<STR_LIT>'.format(query))<EOL>async with self.http.get(self.rest_uri + quote(query), headers={'<STR_LIT>': self.password}) as res:<EOL><INDENT>return await res.json(content_type=None)<EOL><DEDENT>

Returns a Dictionary containing search results for a given query.


guild_id = int(data['<STR_LIT>'])<EOL>if guild_id in self.players:<EOL><INDENT>player = self.players.get(guild_id)<EOL>player.position = data['<STR_LIT:state>'].get('<STR_LIT>', <NUM_LIT:0>)<EOL>player.position_timestamp = data['<STR_LIT:state>']['<STR_LIT:time>']<EOL><DEDENT>

Updates a player's state when a payload with opcode ``playerUpdate`` is received.


log.debug('<STR_LIT>'.format(event.__class__.__name__, len(self.hooks)))<EOL>for hook in self.hooks:<EOL><INDENT>try:<EOL><INDENT>if asyncio.iscoroutinefunction(hook):<EOL><INDENT>await hook(event)<EOL><DEDENT>else:<EOL><INDENT>hook(event)<EOL><DEDENT><DEDENT>except Exception as e:  <EOL><INDENT>log.warning(<EOL>'<STR_LIT>'.format(hook.__name__, str(e)))<EOL><DEDENT><DEDENT>if isinstance(event, (TrackEndEvent, TrackExceptionEvent, TrackStuckEvent)) and event.player:<EOL><INDENT>await event.player.handle_event(event)<EOL><DEDENT>

Dispatches an event to all registered hooks.


@commands.command(name='<STR_LIT>')<EOL><INDENT>@commands.guild_only()<EOL>async def _find(self, ctx, *, query):<DEDENT>

if not query.startswith('<STR_LIT>') and not query.startswith('<STR_LIT>'):<EOL><INDENT>query = '<STR_LIT>' + query<EOL><DEDENT>tracks = await self.bot.lavalink.get_tracks(query)<EOL>if not tracks:<EOL><INDENT>return await ctx.send('<STR_LIT>')<EOL><DEDENT>tracks = tracks[:<NUM_LIT:10>]  <EOL>o = '<STR_LIT>'<EOL>for index, track in enumerate(tracks, start=<NUM_LIT:1>):<EOL><INDENT>track_title = track["<STR_LIT:info>"]["<STR_LIT:title>"]<EOL>track_uri = track["<STR_LIT:info>"]["<STR_LIT>"]<EOL>o += f'<STR_LIT>'<EOL><DEDENT>embed = discord.Embed(color=discord.Color.blurple(), description=o)<EOL>await ctx.send(embed=embed)<EOL>

Lists the first 10 search results from a given query.


@commands.command(name='<STR_LIT>')<EOL><INDENT>@commands.guild_only()<EOL>async def _stop(self, ctx):<DEDENT>

player = self.bot.lavalink.players.get(ctx.guild.id)<EOL>if not player.is_playing:<EOL><INDENT>return await ctx.send('<STR_LIT>')<EOL><DEDENT>player.queue.clear()<EOL>await player.stop()<EOL>await ctx.send('<STR_LIT>')<EOL>