def tasks_file_to_task_descriptors(tasks, retries, input_file_param_util,<EOL>output_file_param_util):

task_descriptors = []<EOL>path = tasks['<STR_LIT:path>']<EOL>task_min = tasks.get('<STR_LIT>')<EOL>task_max = tasks.get('<STR_LIT>')<EOL>param_file = dsub_util.load_file(path)<EOL>reader = csv.reader(param_file, delimiter='<STR_LIT:\t>')<EOL>header = six.advance_iterator(reader)<EOL>job_params = parse_tasks_file_header(header, input_file_param_util,<EOL>output_file_param_util)<EOL>for row in reader:<EOL><INDENT>task_id = reader.line_num - <NUM_LIT:1><EOL>if task_min and task_id < task_min:<EOL><INDENT>continue<EOL><DEDENT>if task_max and task_id > task_max:<EOL><INDENT>continue<EOL><DEDENT>if len(row) != len(job_params):<EOL><INDENT>dsub_util.print_error('<STR_LIT>' %<EOL>(len(row), len(job_params), reader.line_num))<EOL><DEDENT>envs = set()<EOL>inputs = set()<EOL>outputs = set()<EOL>labels = set()<EOL>for i in range(<NUM_LIT:0>, len(job_params)):<EOL><INDENT>param = job_params[i]<EOL>name = param.name<EOL>if isinstance(param, job_model.EnvParam):<EOL><INDENT>envs.add(job_model.EnvParam(name, row[i]))<EOL><DEDENT>elif isinstance(param, job_model.LabelParam):<EOL><INDENT>labels.add(job_model.LabelParam(name, row[i]))<EOL><DEDENT>elif isinstance(param, job_model.InputFileParam):<EOL><INDENT>inputs.add(<EOL>input_file_param_util.make_param(name, row[i], param.recursive))<EOL><DEDENT>elif isinstance(param, job_model.OutputFileParam):<EOL><INDENT>outputs.add(<EOL>output_file_param_util.make_param(name, row[i], param.recursive))<EOL><DEDENT><DEDENT>task_descriptors.append(<EOL>job_model.TaskDescriptor({<EOL>'<STR_LIT>': task_id,<EOL>'<STR_LIT>': <NUM_LIT:1> if retries else None<EOL>}, {<EOL>'<STR_LIT>': labels,<EOL>'<STR_LIT>': envs,<EOL>'<STR_LIT>': inputs,<EOL>'<STR_LIT>': outputs<EOL>}, job_model.Resources()))<EOL><DEDENT>if not task_descriptors:<EOL><INDENT>raise ValueError('<STR_LIT>' % path)<EOL><DEDENT>return task_descriptors<EOL>

Parses task parameters from a TSV.

    Args:
      tasks: Dict containing the path to a TSV file and task numbers to run
      variables, input, and output parameters as column headings. Subsequent
      lines specify parameter values, one row per job.
      retries: Number of retries allowed.
      input_file_param_util: Utility for producing InputFileParam objects.
      output_file_param_util: Utility for producing OutputFileParam objects.

    Returns:
      task_descriptors: an array of records, each containing the task-id,
      task-attempt, 'envs', 'inputs', 'outputs', 'labels' that defines the set of
      parameters for each task of the job.

    Raises:
      ValueError: If no job records were provided


label_data = set()<EOL>for arg in labels:<EOL><INDENT>name, value = split_pair(arg, '<STR_LIT:=>', nullable_idx=<NUM_LIT:1>)<EOL>label_data.add(argclass(name, value))<EOL><DEDENT>return label_data<EOL>

Parse flags of key=value pairs and return a list of argclass.

    For pair variables, we need to:
       * split the input into name=value pairs (value optional)
       * Create the EnvParam object

    Args:
      labels: list of 'key' or 'key=value' strings.
      argclass: Container class for args, must instantiate with argclass(k, v).

    Returns:
      list of argclass objects.


def args_to_job_params(envs, labels, inputs, inputs_recursive, outputs,<EOL>outputs_recursive, mounts, input_file_param_util,<EOL>output_file_param_util, mount_param_util):

<EOL>env_data = parse_pair_args(envs, job_model.EnvParam)<EOL>label_data = parse_pair_args(labels, job_model.LabelParam)<EOL>input_data = set()<EOL>for (recursive, args) in ((False, inputs), (True, inputs_recursive)):<EOL><INDENT>for arg in args:<EOL><INDENT>name, value = split_pair(arg, '<STR_LIT:=>', nullable_idx=<NUM_LIT:0>)<EOL>name = input_file_param_util.get_variable_name(name)<EOL>input_data.add(input_file_param_util.make_param(name, value, recursive))<EOL><DEDENT><DEDENT>output_data = set()<EOL>for (recursive, args) in ((False, outputs), (True, outputs_recursive)):<EOL><INDENT>for arg in args:<EOL><INDENT>name, value = split_pair(arg, '<STR_LIT:=>', <NUM_LIT:0>)<EOL>name = output_file_param_util.get_variable_name(name)<EOL>output_data.add(output_file_param_util.make_param(name, value, recursive))<EOL><DEDENT><DEDENT>mount_data = set()<EOL>for arg in mounts:<EOL><INDENT>if '<STR_LIT:U+0020>' in arg:<EOL><INDENT>key_value_pair, disk_size = arg.split('<STR_LIT:U+0020>')<EOL>name, value = split_pair(key_value_pair, '<STR_LIT:=>', <NUM_LIT:1>)<EOL>mount_data.add(mount_param_util.make_param(name, value, disk_size))<EOL><DEDENT>else:<EOL><INDENT>name, value = split_pair(arg, '<STR_LIT:=>', <NUM_LIT:1>)<EOL>mount_data.add(mount_param_util.make_param(name, value, disk_size=None))<EOL><DEDENT><DEDENT>return {<EOL>'<STR_LIT>': env_data,<EOL>'<STR_LIT>': input_data,<EOL>'<STR_LIT>': output_data,<EOL>'<STR_LIT>': label_data,<EOL>'<STR_LIT>': mount_data,<EOL>}<EOL>

Parse env, input, and output parameters into a job parameters and data.

    Passing arguments on the command-line allows for launching a single job.
    The env, input, and output arguments encode both the definition of the
    job as well as the single job's values.

    Env arguments are simple name=value pairs.
    Input and output file arguments can contain name=value pairs or just values.
    Either of the following is valid:

      uri
      myfile=uri

    Args:
      envs: list of environment variable job parameters
      labels: list of labels to attach to the tasks
      inputs: list of file input parameters
      inputs_recursive: list of recursive directory input parameters
      outputs: list of file output parameters
      outputs_recursive: list of recursive directory output parameters
      mounts: list of gcs buckets to mount
      input_file_param_util: Utility for producing InputFileParam objects.
      output_file_param_util: Utility for producing OutputFileParam objects.
      mount_param_util: Utility for producing MountParam objects.

    Returns:
      job_params: a dictionary of 'envs', 'inputs', and 'outputs' that defines the
      set of parameters and data for a job.


def validate_submit_args_or_fail(job_descriptor, provider_name, input_providers,<EOL>output_providers, logging_providers):

job_resources = job_descriptor.job_resources<EOL>job_params = job_descriptor.job_params<EOL>task_descriptors = job_descriptor.task_descriptors<EOL>_validate_providers([job_resources.logging], '<STR_LIT>', logging_providers,<EOL>provider_name)<EOL>_validate_providers(job_params['<STR_LIT>'], '<STR_LIT:input>', input_providers,<EOL>provider_name)<EOL>_validate_providers(job_params['<STR_LIT>'], '<STR_LIT>', output_providers,<EOL>provider_name)<EOL>for task_descriptor in task_descriptors:<EOL><INDENT>_validate_providers(task_descriptor.task_params['<STR_LIT>'], '<STR_LIT:input>',<EOL>input_providers, provider_name)<EOL>_validate_providers(task_descriptor.task_params['<STR_LIT>'], '<STR_LIT>',<EOL>output_providers, provider_name)<EOL><DEDENT>

Validate that arguments passed to submit_job have valid file providers.

    This utility function takes resources and task data args from `submit_job`
    in the base provider. This function will fail with a value error if any of the
    parameters are not valid. See the following example;

    >>> job_resources = type('', (object,),
    ...    {"logging": job_model.LoggingParam('gs://logtemp', job_model.P_GCS)})()
    >>> job_params={'inputs': set(), 'outputs': set(), 'mounts': set()}
    >>> task_descriptors = [
    ...     job_model.TaskDescriptor(None, {
    ...       'inputs': {
    ...           job_model.FileParam('IN', uri='gs://in/*',
    ...                               file_provider=job_model.P_GCS)},
    ...       'outputs': set()}, None),
    ...     job_model.TaskDescriptor(None, {
    ...       'inputs': set(),
    ...       'outputs': {
    ...           job_model.FileParam('OUT', uri='gs://out/*',
    ...                               file_provider=job_model.P_GCS)}}, None)]
    ...
    >>> validate_submit_args_or_fail(job_model.JobDescriptor(None, job_params,
    ...                              job_resources, task_descriptors),
    ...                              provider_name='MYPROVIDER',
    ...                              input_providers=[job_model.P_GCS],
    ...                              output_providers=[job_model.P_GCS],
    ...                              logging_providers=[job_model.P_GCS])
    ...
    >>> validate_submit_args_or_fail(job_model.JobDescriptor(None, job_params,
    ...                              job_resources, task_descriptors),
    ...                              provider_name='MYPROVIDER',
    ...                              input_providers=[job_model.P_GCS],
    ...                              output_providers=[job_model.P_LOCAL],
    ...                              logging_providers=[job_model.P_GCS])
    Traceback (most recent call last):
         ...
    ValueError: Unsupported output path (gs://out/*) for provider 'MYPROVIDER'.

    Args:
      job_descriptor: instance of job_model.JobDescriptor.
      provider_name: (str) the name of the execution provider.
      input_providers: (string collection) whitelist of file providers for input.
      output_providers: (string collection) whitelist of providers for output.
      logging_providers: (string collection) whitelist of providers for logging.

    Raises:
      ValueError: if any file providers do not match the whitelists.


return directory.rstrip('<STR_LIT:/>') + '<STR_LIT:/>'<EOL>

In ensure that directories end with '/'.

    Frequently we need to ensure that directory paths end with a forward slash.
    Pythons dirname and split functions in the path library treat this
    inconsistently creating this requirement. This function is simple but was
    written to centralize documentation of an often used (and often explained)
    requirement in this codebase.

    >>> os.path.dirname('gs://bucket/folder/file.txt')
    'gs://bucket/folder'
    >>> directory_fmt(os.path.dirname('gs://bucket/folder/file.txt'))
    'gs://bucket/folder/'
    >>> os.path.dirname('/newfile')
    '/'
    >>> directory_fmt(os.path.dirname('/newfile'))
    '/'

    Specifically we need this since copy commands must know whether the
    destination is a directory to function properly. See the following shell
    interaction for an example of the inconsistency. Notice that text files are
    copied as expected but the bam is copied over the directory name.

    Multiple files copy, works as intended in all cases:
        $ touch a.txt b.txt
        $ gsutil cp ./*.txt gs://mybucket/text_dest
        $ gsutil ls gs://mybucket/text_dest/
              0  2017-07-19T21:44:36Z  gs://mybucket/text_dest/a.txt
              0  2017-07-19T21:44:36Z  gs://mybucket/text_dest/b.txt
        TOTAL: 2 objects, 0 bytes (0 B)

    Single file copy fails to copy into a directory:
        $ touch 1.bam
        $ gsutil cp ./*.bam gs://mybucket/bad_dest
        $ gsutil ls gs://mybucket/bad_dest
                 0  2017-07-19T21:46:16Z  gs://mybucket/bad_dest
        TOTAL: 1 objects, 0 bytes (0 B)

    Adding a trailing forward slash fixes this:
        $ touch my.sam
        $ gsutil cp ./*.sam gs://mybucket/good_folder
        $ gsutil ls gs://mybucket/good_folder
                 0  2017-07-19T21:46:16Z  gs://mybucket/good_folder/my.sam
        TOTAL: 1 objects, 0 bytes (0 B)

    Args:
      directory (str): a uri without an blob or file basename.

    Returns:
      the directory with a trailing slash.


parser = argparse.ArgumentParser(description='<STR_LIT>', add_help=False)<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', dest='<STR_LIT:version>', action='<STR_LIT:store_true>')<EOL>parser.set_defaults(version=False)<EOL>args, _ = parser.parse_known_args()<EOL>if args.version:<EOL><INDENT>print('<STR_LIT>' % DSUB_VERSION)<EOL>sys.exit()<EOL><DEDENT>

If the --version flag is passed, print version to stdout and exit.

    Within dsub commands, --version should be the highest priority flag.
    This function supplies a repeatable and DRY way of checking for the
    version flag and printing the version. Callers still need to define a version
    flag in the command's flags so that it shows up in help output.


if not age:<EOL><INDENT>return None<EOL><DEDENT>if not from_time:<EOL><INDENT>from_time = dsub_util.replace_timezone(datetime.datetime.now(), tzlocal())<EOL><DEDENT>try:<EOL><INDENT>last_char = age[-<NUM_LIT:1>]<EOL>if last_char == '<STR_LIT:s>':<EOL><INDENT>return from_time - datetime.timedelta(seconds=int(age[:-<NUM_LIT:1>]))<EOL><DEDENT>elif last_char == '<STR_LIT:m>':<EOL><INDENT>return from_time - datetime.timedelta(minutes=int(age[:-<NUM_LIT:1>]))<EOL><DEDENT>elif last_char == '<STR_LIT:h>':<EOL><INDENT>return from_time - datetime.timedelta(hours=int(age[:-<NUM_LIT:1>]))<EOL><DEDENT>elif last_char == '<STR_LIT:d>':<EOL><INDENT>return from_time - datetime.timedelta(days=int(age[:-<NUM_LIT:1>]))<EOL><DEDENT>elif last_char == '<STR_LIT:w>':<EOL><INDENT>return from_time - datetime.timedelta(weeks=int(age[:-<NUM_LIT:1>]))<EOL><DEDENT>else:<EOL><INDENT>return dsub_util.replace_timezone(<EOL>datetime.datetime.utcfromtimestamp(int(age)), pytz.utc)<EOL><DEDENT><DEDENT>except (ValueError, OverflowError) as e:<EOL><INDENT>raise ValueError('<STR_LIT>' % (age, e))<EOL><DEDENT>

Compute the create time (UTC) for the list filter.

    If the age is an integer value it is treated as a UTC date.
    Otherwise the value must be of the form "<integer><unit>" where supported
    units are s, m, h, d, w (seconds, minutes, hours, days, weeks).

    Args:
      age: A "<integer><unit>" string or integer value.
      from_time:

    Returns:
      A timezone-aware datetime or None if age parameter is empty.


if not interval:<EOL><INDENT>return None<EOL><DEDENT>try:<EOL><INDENT>last_char = interval[-<NUM_LIT:1>]<EOL>if last_char == '<STR_LIT:s>' and '<STR_LIT:s>' in valid_units:<EOL><INDENT>return str(float(interval[:-<NUM_LIT:1>])) + '<STR_LIT:s>'<EOL><DEDENT>elif last_char == '<STR_LIT:m>' and '<STR_LIT:m>' in valid_units:<EOL><INDENT>return str(float(interval[:-<NUM_LIT:1>]) * <NUM_LIT>) + '<STR_LIT:s>'<EOL><DEDENT>elif last_char == '<STR_LIT:h>' and '<STR_LIT:h>' in valid_units:<EOL><INDENT>return str(float(interval[:-<NUM_LIT:1>]) * <NUM_LIT> * <NUM_LIT>) + '<STR_LIT:s>'<EOL><DEDENT>elif last_char == '<STR_LIT:d>' and '<STR_LIT:d>' in valid_units:<EOL><INDENT>return str(float(interval[:-<NUM_LIT:1>]) * <NUM_LIT> * <NUM_LIT> * <NUM_LIT>) + '<STR_LIT:s>'<EOL><DEDENT>elif last_char == '<STR_LIT:w>' and '<STR_LIT:w>' in valid_units:<EOL><INDENT>return str(float(interval[:-<NUM_LIT:1>]) * <NUM_LIT> * <NUM_LIT> * <NUM_LIT> * <NUM_LIT:7>) + '<STR_LIT:s>'<EOL><DEDENT>else:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>' % (interval, last_char))<EOL><DEDENT><DEDENT>except (ValueError, OverflowError) as e:<EOL><INDENT>raise ValueError('<STR_LIT>' % (interval, e))<EOL><DEDENT>

Convert the timeout duration to seconds.

    The value must be of the form "<integer><unit>" where supported
    units are s, m, h, d, w (seconds, minutes, hours, days, weeks).

    Args:
      interval: A "<integer><unit>" string.
      valid_units: A list of supported units.

    Returns:
      A string of the form "<integer>s" or None if timeout is empty.


if not name:<EOL><INDENT>name = '<STR_LIT>' % (self._auto_prefix, self._auto_index)<EOL>self._auto_index += <NUM_LIT:1><EOL><DEDENT>return name<EOL>

Produce a default variable name if none is specified.


def rewrite_uris(self, raw_uri, file_provider):

if file_provider == job_model.P_GCS:<EOL><INDENT>normalized, docker_path = _gcs_uri_rewriter(raw_uri)<EOL><DEDENT>elif file_provider == job_model.P_LOCAL:<EOL><INDENT>normalized, docker_path = _local_uri_rewriter(raw_uri)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' % file_provider)<EOL><DEDENT>return normalized, os.path.join(self._relative_path, docker_path)<EOL>

Accept a raw uri and return rewritten versions.

        This function returns a normalized URI and a docker path. The normalized
        URI may have minor alterations meant to disambiguate and prepare for use
        by shell utilities that may require a specific format.

        The docker rewriter makes substantial modifications to the raw URI when
        constructing a docker path, but modifications must follow these rules:
          1) System specific characters are not allowed (ex. indirect paths).
          2) The path, if it is a directory, must end in a forward slash.
          3) The path will begin with the value set in self._relative_path.
          4) The path will have an additional prefix (after self._relative_path) set
             by the file provider-specific rewriter.

        Rewrite output for the docker path:
          >>> out_util = FileParamUtil('AUTO_', 'output')
          >>> out_util.rewrite_uris('gs://mybucket/myfile.txt', job_model.P_GCS)[1]
          'output/gs/mybucket/myfile.txt'
          >>> out_util.rewrite_uris('./data/myfolder/', job_model.P_LOCAL)[1]
          'output/file/data/myfolder/'

        When normalizing the URI for cloud buckets, no rewrites are done. For local
        files, the user directory will be expanded and relative paths will be
        converted to absolute:
          >>> in_util = FileParamUtil('AUTO_', 'input')
          >>> in_util.rewrite_uris('gs://mybucket/gcs_dir/', job_model.P_GCS)[0]
          'gs://mybucket/gcs_dir/'
          >>> in_util.rewrite_uris('/data/./dir_a/../myfile.txt',
          ...   job_model.P_LOCAL)[0]
          '/data/myfile.txt'
          >>> in_util.rewrite_uris('file:///tmp/data/*.bam', job_model.P_LOCAL)[0]
          '/tmp/data/*.bam'

        Args:
          raw_uri: (str) the path component of the raw URI.
          file_provider: a valid provider (contained in job_model.FILE_PROVIDERS).

        Returns:
          normalized: a cleaned version of the uri provided by command line.
          docker_path: the uri rewritten in the format required for mounting inside
                       a docker worker.

        Raises:
          ValueError: if file_provider is not valid.


providers = {'<STR_LIT>': job_model.P_GCS, '<STR_LIT:file>': job_model.P_LOCAL}<EOL>provider_found = re.match(r'<STR_LIT>', uri)<EOL>if provider_found:<EOL><INDENT>prefix = provider_found.group(<NUM_LIT:1>).lower()<EOL><DEDENT>else:<EOL><INDENT>prefix = '<STR_LIT:file>'<EOL><DEDENT>if prefix in providers:<EOL><INDENT>return providers[prefix]<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' % prefix)<EOL><DEDENT>


path, filename = os.path.split(uri)<EOL>if '<STR_LIT:[>' in uri or '<STR_LIT:]>' in uri:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>' % uri)<EOL><DEDENT>if '<STR_LIT:?>' in uri:<EOL><INDENT>raise ValueError('<STR_LIT>' % uri)<EOL><DEDENT>if '<STR_LIT:*>' in path:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>' % uri)<EOL><DEDENT>if '<STR_LIT>' in filename:<EOL><INDENT>raise ValueError('<STR_LIT>' % uri)<EOL><DEDENT>if filename in ('<STR_LIT:..>', '<STR_LIT:.>'):<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>' % uri)<EOL><DEDENT>if not recursive and not filename:<EOL><INDENT>raise ValueError('<STR_LIT>'<EOL>'<STR_LIT>' % uri)<EOL><DEDENT>

Do basic validation of the uri, return the path and filename.


def parse_uri(self, raw_uri, recursive):

<EOL>if recursive:<EOL><INDENT>raw_uri = directory_fmt(raw_uri)<EOL><DEDENT>file_provider = self.parse_file_provider(raw_uri)<EOL>self._validate_paths_or_fail(raw_uri, recursive)<EOL>uri, docker_uri = self.rewrite_uris(raw_uri, file_provider)<EOL>uri_parts = job_model.UriParts(<EOL>directory_fmt(os.path.dirname(uri)), os.path.basename(uri))<EOL>return docker_uri, uri_parts, file_provider<EOL>

Return a valid docker_path, uri, and file provider from a flag value.


def make_param(self, name, raw_uri, recursive):

if not raw_uri:<EOL><INDENT>return self.param_class(name, None, None, None, recursive, None)<EOL><DEDENT>docker_path, uri_parts, provider = self.parse_uri(raw_uri, recursive)<EOL>return self.param_class(name, raw_uri, docker_path, uri_parts, recursive,<EOL>provider)<EOL>


<EOL>docker_uri = os.path.join(self._relative_path,<EOL>raw_uri.replace('<STR_LIT>', '<STR_LIT>', <NUM_LIT:1>))<EOL>return docker_uri<EOL>

Return a valid docker_path from a Google Persistent Disk url.


raw_uri = directory_fmt(raw_uri)<EOL>_, docker_path = _local_uri_rewriter(raw_uri)<EOL>local_path = docker_path[len('<STR_LIT:file>'):]<EOL>docker_uri = os.path.join(self._relative_path, docker_path)<EOL>return local_path, docker_uri<EOL>

Return a valid docker_path for a local file path.


<EOL>raw_uri = directory_fmt(raw_uri)<EOL>_, docker_path = _gcs_uri_rewriter(raw_uri)<EOL>docker_uri = os.path.join(self._relative_path, docker_path)<EOL>return docker_uri<EOL>

Return a valid docker_path for a GCS bucket.


def make_param(self, name, raw_uri, disk_size):

if raw_uri.startswith('<STR_LIT>'):<EOL><INDENT>docker_path = self._parse_image_uri(raw_uri)<EOL>return job_model.PersistentDiskMountParam(<EOL>name, raw_uri, docker_path, disk_size, disk_type=None)<EOL><DEDENT>elif raw_uri.startswith('<STR_LIT>'):<EOL><INDENT>local_path, docker_path = self._parse_local_mount_uri(raw_uri)<EOL>return job_model.LocalMountParam(name, raw_uri, docker_path, local_path)<EOL><DEDENT>elif raw_uri.startswith('<STR_LIT>'):<EOL><INDENT>docker_path = self._parse_gcs_uri(raw_uri)<EOL>return job_model.GCSMountParam(name, raw_uri, docker_path)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>'.format(raw_uri))<EOL><DEDENT>

Return a MountParam given a GCS bucket, disk image or local path.


printer = _Printer(fileobj)<EOL>previous_stdout = sys.stdout<EOL>sys.stdout = printer<EOL>try:<EOL><INDENT>yield printer<EOL><DEDENT>finally:<EOL><INDENT>sys.stdout = previous_stdout<EOL><DEDENT>

Sys.out replacer, by default with stderr.

    Use it like this:
    with replace_print_with(fileobj):
      print "hello"  # writes to the file
    print "done"  # prints to stdout

    Args:
      fileobj: a file object to replace stdout.

    Yields:
      The printer.


print(msg, file=sys.stderr)<EOL>

Utility routine to emit messages to stderr.


return pwd.getpwuid(os.getuid())[<NUM_LIT:0>]<EOL>

Returns the current OS user, this may be different from the dsub user.


return set([t.get_field('<STR_LIT>') for t in task_list])<EOL>

Returns the set of job IDs for the given tasks.


if not value_list:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>value_list.sort()<EOL>interval_list = []<EOL>curr = []<EOL>for val in value_list:<EOL><INDENT>if curr and (val > curr[-<NUM_LIT:1>] + <NUM_LIT:1>):<EOL><INDENT>interval_list.append((curr[<NUM_LIT:0>], curr[-<NUM_LIT:1>]))<EOL>curr = [val]<EOL><DEDENT>else:<EOL><INDENT>curr.append(val)<EOL><DEDENT><DEDENT>if curr:<EOL><INDENT>interval_list.append((curr[<NUM_LIT:0>], curr[-<NUM_LIT:1>]))<EOL><DEDENT>return '<STR_LIT:U+002C>'.join([<EOL>'<STR_LIT>'.format(pair[<NUM_LIT:0>], pair[<NUM_LIT:1>]) if pair[<NUM_LIT:0>] != pair[<NUM_LIT:1>] else str(pair[<NUM_LIT:0>])<EOL>for pair in interval_list<EOL>])<EOL>

Compact a list of integers into a comma-separated string of intervals.

    Args:
      value_list: A list of sortable integers such as a list of numbers

    Returns:
      A compact string representation, such as "1-5,8,12-15"


if credentials is None:<EOL><INDENT>credentials = oauth2client.client.GoogleCredentials.get_application_default(<EOL>)<EOL><DEDENT>return discovery.build('<STR_LIT>', '<STR_LIT>', credentials=credentials)<EOL>

Get a storage client using the provided credentials or defaults.


now = datetime.now().strftime('<STR_LIT>')<EOL>print_error(<EOL>'<STR_LIT>' % (now, type(exception).__name__, str(exception)))<EOL>return isinstance(exception, oauth2client.client.AccessTokenRefreshError)<EOL>

Return True if we should retry, False otherwise.


gcs_service = _get_storage_service(credentials)<EOL>bucket_name, object_name = gcs_file_path[len('<STR_LIT>'):].split('<STR_LIT:/>', <NUM_LIT:1>)<EOL>request = gcs_service.objects().get_media(<EOL>bucket=bucket_name, object=object_name)<EOL>file_handle = io.BytesIO()<EOL>downloader = MediaIoBaseDownload(file_handle, request, chunksize=<NUM_LIT> * <NUM_LIT>)<EOL>done = False<EOL>while not done:<EOL><INDENT>_, done = _downloader_next_chunk(downloader)<EOL><DEDENT>filevalue = file_handle.getvalue()<EOL>if not isinstance(filevalue, six.string_types):<EOL><INDENT>filevalue = filevalue.decode()<EOL><DEDENT>return six.StringIO(filevalue)<EOL>

Load context from a text file in gcs.

    Args:
      gcs_file_path: The target file path; should have the 'gs://' prefix.
      credentials: Optional credential to be used to load the file from gcs.

    Returns:
      The content of the text file as a string.


if file_path.startswith('<STR_LIT>'):<EOL><INDENT>return _load_file_from_gcs(file_path, credentials)<EOL><DEDENT>else:<EOL><INDENT>return open(file_path, '<STR_LIT:r>')<EOL><DEDENT>

Load a file from either local or gcs.

    Args:
      file_path: The target file path, which should have the prefix 'gs://' if
                 to be loaded from gcs.
      credentials: Optional credential to be used to load the file from gcs.

    Returns:
      A python File object if loading file from local or a StringIO object if
      loading from gcs.


gcs_service = _get_storage_service(credentials)<EOL>bucket_name, object_name = gcs_file_path[len('<STR_LIT>'):].split('<STR_LIT:/>', <NUM_LIT:1>)<EOL>request = gcs_service.objects().get(<EOL>bucket=bucket_name, object=object_name, projection='<STR_LIT>')<EOL>try:<EOL><INDENT>request.execute()<EOL>return True<EOL><DEDENT>except errors.HttpError:<EOL><INDENT>return False<EOL><DEDENT>

Check whether the file exists, in GCS.

    Args:
      gcs_file_path: The target file path; should have the 'gs://' prefix.
      credentials: Optional credential to be used to load the file from gcs.

    Returns:
      True if the file's there.


if file_path.startswith('<STR_LIT>'):<EOL><INDENT>return _file_exists_in_gcs(file_path, credentials)<EOL><DEDENT>else:<EOL><INDENT>return os.path.isfile(file_path)<EOL><DEDENT>

Check whether the file exists, on local disk or GCS.

    Args:
      file_path: The target file path; should have the 'gs://' prefix if in gcs.
      credentials: Optional credential to be used to load the file from gcs.

    Returns:
      True if the file's there.


gcs_service = _get_storage_service(credentials)<EOL>bucket_name, prefix = gcs_prefix[len('<STR_LIT>'):].split('<STR_LIT:/>', <NUM_LIT:1>)<EOL>request = gcs_service.objects().list(<EOL>bucket=bucket_name, prefix=prefix, maxResults=<NUM_LIT:1>)<EOL>response = request.execute()<EOL>return response.get('<STR_LIT>', None)<EOL>

Check whether there is a GCS object whose name starts with the prefix.

    Since GCS doesn't actually have folders, this is how we check instead.

    Args:
      gcs_prefix: The path; should start with 'gs://'.
      credentials: Optional credential to be used to load the file from gcs.

    Returns:
      True if the prefix matches at least one object in GCS.

    Raises:
      errors.HttpError: if it can't talk to the server


if '<STR_LIT:*>' not in file_pattern:<EOL><INDENT>return _file_exists_in_gcs(file_pattern, credentials)<EOL><DEDENT>if not file_pattern.startswith('<STR_LIT>'):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>gcs_service = _get_storage_service(credentials)<EOL>bucket_name, prefix = file_pattern[len('<STR_LIT>'):].split('<STR_LIT:/>', <NUM_LIT:1>)<EOL>if '<STR_LIT:*>' in bucket_name:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>assert '<STR_LIT:*>' in prefix<EOL>prefix_no_wildcard = prefix[:prefix.index('<STR_LIT:*>')]<EOL>request = gcs_service.objects().list(<EOL>bucket=bucket_name, prefix=prefix_no_wildcard)<EOL>response = request.execute()<EOL>if '<STR_LIT>' not in response:<EOL><INDENT>return False<EOL><DEDENT>items_list = [i['<STR_LIT:name>'] for i in response['<STR_LIT>']]<EOL>return any(fnmatch.fnmatch(i, prefix) for i in items_list)<EOL>

True iff an object exists matching the input GCS pattern.

    The GCS pattern must be a full object reference or a "simple pattern" that
    conforms to the dsub input and output parameter restrictions:

      * No support for **, ? wildcards or [] character ranges
      * Wildcards may only appear in the file name

    Args:
      file_pattern: eg. 'gs://foo/ba*'
      credentials: Optional credential to be used to load the file from gcs.

    Raises:
      ValueError: if file_pattern breaks the rules.

    Returns:
      True iff a file exists that matches that pattern.


<EOL>for o in outputs:<EOL><INDENT>if not o.value:<EOL><INDENT>continue<EOL><DEDENT>if o.recursive:<EOL><INDENT>if not folder_exists(o.value):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if not simple_pattern_exists_in_gcs(o.value):<EOL><INDENT>return False<EOL><DEDENT><DEDENT><DEDENT>return True<EOL>

True if each output contains at least one file or no output specified.


<EOL>if self.priority == other.priority:<EOL><INDENT>localkeys = self.next_value.keys()<EOL>otherkeys = other.next_value.keys()<EOL>shared_keys = sorted(set(localkeys).intersection(otherkeys))<EOL>for key in shared_keys:<EOL><INDENT>if self.next_value[key] != other.next_value[key]:<EOL><INDENT>return self.next_value[key] < other.next_value[key]<EOL><DEDENT><DEDENT><DEDENT>return self.priority < other.priority<EOL>

Allow order testing via '<' operator, needed by priority queue.


def __init__(self, message, error_list, launched_job):

super(JobError, self).__init__(message)<EOL>self.message = message<EOL>self.error_list = error_list<EOL>self.launched_job = launched_job<EOL>

Create a JobError to indicate something went wrong.

        Args:
            message: user-friendly message
            error_list: what went wrong
            launched_job: if the job is launched, but has errors in
              "--wait"ing on the tasks.


<EOL>if not re.match(r'<STR_LIT>', name):<EOL><INDENT>raise ValueError('<STR_LIT>' % (param_type, name))<EOL><DEDENT>

Validate that the name follows posix conventions for env variables.


if not bucket.startswith('<STR_LIT>'):<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>' % bucket)<EOL><DEDENT>bucket_name = bucket[len('<STR_LIT>'):]<EOL>if not re.search(r'<STR_LIT>', bucket_name):<EOL><INDENT>raise ValueError('<STR_LIT>' % bucket)<EOL><DEDENT>

Validate that the name is a valid GCS bucket.


<EOL>accepted_characters = string.ascii_lowercase + string.digits + '<STR_LIT:->'<EOL>def label_char_transform(char):<EOL><INDENT>if char in accepted_characters:<EOL><INDENT>return char<EOL><DEDENT>if char in string.ascii_uppercase:<EOL><INDENT>return char.lower()<EOL><DEDENT>return '<STR_LIT:->'<EOL><DEDENT>return '<STR_LIT>'.join(label_char_transform(c) for c in s)<EOL>

Turn the specified name and value into a valid Google label.


for param in [<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>'<EOL>]:<EOL><INDENT>if not job_params.get(param):<EOL><INDENT>job_params[param] = set()<EOL><DEDENT><DEDENT>

For the job, ensure that each param entry is not None.


for task_desc in task_descriptors:<EOL><INDENT>for param in [<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>'<EOL>]:<EOL><INDENT>if not task_desc.task_params.get(param):<EOL><INDENT>task_desc.task_params[param] = set()<EOL><DEDENT><DEDENT><DEDENT>

For each task, ensure that each task param entry is not None.


new_dict = {}<EOL>for k, v in d.items():<EOL><INDENT>if k in required:<EOL><INDENT>new_dict[k] = v<EOL><DEDENT>elif isinstance(v, int) or v:<EOL><INDENT>new_dict[k] = v<EOL><DEDENT><DEDENT>return new_dict<EOL>

Return a new dict with any empty items removed.

    Note that this is not a deep check. If d contains a dictionary which
    itself contains empty items, those are never checked.

    This method exists to make to_serializable() functions cleaner.
    We could revisit this some day, but for now, the serialized objects are
    stripped of empty values to keep the output YAML more compact.

    Args:
      d: a dictionary
      required: list of required keys (for example, TaskDescriptors always emit
        the "task-id", even if None)

    Returns:
      A dictionary with empty items removed.


for task_descriptor in job_descriptor.task_descriptors:<EOL><INDENT>jd = JobDescriptor(job_descriptor.job_metadata, job_descriptor.job_params,<EOL>job_descriptor.job_resources, [task_descriptor])<EOL>yield jd<EOL><DEDENT>

Generator that yields a task-specific view of the job.

    This generator exists to make it easy for callers to iterate over the tasks
    in a JobDescriptor. Each pass yields a new JobDescriptor with a single task.

    Args:
      job_descriptor: A JobDescriptor with 1 or more tasks.

    Yields:
      A JobDescriptor with a single task.


<EOL>if task_id is not None:<EOL><INDENT>if task_id.startswith('<STR_LIT>'):<EOL><INDENT>return int(task_id[len('<STR_LIT>'):])<EOL><DEDENT>else:<EOL><INDENT>return int(task_id)<EOL><DEDENT><DEDENT>

Converts a task-id to the numeric task-id.

    Args:
      task_id: task-id in either task-n or n format

    Returns:
      n


@classmethod<EOL><INDENT>def _validate_label(cls, name, value):<DEDENT>

<EOL>cls._check_label_name(name)<EOL>cls._check_label_value(value)<EOL>if not cls._allow_reserved_keys and name in RESERVED_LABELS:<EOL><INDENT>raise ValueError('<STR_LIT>' %<EOL>(name, list(RESERVED_LABELS)))<EOL><DEDENT>

Raise ValueError if the label is invalid.


task_metadata = self.task_metadata<EOL>task_params = self.task_params<EOL>task_resources = self.task_resources<EOL>task_id = None<EOL>if task_metadata.get('<STR_LIT>') is not None:<EOL><INDENT>task_id = str(task_metadata.get('<STR_LIT>'))<EOL><DEDENT>task = {'<STR_LIT>': task_id}<EOL>task['<STR_LIT>'] = task_metadata.get('<STR_LIT>')<EOL>task['<STR_LIT>'] = task_metadata.get('<STR_LIT>')<EOL>if task_resources.logging_path:<EOL><INDENT>task['<STR_LIT>'] = str(task_resources.logging_path.uri)<EOL><DEDENT>task['<STR_LIT>'] = {var.name: var.value for var in task_params['<STR_LIT>']}<EOL>task['<STR_LIT>'] = {var.name: var.value for var in task_params['<STR_LIT>']}<EOL>task['<STR_LIT>'] = {<EOL>var.name: var.value<EOL>for var in task_params['<STR_LIT>']<EOL>if not var.recursive<EOL>}<EOL>task['<STR_LIT>'] = {<EOL>var.name: var.value<EOL>for var in task_params['<STR_LIT>']<EOL>if var.recursive<EOL>}<EOL>task['<STR_LIT>'] = {<EOL>var.name: var.value<EOL>for var in task_params['<STR_LIT>']<EOL>if not var.recursive<EOL>}<EOL>task['<STR_LIT>'] = {<EOL>var.name: var.value<EOL>for var in task_params['<STR_LIT>']<EOL>if var.recursive<EOL>}<EOL>return _remove_empty_items(task, ['<STR_LIT>'])<EOL>

Return a dict populated for serialization (as YAML/JSON).


job_metadata = self.job_metadata<EOL>job_resources = self.job_resources<EOL>job_params = self.job_params<EOL>task_descriptors = self.task_descriptors<EOL>job = {<EOL>'<STR_LIT>': job_metadata.get('<STR_LIT>'),<EOL>'<STR_LIT>': job_metadata.get('<STR_LIT>'),<EOL>'<STR_LIT>': job_metadata.get('<STR_LIT>'),<EOL>'<STR_LIT>': job_metadata.get('<STR_LIT>'),<EOL>'<STR_LIT>': job_metadata.get('<STR_LIT>'),<EOL>'<STR_LIT>': job_metadata.get('<STR_LIT>'),<EOL>'<STR_LIT>': job_metadata.get('<STR_LIT>'),<EOL>'<STR_LIT>': job_metadata['<STR_LIT>'].name,<EOL>}<EOL>if job_resources.logging:<EOL><INDENT>job['<STR_LIT>'] = str(job_resources.logging.uri)<EOL><DEDENT>job['<STR_LIT>'] = {var.name: var.value for var in job_params['<STR_LIT>']}<EOL>job['<STR_LIT>'] = {var.name: var.value for var in job_params['<STR_LIT>']}<EOL>job['<STR_LIT>'] = {<EOL>var.name: var.value<EOL>for var in job_params['<STR_LIT>']<EOL>if not var.recursive<EOL>}<EOL>job['<STR_LIT>'] = {<EOL>var.name: var.value<EOL>for var in job_params['<STR_LIT>']<EOL>if var.recursive<EOL>}<EOL>job['<STR_LIT>'] = {<EOL>var.name: var.value<EOL>for var in job_params['<STR_LIT>']<EOL>if not var.recursive<EOL>}<EOL>job['<STR_LIT>'] = {<EOL>var.name: var.value<EOL>for var in job_params['<STR_LIT>']<EOL>if var.recursive<EOL>}<EOL>job['<STR_LIT>'] = {var.name: var.value for var in job_params['<STR_LIT>']}<EOL>tasks = []<EOL>for task_descriptor in task_descriptors:<EOL><INDENT>tasks.append(task_descriptor.to_serializable())<EOL><DEDENT>job['<STR_LIT>'] = tasks<EOL>return _remove_empty_items(job, [])<EOL>


return yaml.dump(self.to_serializable(), default_flow_style=False)<EOL>

Return a YAML string representing the job and task data.

        A provider's internal representation of a dsub task typically does not map
        1-1 to the dsub representation. For example, the Google Genomics Pipeline
        does not natively support "input-recursive" or "output-recursive", so the
        google provider cannot easily reconstruct the user inputs from the
        pipeline's associated Operation object.

        All providers are likely to need a way to reliably serialize job and task-
        related information, either for dstat or for any type of "retry" mechanism
        we might want to build.

        Returns:
          YAML string


<EOL>job_metadata = {}<EOL>for key in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>job_metadata[key] = job.get(key)<EOL><DEDENT>job_metadata['<STR_LIT>'] = dsub_util.replace_timezone(<EOL>datetime.datetime.strptime(job['<STR_LIT>'], '<STR_LIT>'),<EOL>tzlocal())<EOL>job_resources = Resources()<EOL>params = {}<EOL>labels = job.get('<STR_LIT>', {})<EOL>if '<STR_LIT>' in labels:<EOL><INDENT>job_metadata['<STR_LIT>'] = labels['<STR_LIT>']<EOL>del labels['<STR_LIT>']<EOL><DEDENT>params['<STR_LIT>'] = cls._label_params_from_dict(labels)<EOL>params['<STR_LIT>'] = cls._env_params_from_dict(job.get('<STR_LIT>', {}))<EOL>params['<STR_LIT>'] = cls._input_file_params_from_dict(<EOL>job.get('<STR_LIT>', {}), False)<EOL>params['<STR_LIT>'] = cls._output_file_params_from_dict(<EOL>job.get('<STR_LIT>', {}), False)<EOL>if job.get('<STR_LIT>') is None:<EOL><INDENT>job_params = params<EOL>task_metadata = {'<STR_LIT>': None}<EOL>task_params = {}<EOL><DEDENT>else:<EOL><INDENT>job_params = {}<EOL>task_metadata = {'<STR_LIT>': str(job.get('<STR_LIT>'))}<EOL>task_params = params<EOL><DEDENT>task_resources = Resources(logging_path=job.get('<STR_LIT>'))<EOL>task_descriptors = [<EOL>TaskDescriptor.get_complete_descriptor(task_metadata, task_params,<EOL>task_resources)<EOL>]<EOL>return JobDescriptor.get_complete_descriptor(<EOL>job_metadata, job_params, job_resources, task_descriptors)<EOL>

Populate a JobDescriptor from the local provider's original meta.yaml.

        The local job provider had the first incarnation of a YAML file for each
        task. That idea was extended here in the JobDescriptor and the local
        provider adopted the JobDescriptor.to_yaml() call to write its meta.yaml.

        The JobDescriptor.from_yaml() detects if it receives a local provider's
        "v0" meta.yaml and calls this function.

        Args:
          job: an object produced from decoding meta.yaml.

        Returns:
          A JobDescriptor populated as best we can from the old meta.yaml.


try:<EOL><INDENT>job = yaml.full_load(yaml_string)<EOL><DEDENT>except AttributeError:<EOL><INDENT>job = yaml.load(yaml_string)<EOL><DEDENT>dsub_version = job.get('<STR_LIT>')<EOL>if not dsub_version:<EOL><INDENT>return cls._from_yaml_v0(job)<EOL><DEDENT>job_metadata = {}<EOL>for key in [<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>'<EOL>]:<EOL><INDENT>if job.get(key) is not None:<EOL><INDENT>job_metadata[key] = job.get(key)<EOL><DEDENT><DEDENT>job_metadata['<STR_LIT>'] = dsub_util.replace_timezone(<EOL>job.get('<STR_LIT>'), pytz.utc)<EOL>job_resources = Resources(logging=job.get('<STR_LIT>'))<EOL>job_params = {}<EOL>job_params['<STR_LIT>'] = cls._label_params_from_dict(job.get('<STR_LIT>', {}))<EOL>job_params['<STR_LIT>'] = cls._env_params_from_dict(job.get('<STR_LIT>', {}))<EOL>job_params['<STR_LIT>'] = cls._input_file_params_from_dict(<EOL>job.get('<STR_LIT>', {}), False)<EOL>job_params['<STR_LIT>'] = cls._input_file_params_from_dict(<EOL>job.get('<STR_LIT>', {}), True)<EOL>job_params['<STR_LIT>'] = cls._output_file_params_from_dict(<EOL>job.get('<STR_LIT>', {}), False)<EOL>job_params['<STR_LIT>'] = cls._output_file_params_from_dict(<EOL>job.get('<STR_LIT>', {}), True)<EOL>job_params['<STR_LIT>'] = cls._mount_params_from_dict(job.get('<STR_LIT>', {}))<EOL>task_descriptors = []<EOL>for task in job.get('<STR_LIT>', []):<EOL><INDENT>task_metadata = {'<STR_LIT>': task.get('<STR_LIT>')}<EOL>create_time = task.get('<STR_LIT>')<EOL>if create_time:<EOL><INDENT>task_metadata['<STR_LIT>'] = dsub_util.replace_timezone(<EOL>create_time, pytz.utc)<EOL><DEDENT>if task.get('<STR_LIT>') is not None:<EOL><INDENT>task_metadata['<STR_LIT>'] = task.get('<STR_LIT>')<EOL><DEDENT>task_params = {}<EOL>task_params['<STR_LIT>'] = cls._label_params_from_dict(<EOL>task.get('<STR_LIT>', {}))<EOL>task_params['<STR_LIT>'] = cls._env_params_from_dict(task.get('<STR_LIT>', {}))<EOL>task_params['<STR_LIT>'] = cls._input_file_params_from_dict(<EOL>task.get('<STR_LIT>', {}), False)<EOL>task_params['<STR_LIT>'] = cls._input_file_params_from_dict(<EOL>task.get('<STR_LIT>', {}), True)<EOL>task_params['<STR_LIT>'] = cls._output_file_params_from_dict(<EOL>task.get('<STR_LIT>', {}), False)<EOL>task_params['<STR_LIT>'] = cls._output_file_params_from_dict(<EOL>task.get('<STR_LIT>', {}), True)<EOL>task_resources = Resources(logging_path=task.get('<STR_LIT>'))<EOL>task_descriptors.append(<EOL>TaskDescriptor(task_metadata, task_params, task_resources))<EOL><DEDENT>return JobDescriptor(job_metadata, job_params, job_resources,<EOL>task_descriptors)<EOL>

Populate and return a JobDescriptor from a YAML string.


<EOL>for task_descriptor in self.task_descriptors:<EOL><INDENT>if task_descriptor.task_metadata.get('<STR_LIT>') == task_id:<EOL><INDENT>return task_descriptor<EOL><DEDENT><DEDENT>return None<EOL>

Returns the task_descriptor corresponding to task_id.


env = {}<EOL>for param in file_params:<EOL><INDENT>env[param.name] = os.path.join(<EOL>DATA_MOUNT_POINT, param.docker_path.rstrip('<STR_LIT:/>')) if param.value else '<STR_LIT>'<EOL><DEDENT>return env<EOL>

Return a dictionary of environment variables for the user container.


export_input_dirs = '<STR_LIT:\n>'.join([<EOL>'<STR_LIT>'.format(var.name, destination.rstrip('<STR_LIT:/>'),<EOL>var.docker_path.rstrip('<STR_LIT:/>'))<EOL>for var in inputs<EOL>if var.recursive and var.docker_path<EOL>])<EOL>return export_input_dirs<EOL>

Return a multi-line string with export statements for the variables.

    Arguments:
      destination: Folder where the data will be put.
                   For example /mnt/data
      inputs: a list of InputFileParam

    Returns:
      a multi-line string with a shell script that sets environment variables
      corresponding to the inputs.


def build_recursive_localize_command(destination, inputs, file_provider):

command = _LOCALIZE_COMMAND_MAP[file_provider]<EOL>filtered_inputs = [<EOL>var for var in inputs<EOL>if var.recursive and var.file_provider == file_provider<EOL>]<EOL>copy_input_dirs = '<STR_LIT:\n>'.join([<EOL>textwrap.dedent("""<STR_LIT>""").format(<EOL>command=command,<EOL>source_uri=var.uri,<EOL>data_mount=destination.rstrip('<STR_LIT:/>'),<EOL>docker_path=var.docker_path) for var in filtered_inputs<EOL>])<EOL>return copy_input_dirs<EOL>

Return a multi-line string with a shell script to copy recursively.

    Arguments:
      destination: Folder where to put the data.
                   For example /mnt/data
      inputs: a list of InputFileParam
      file_provider: file provider string used to filter the output params; the
                   returned command will only apply outputs whose file provider
                   matches this file filter.

    Returns:
      a multi-line string with a shell script that copies the inputs
      recursively from GCS.


filtered_outs = [<EOL>var for var in outputs<EOL>if var.recursive and var.file_provider == job_model.P_GCS<EOL>]<EOL>return '<STR_LIT:\n>'.join([<EOL>'<STR_LIT>'.format(var.name,<EOL>source.rstrip('<STR_LIT:/>'),<EOL>var.docker_path.rstrip('<STR_LIT:/>'))<EOL>for var in filtered_outs<EOL>])<EOL>

Return a multi-line string with export statements for the variables.

    Arguments:
      source: Folder with the data.
              For example /mnt/data
      outputs: a list of OutputFileParam

    Returns:
      a multi-line string with a shell script that sets environment variables
      corresponding to the outputs.


def build_recursive_delocalize_command(source, outputs, file_provider):

command = _LOCALIZE_COMMAND_MAP[file_provider]<EOL>filtered_outputs = [<EOL>var for var in outputs<EOL>if var.recursive and var.file_provider == file_provider<EOL>]<EOL>return '<STR_LIT:\n>'.join([<EOL>textwrap.dedent("""<STR_LIT>""").format(<EOL>command=command,<EOL>data_mount=source.rstrip('<STR_LIT:/>'),<EOL>docker_path=var.docker_path,<EOL>destination_uri=var.uri) for var in filtered_outputs<EOL>])<EOL>

Return a multi-line string with a shell script to copy recursively.

    Arguments:
      source: Folder with the data.
              For example /mnt/data
      outputs: a list of OutputFileParam.
      file_provider: file provider string used to filter the output params; the
                   returned command will only apply outputs whose file provider
                   matches this file filter.

    Returns:
      a multi-line string with a shell script that copies the inputs
      recursively to GCS.


task_metadata = job_metadata.copy()<EOL>task_metadata['<STR_LIT>'] = task_id<EOL>return task_metadata<EOL>

Returns a dict combining job metadata with the task id.


return '<STR_LIT:\n>'.join([<EOL>'<STR_LIT>'.format(var.name, source.rstrip('<STR_LIT:/>'),<EOL>var.docker_path.rstrip('<STR_LIT:/>')) for var in mounts<EOL>])<EOL>

Return a multi-line string with export statements for the variables.

    Arguments:
      source: Folder with the data. For example /mnt/data
      mounts: a list of MountParam

    Returns:
      a multi-line string with a shell script that sets environment variables
      corresponding to the mounts.


def get_job_and_task_param(job_params, task_params, field):

return job_params.get(field, set()) | task_params.get(field, set())<EOL>

Returns a dict combining the field for job and task params.


@classmethod<EOL><INDENT>def _build_pipeline_input_file_param(cls, var_name, docker_path):<DEDENT>

<EOL>path, filename = os.path.split(docker_path)<EOL>if '<STR_LIT:*>' in filename:<EOL><INDENT>return cls._build_pipeline_file_param(var_name, path + '<STR_LIT:/>')<EOL><DEDENT>else:<EOL><INDENT>return cls._build_pipeline_file_param(var_name, docker_path)<EOL><DEDENT>

Return a dict object representing a pipeline input argument.


@classmethod<EOL><INDENT>def _build_pipeline_file_param(cls, var_name, docker_path):<DEDENT>

return {<EOL>'<STR_LIT:name>': var_name,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:path>': docker_path,<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>}<EOL>}<EOL>

Return a dict object representing a pipeline input or output argument.


@classmethod<EOL><INDENT>def _build_pipeline_docker_command(cls, script_name, inputs, outputs, envs):<DEDENT>

<EOL>recursive_input_dirs = [<EOL>var for var in inputs if var.recursive and var.value<EOL>]<EOL>recursive_output_dirs = [<EOL>var for var in outputs if var.recursive and var.value<EOL>]<EOL>install_cloud_sdk = '<STR_LIT>'<EOL>if recursive_input_dirs or recursive_output_dirs:<EOL><INDENT>install_cloud_sdk = INSTALL_CLOUD_SDK<EOL><DEDENT>export_input_dirs = '<STR_LIT>'<EOL>copy_input_dirs = '<STR_LIT>'<EOL>if recursive_input_dirs:<EOL><INDENT>export_input_dirs = providers_util.build_recursive_localize_env(<EOL>providers_util.DATA_MOUNT_POINT, inputs)<EOL>copy_input_dirs = providers_util.build_recursive_localize_command(<EOL>providers_util.DATA_MOUNT_POINT, inputs, job_model.P_GCS)<EOL><DEDENT>export_output_dirs = '<STR_LIT>'<EOL>copy_output_dirs = '<STR_LIT>'<EOL>if recursive_output_dirs:<EOL><INDENT>export_output_dirs = providers_util.build_recursive_gcs_delocalize_env(<EOL>providers_util.DATA_MOUNT_POINT, outputs)<EOL>copy_output_dirs = providers_util.build_recursive_delocalize_command(<EOL>providers_util.DATA_MOUNT_POINT, outputs, job_model.P_GCS)<EOL><DEDENT>docker_paths = [<EOL>var.docker_path if var.recursive else os.path.dirname(var.docker_path)<EOL>for var in outputs<EOL>if var.value<EOL>]<EOL>mkdirs = '<STR_LIT:\n>'.join([<EOL>'<STR_LIT>'.format(providers_util.DATA_MOUNT_POINT, path)<EOL>for path in docker_paths<EOL>])<EOL>inputs_with_wildcards = [<EOL>var for var in inputs if not var.recursive and var.docker_path and<EOL>'<STR_LIT:*>' in os.path.basename(var.docker_path)<EOL>]<EOL>export_inputs_with_wildcards = '<STR_LIT:\n>'.join([<EOL>'<STR_LIT>'.format(var.name, providers_util.DATA_MOUNT_POINT,<EOL>var.docker_path)<EOL>for var in inputs_with_wildcards<EOL>])<EOL>export_empty_envs = '<STR_LIT:\n>'.join([<EOL>'<STR_LIT>'.format(var.name)<EOL>for var in envs | inputs | outputs<EOL>if not var.value<EOL>])<EOL>return DOCKER_COMMAND.format(<EOL>mk_runtime_dirs=MK_RUNTIME_DIRS_COMMAND,<EOL>script_path='<STR_LIT>' % (providers_util.SCRIPT_DIR, script_name),<EOL>install_cloud_sdk=install_cloud_sdk,<EOL>export_inputs_with_wildcards=export_inputs_with_wildcards,<EOL>export_input_dirs=export_input_dirs,<EOL>copy_input_dirs=copy_input_dirs,<EOL>mk_output_dirs=mkdirs,<EOL>export_output_dirs=export_output_dirs,<EOL>export_empty_envs=export_empty_envs,<EOL>tmpdir=providers_util.TMP_DIR,<EOL>working_dir=providers_util.WORKING_DIR,<EOL>copy_output_dirs=copy_output_dirs)<EOL>

Return a multi-line string of the full pipeline docker command.


@classmethod<EOL><INDENT>def build_pipeline(cls, project, zones, min_cores, min_ram, disk_size,<EOL>boot_disk_size, preemptible, accelerator_type,<EOL>accelerator_count, image, script_name, envs, inputs,<EOL>outputs, pipeline_name):<DEDENT>

if min_cores is None:<EOL><INDENT>min_cores = job_model.DEFAULT_MIN_CORES<EOL><DEDENT>if min_ram is None:<EOL><INDENT>min_ram = job_model.DEFAULT_MIN_RAM<EOL><DEDENT>if disk_size is None:<EOL><INDENT>disk_size = job_model.DEFAULT_DISK_SIZE<EOL><DEDENT>if boot_disk_size is None:<EOL><INDENT>boot_disk_size = job_model.DEFAULT_BOOT_DISK_SIZE<EOL><DEDENT>if preemptible is None:<EOL><INDENT>preemptible = job_model.DEFAULT_PREEMPTIBLE<EOL><DEDENT>docker_command = cls._build_pipeline_docker_command(script_name, inputs,<EOL>outputs, envs)<EOL>input_envs = [{<EOL>'<STR_LIT:name>': SCRIPT_VARNAME<EOL>}] + [{<EOL>'<STR_LIT:name>': env.name<EOL>} for env in envs if env.value]<EOL>input_files = [<EOL>cls._build_pipeline_input_file_param(var.name, var.docker_path)<EOL>for var in inputs<EOL>if not var.recursive and var.value<EOL>]<EOL>output_files = [<EOL>cls._build_pipeline_file_param(var.name, var.docker_path)<EOL>for var in outputs<EOL>if not var.recursive and var.value<EOL>]<EOL>return {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': project,<EOL>'<STR_LIT:name>': pipeline_name,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': min_cores,<EOL>'<STR_LIT>': min_ram,<EOL>'<STR_LIT>': boot_disk_size,<EOL>'<STR_LIT>': preemptible,<EOL>'<STR_LIT>': google_base.get_zones(zones),<EOL>'<STR_LIT>': accelerator_type,<EOL>'<STR_LIT>': accelerator_count,<EOL>'<STR_LIT>': [{<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT>': True,<EOL>'<STR_LIT>': disk_size,<EOL>'<STR_LIT>': providers_util.DATA_MOUNT_POINT,<EOL>}],<EOL>},<EOL>'<STR_LIT>': input_envs + input_files,<EOL>'<STR_LIT>': output_files,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': image,<EOL>'<STR_LIT>': docker_command,<EOL>}<EOL>}<EOL>}<EOL>

Builds a pipeline configuration for execution.

        Args:
          project: string name of project.
          zones: list of zone names for jobs to be run at.
          min_cores: int number of CPU cores required per job.
          min_ram: int GB of RAM required per job.
          disk_size: int GB of disk to attach under /mnt/data.
          boot_disk_size: int GB of disk for boot.
          preemptible: use a preemptible VM for the job
          accelerator_type: string GCE defined accelerator type.
          accelerator_count: int number of accelerators of the specified type to
            attach.
          image: string Docker image name in which to run.
          script_name: file name of the script to run.
          envs: list of EnvParam objects specifying environment variables to set
            within each job.
          inputs: list of FileParam objects specifying input variables to set
            within each job.
          outputs: list of FileParam objects specifying output variables to set
            within each job.
          pipeline_name: string name of pipeline.

        Returns:
          A nested dictionary with one entry under the key ephemeralPipeline
          containing the pipeline configuration.


@classmethod<EOL><INDENT>def build_pipeline_args(cls, project, script, job_params, task_params,<EOL>reserved_labels, preemptible, logging_uri, scopes,<EOL>keep_alive):<DEDENT>

<EOL>inputs = {}<EOL>inputs.update({SCRIPT_VARNAME: script})<EOL>inputs.update({<EOL>var.name: var.value<EOL>for var in job_params['<STR_LIT>'] | task_params['<STR_LIT>']<EOL>if var.value<EOL>})<EOL>inputs.update({<EOL>var.name: var.uri<EOL>for var in job_params['<STR_LIT>'] | task_params['<STR_LIT>']<EOL>if not var.recursive and var.value<EOL>})<EOL>outputs = {}<EOL>for var in job_params['<STR_LIT>'] | task_params['<STR_LIT>']:<EOL><INDENT>if var.recursive or not var.value:<EOL><INDENT>continue<EOL><DEDENT>if '<STR_LIT:*>' in var.uri.basename:<EOL><INDENT>outputs[var.name] = var.uri.path<EOL><DEDENT>else:<EOL><INDENT>outputs[var.name] = var.uri<EOL><DEDENT><DEDENT>labels = {}<EOL>labels.update({<EOL>label.name: label.value if label.value else '<STR_LIT>'<EOL>for label in (reserved_labels | job_params['<STR_LIT>']<EOL>| task_params['<STR_LIT>'])<EOL>})<EOL>args = {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': project,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': preemptible,<EOL>},<EOL>'<STR_LIT>': inputs,<EOL>'<STR_LIT>': outputs,<EOL>'<STR_LIT>': labels,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:email>': '<STR_LIT:default>',<EOL>'<STR_LIT>': scopes,<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': logging_uri<EOL>},<EOL>}<EOL>}<EOL>if keep_alive:<EOL><INDENT>args['<STR_LIT>'][<EOL>'<STR_LIT>'] = '<STR_LIT>' % keep_alive<EOL><DEDENT>return args<EOL>

Builds pipeline args for execution.

        Args:
          project: string name of project.
          script: Body of the script to execute.
          job_params: dictionary of values for labels, envs, inputs, and outputs
              for this job.
          task_params: dictionary of values for labels, envs, inputs, and outputs
              for this task.
          reserved_labels: dictionary of reserved labels (e.g. task-id,
              task-attempt)
          preemptible: use a preemptible VM for the job
          logging_uri: path for job logging output.
          scopes: list of scope.
          keep_alive: Seconds to keep VM alive on failure

        Returns:
          A nested dictionary with one entry under the key pipelineArgs containing
          the pipeline arguments.


if date is None:<EOL><INDENT>return None<EOL><DEDENT>epoch = dsub_util.replace_timezone(datetime.utcfromtimestamp(<NUM_LIT:0>), pytz.utc)<EOL>return (date - epoch).total_seconds()<EOL>

Convert the integer UTC time value into a local datetime.


ops_filter = ['<STR_LIT>' % project]<EOL>if status and status != '<STR_LIT:*>':<EOL><INDENT>ops_filter.append('<STR_LIT>' % status)<EOL><DEDENT>if user_id != '<STR_LIT:*>':<EOL><INDENT>ops_filter.append('<STR_LIT>' % user_id)<EOL><DEDENT>if job_id != '<STR_LIT:*>':<EOL><INDENT>ops_filter.append('<STR_LIT>' % job_id)<EOL><DEDENT>if job_name != '<STR_LIT:*>':<EOL><INDENT>ops_filter.append('<STR_LIT>' % job_name)<EOL><DEDENT>if task_id != '<STR_LIT:*>':<EOL><INDENT>ops_filter.append('<STR_LIT>' % task_id)<EOL><DEDENT>if task_attempt != '<STR_LIT:*>':<EOL><INDENT>ops_filter.append('<STR_LIT>' % task_attempt)<EOL><DEDENT>if labels:<EOL><INDENT>for l in labels:<EOL><INDENT>ops_filter.append('<STR_LIT>' % (l.name, l.value))<EOL><DEDENT><DEDENT>epoch = dsub_util.replace_timezone(datetime.utcfromtimestamp(<NUM_LIT:0>), pytz.utc)<EOL>if create_time_min:<EOL><INDENT>create_time_min_utc_int = (create_time_min - epoch).total_seconds()<EOL>ops_filter.append('<STR_LIT>' % create_time_min_utc_int)<EOL><DEDENT>if create_time_max:<EOL><INDENT>create_time_max_utc_int = (create_time_max - epoch).total_seconds()<EOL>ops_filter.append('<STR_LIT>' % create_time_max_utc_int)<EOL><DEDENT>return '<STR_LIT>'.join(ops_filter)<EOL>

Return a filter string for operations.list().


return cls.get_operation_type(<EOL>op) == '<STR_LIT>'<EOL>

Check that an operation is a genomics pipeline run.

        An operation is a Genomics Pipeline run if the request metadata's @type
        is "type.googleapis.com/google.genomics.v1alpha2.RunPipelineRequest.

        Args:
          op: a pipelines operation.

        Returns:
          Boolean, true if the operation is a RunPipelineRequest.


if not cls.is_pipelines_operation(op):<EOL><INDENT>return False<EOL><DEDENT>for name in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>if not cls.get_operation_label(op, name):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>

Determine if a pipelines operation is a dsub request.

        We don't have a rigorous way to identify an operation as being submitted
        by dsub. Our best option is to check for certain fields that have always
        been part of dsub operations.

        - labels: job-id, job-name, and user-id have always existed
        - envs: _SCRIPT has always existed.

        In order to keep a simple heuristic this test only uses labels.
        Args:
          op: a pipelines operation.

        Returns:
          Boolean, true if the pipeline run was generated by dsub.


@classmethod<EOL><INDENT>def list(cls, service, ops_filter, page_size=<NUM_LIT:0>):<DEDENT>

page_token = None<EOL>more_operations = True<EOL>documented_default_page_size = <NUM_LIT><EOL>documented_max_page_size = <NUM_LIT><EOL>if not page_size:<EOL><INDENT>page_size = documented_default_page_size<EOL><DEDENT>page_size = min(page_size, documented_max_page_size)<EOL>while more_operations:<EOL><INDENT>api = service.operations().list(<EOL>name='<STR_LIT>',<EOL>filter=ops_filter,<EOL>pageToken=page_token,<EOL>pageSize=page_size)<EOL>response = google_base.Api.execute(api)<EOL>ops = response.get('<STR_LIT>', [])<EOL>for op in ops:<EOL><INDENT>if cls.is_dsub_operation(op):<EOL><INDENT>yield GoogleOperation(op)<EOL><DEDENT><DEDENT>page_token = response.get('<STR_LIT>')<EOL>more_operations = bool(page_token)<EOL><DEDENT>

Gets the list of operations for the specified filter.

        Args:
          service: Google Genomics API service object
          ops_filter: string filter of operations to return
          page_size: the number of operations to requested on each list operation to
            the pipelines API (if 0 or None, the API default is used)

        Yields:
          Operations matching the filter criteria.


def prepare_job_metadata(self, script, job_name, user_id, create_time):

return google_base.prepare_job_metadata(script, job_name, user_id,<EOL>create_time)<EOL>

Returns a dictionary of metadata fields for the job.


job_metadata = task_view.job_metadata<EOL>job_params = task_view.job_params<EOL>job_resources = task_view.job_resources<EOL>task_metadata = task_view.task_descriptors[<NUM_LIT:0>].task_metadata<EOL>task_params = task_view.task_descriptors[<NUM_LIT:0>].task_params<EOL>task_resources = task_view.task_descriptors[<NUM_LIT:0>].task_resources<EOL>script = task_view.job_metadata['<STR_LIT>']<EOL>reserved_labels = google_base.build_pipeline_labels(<EOL>job_metadata, task_metadata, task_id_pattern='<STR_LIT>')<EOL>pipeline = _Pipelines.build_pipeline(<EOL>project=self._project,<EOL>zones=job_resources.zones,<EOL>min_cores=job_resources.min_cores,<EOL>min_ram=job_resources.min_ram,<EOL>disk_size=job_resources.disk_size,<EOL>boot_disk_size=job_resources.boot_disk_size,<EOL>preemptible=job_resources.preemptible,<EOL>accelerator_type=job_resources.accelerator_type,<EOL>accelerator_count=job_resources.accelerator_count,<EOL>image=job_resources.image,<EOL>script_name=script.name,<EOL>envs=job_params['<STR_LIT>'] | task_params['<STR_LIT>'],<EOL>inputs=job_params['<STR_LIT>'] | task_params['<STR_LIT>'],<EOL>outputs=job_params['<STR_LIT>'] | task_params['<STR_LIT>'],<EOL>pipeline_name=job_metadata['<STR_LIT>'])<EOL>logging_uri = task_resources.logging_path.uri<EOL>scopes = job_resources.scopes or google_base.DEFAULT_SCOPES<EOL>pipeline.update(<EOL>_Pipelines.build_pipeline_args(self._project, script.value, job_params,<EOL>task_params, reserved_labels,<EOL>job_resources.preemptible, logging_uri,<EOL>scopes, job_resources.keep_alive))<EOL>return pipeline<EOL>


def submit_job(self, job_descriptor, skip_if_output_present):

<EOL>param_util.validate_submit_args_or_fail(<EOL>job_descriptor,<EOL>provider_name=_PROVIDER_NAME,<EOL>input_providers=_SUPPORTED_INPUT_PROVIDERS,<EOL>output_providers=_SUPPORTED_OUTPUT_PROVIDERS,<EOL>logging_providers=_SUPPORTED_LOGGING_PROVIDERS)<EOL>launched_tasks = []<EOL>requests = []<EOL>for task_view in job_model.task_view_generator(job_descriptor):<EOL><INDENT>job_params = task_view.job_params<EOL>task_params = task_view.task_descriptors[<NUM_LIT:0>].task_params<EOL>outputs = job_params['<STR_LIT>'] | task_params['<STR_LIT>']<EOL>if skip_if_output_present:<EOL><INDENT>if dsub_util.outputs_are_present(outputs):<EOL><INDENT>print('<STR_LIT>')<EOL>continue<EOL><DEDENT><DEDENT>request = self._build_pipeline_request(task_view)<EOL>if self._dry_run:<EOL><INDENT>requests.append(request)<EOL><DEDENT>else:<EOL><INDENT>task_id = self._submit_pipeline(request)<EOL>launched_tasks.append(task_id)<EOL><DEDENT><DEDENT>if self._dry_run:<EOL><INDENT>print(json.dumps(requests, indent=<NUM_LIT:2>, sort_keys=True))<EOL><DEDENT>if not requests and not launched_tasks:<EOL><INDENT>return {'<STR_LIT>': dsub_util.NO_JOB}<EOL><DEDENT>return {<EOL>'<STR_LIT>': job_descriptor.job_metadata['<STR_LIT>'],<EOL>'<STR_LIT>': job_descriptor.job_metadata['<STR_LIT>'],<EOL>'<STR_LIT>': [task_id for task_id in launched_tasks if task_id],<EOL>}<EOL>

Submit the job (or tasks) to be executed.

        Args:
          job_descriptor: all parameters needed to launch all job tasks
          skip_if_output_present: (boolean) if true, skip tasks whose output
            is present (see --skip flag for more explanation).

        Returns:
          A dictionary containing the 'user-id', 'job-id', and 'task-id' list.
          For jobs that are not task array jobs, the task-id list should be empty.

        Raises:
          ValueError: if job resources or task data contain illegal values.


<EOL>statuses = statuses if statuses else {'<STR_LIT:*>'}<EOL>user_ids = user_ids if user_ids else {'<STR_LIT:*>'}<EOL>job_ids = job_ids if job_ids else {'<STR_LIT:*>'}<EOL>job_names = job_names if job_names else {'<STR_LIT:*>'}<EOL>task_ids = task_ids if task_ids else {'<STR_LIT:*>'}<EOL>task_attempts = task_attempts if task_attempts else {'<STR_LIT:*>'}<EOL>task_ids = {'<STR_LIT>'.format(t) if t.isdigit() else t for t in task_ids}<EOL>if job_ids != {'<STR_LIT:*>'} and job_names != {'<STR_LIT:*>'}:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>')<EOL><DEDENT>labels = labels if labels else set()<EOL>now = datetime.now()<EOL>def _desc_date_sort_key(t):<EOL><INDENT>return now - dsub_util.replace_timezone(t.get_field('<STR_LIT>'), None)<EOL><DEDENT>query_queue = sorting_util.SortedGeneratorIterator(key=_desc_date_sort_key)<EOL>for status, job_id, job_name, user_id, task_id, task_attempt in (<EOL>itertools.product(statuses, job_ids, job_names, user_ids, task_ids,<EOL>task_attempts)):<EOL><INDENT>ops_filter = _Operations.get_filter(<EOL>self._project,<EOL>status=status,<EOL>user_id=user_id,<EOL>job_id=job_id,<EOL>job_name=job_name,<EOL>labels=labels,<EOL>task_id=task_id,<EOL>task_attempt=task_attempt,<EOL>create_time_min=create_time_min,<EOL>create_time_max=create_time_max)<EOL>stream = _Operations.list(self._service, ops_filter, page_size=page_size)<EOL>query_queue.add_generator(stream)<EOL><DEDENT>tasks_yielded = <NUM_LIT:0><EOL>for task in query_queue:<EOL><INDENT>yield task<EOL>tasks_yielded += <NUM_LIT:1><EOL>if <NUM_LIT:0> < max_tasks <= tasks_yielded:<EOL><INDENT>break<EOL><DEDENT><DEDENT>

Yields operations based on the input criteria.

        If any of the filters are empty or {'*'}, then no filtering is performed on
        that field. Filtering by both a job id list and job name list is
        unsupported.

        Args:
          statuses: {'*'}, or a list of job status strings to return. Valid
            status strings are 'RUNNING', 'SUCCESS', 'FAILURE', or 'CANCELED'.
          user_ids: a list of ids for the user(s) who launched the job.
          job_ids: a list of job ids to return.
          job_names: a list of job names to return.
          task_ids: a list of specific tasks within the specified job(s) to return.
          task_attempts: a list of specific attempts within the specified tasks(s)
            to return.
          labels: a list of LabelParam with user-added labels. All labels must
                  match the task being fetched.
          create_time_min: a timezone-aware datetime value for the earliest create
                           time of a task, inclusive.
          create_time_max: a timezone-aware datetime value for the most recent
                           create time of a task, inclusive.
          max_tasks: the maximum number of job tasks to return or 0 for no limit.
          page_size: the page size to use for each query to the pipelins API.

        Raises:
          ValueError: if both a job id list and a job name list are provided

        Yeilds:
          Genomics API Operations objects.


Kills the operations associated with the specified job or job.task.

        Args:
          user_ids: List of user ids who "own" the job(s) to cancel.
          job_ids: List of job_ids to cancel.
          task_ids: List of task-ids to cancel.
          labels: List of LabelParam, each must match the job(s) to be canceled.
          create_time_min: a timezone-aware datetime value for the earliest create
                           time of a task, inclusive.
          create_time_max: a timezone-aware datetime value for the most recent
                           create time of a task, inclusive.

        Returns:
          A list of tasks canceled and a list of error messages.


def get_field(self, field, default=None):

metadata = self._op.get('<STR_LIT>')<EOL>value = None<EOL>if field == '<STR_LIT>':<EOL><INDENT>value = self._op['<STR_LIT:name>']<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = metadata['<STR_LIT>'].get('<STR_LIT>')<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = metadata['<STR_LIT>'].get('<STR_LIT>')<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = metadata['<STR_LIT>'].get('<STR_LIT>')<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = metadata['<STR_LIT>'].get('<STR_LIT>')<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = metadata['<STR_LIT>'].get('<STR_LIT>')<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = metadata['<STR_LIT>'].get('<STR_LIT>')<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = self._operation_status()<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = metadata['<STR_LIT>']['<STR_LIT>']['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = self._get_operation_input_field_values(metadata, False)<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = {<EOL>k: v<EOL>for k, v in metadata['<STR_LIT>'].items()<EOL>if k not in job_model.RESERVED_LABELS<EOL>}<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = self._get_operation_input_field_values(metadata, True)<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = self._get_operation_output_field_values(metadata)<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = None<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = google_base.parse_rfc3339_utc_string(metadata['<STR_LIT>'])<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>start_events = [<EOL>e for e in metadata.get('<STR_LIT>', []) if e['<STR_LIT:description>'] == '<STR_LIT:start>'<EOL>]<EOL>if start_events:<EOL><INDENT>value = google_base.parse_rfc3339_utc_string(<EOL>start_events[-<NUM_LIT:1>]['<STR_LIT>'])<EOL><DEDENT><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>if '<STR_LIT>' in metadata:<EOL><INDENT>value = google_base.parse_rfc3339_utc_string(metadata['<STR_LIT>'])<EOL><DEDENT><DEDENT>elif field == '<STR_LIT:status>':<EOL><INDENT>value = self._operation_status()<EOL><DEDENT>elif field in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>status, last_update = self._operation_status_message()<EOL>value = status<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>status, last_update = self._operation_status_message()<EOL>value = last_update<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>return _PROVIDER_NAME<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>gce_data = metadata.get('<STR_LIT>', {}).get('<STR_LIT>', {})<EOL>if '<STR_LIT>' in gce_data:<EOL><INDENT>machine_type = gce_data.get('<STR_LIT>').rpartition('<STR_LIT:/>')[<NUM_LIT:2>]<EOL><DEDENT>else:<EOL><INDENT>machine_type = None<EOL><DEDENT>instance_name = gce_data.get('<STR_LIT>')<EOL>instance_zone = gce_data.get('<STR_LIT>')<EOL>value = {<EOL>'<STR_LIT>': machine_type,<EOL>'<STR_LIT>': instance_name,<EOL>'<STR_LIT>': instance_zone,<EOL>}<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>events = metadata.get('<STR_LIT>', [])<EOL>value = []<EOL>for event in events:<EOL><INDENT>event_value = {<EOL>'<STR_LIT:name>':<EOL>event.get('<STR_LIT:description>', '<STR_LIT>'),<EOL>'<STR_LIT>':<EOL>google_base.parse_rfc3339_utc_string(event['<STR_LIT>'])<EOL>}<EOL>if '<STR_LIT>' in event:<EOL><INDENT>event_value['<STR_LIT>'] = google_base.parse_rfc3339_utc_string(<EOL>event['<STR_LIT>'])<EOL><DEDENT>value.append(event_value)<EOL><DEDENT><DEDENT>elif field in [<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>'<EOL>]:<EOL><INDENT>value = None<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' % field)<EOL><DEDENT>return value if value else default<EOL>

Returns a value from the operation for a specific set of field names.

        Args:
          field: a dsub-specific job metadata key
          default: default value to return if field does not exist or is empty.

        Returns:
          A text string for the field or a list for 'inputs'.

        Raises:
          ValueError: if the field label is not supported by the operation


if not self._op['<STR_LIT>']:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>if '<STR_LIT:error>' not in self._op:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>if self._op['<STR_LIT:error>'].get('<STR_LIT:code>', <NUM_LIT:0>) == <NUM_LIT:1>:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>return '<STR_LIT>'<EOL>

Returns the status of this operation.

        ie. RUNNING, SUCCESS, CANCELED or FAILURE.

        Returns:
          A printable status string


metadata = self._op['<STR_LIT>']<EOL>if not self._op['<STR_LIT>']:<EOL><INDENT>if '<STR_LIT>' in metadata and metadata['<STR_LIT>']:<EOL><INDENT>last_event = metadata['<STR_LIT>'][-<NUM_LIT:1>]<EOL>msg = last_event['<STR_LIT:description>']<EOL>ds = last_event['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>msg = '<STR_LIT>'<EOL>ds = metadata['<STR_LIT>']<EOL><DEDENT><DEDENT>else:<EOL><INDENT>ds = metadata['<STR_LIT>']<EOL>if '<STR_LIT:error>' in self._op:<EOL><INDENT>msg = self._op['<STR_LIT:error>']['<STR_LIT:message>']<EOL><DEDENT>else:<EOL><INDENT>msg = '<STR_LIT>'<EOL><DEDENT><DEDENT>return (msg, google_base.parse_rfc3339_utc_string(ds))<EOL>

Returns the most relevant status string and last updated date string.

        This string is meant for display only.

        Returns:
          A printable status string and date string.


def _get_operation_input_field_values(self, metadata, file_input):

<EOL>input_args = metadata['<STR_LIT>']['<STR_LIT>']['<STR_LIT>']<EOL>vals_dict = metadata['<STR_LIT>']['<STR_LIT>']['<STR_LIT>']<EOL>names = [<EOL>arg['<STR_LIT:name>'] for arg in input_args if ('<STR_LIT>' in arg) == file_input<EOL>]<EOL>return {name: vals_dict[name] for name in names if name in vals_dict}<EOL>

Returns a dictionary of envs or file inputs for an operation.

        Args:
          metadata: operation metadata field
          file_input: True to return a dict of file inputs, False to return envs.

        Returns:
          A dictionary of input field name value pairs


if '<STR_LIT:error>' in self._op:<EOL><INDENT>if '<STR_LIT>' in self._op['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>job_id = self._op['<STR_LIT>']['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>job_id = self._op['<STR_LIT>']['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>return '<STR_LIT>' % (job_id,<EOL>self._op['<STR_LIT:error>']['<STR_LIT:code>'],<EOL>self._op['<STR_LIT:error>']['<STR_LIT:message>'])<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>

Returns an error message if the operation failed for any reason.

        Failure as defined here means; ended for any reason other than 'success'.
        This means that a successful cancelation will also create an error message
        here.

        Returns:
          string, string will be empty if job did not error.


@abc.abstractmethod<EOL><INDENT>def prepare_job_metadata(self, script, job_name, user_id, create_time):<DEDENT>

raise NotImplementedError()<EOL>

Returns a dictionary of metadata fields for the job.

        Call this before calling submit_job.

        The job metadata is a dictionary of values relevant to the job as a whole,
        such as:

        * job-name
        * job-id
        * user-id
        * script
        * dsub-version

        The creation of the job metadata is done by the provider, as the rules
        around these items are provider-specific. For example, one job provider
        might have an 8 character restriction on job names, while another may be
        restricted to lower-case characters.

        The provider *must* set the job-name, job-id, and user-id. They may be used
        by the dsub infrastructure for display and direct return to callers.

        The provider is free to set any other key that it will need in
        submit_job().

        The "job_name" passed in need not be the same 'job-name' value set in the
        returned job metadata, as provider-specific character restrictions may need
        to be applied.

        Args:
          script: path to the job script
          job_name: user-supplied job name, if any
          user_id: user whose jobs to look for
          create_time: create time for the job


@abc.abstractmethod<EOL><INDENT>def submit_job(self, job_descriptor, skip_if_output_present):<DEDENT>

raise NotImplementedError()<EOL>

Submit the job to be executed.

        Args:
          job_descriptor (job_model.JobDescriptor): parameters needed to launch all
          job tasks
          skip_if_output_present: (boolean) if true, skip tasks whose output
            is present (see --skip flag for more explanation).

        Returns:
          A dictionary containing the 'user-id', 'job-id', and 'task-id' list.
          For jobs that are not task array jobs, the task-id list should be empty.
          If all tasks were skipped, then the job-id is dsub_lib.NO_JOB.

        Raises:
          ValueError: submit_job may validate any of the parameters and raise
            a value error if any parameter (or specific combination of parameters)
            is not supported by the provider.


raise NotImplementedError()<EOL>

Kills the operations associated with the specified job or job.task.

        Some providers may provide only a "cancel" operation, which terminates the
        task but does not truly "delete" it from the "task list".

        Args:
          user_ids: a set of user ids who "own" the job(s) to delete.
          job_ids: a set of job ids to delete.
          task_ids: a set of task ids to delete.
          labels: a set of LabelParam, each must match the job(s) to be cancelled.
          create_time_min: a timezone-aware datetime value for the earliest create
                           time of a task, inclusive.
          create_time_max: a timezone-aware datetime value for the most recent
                           create time of a task, inclusive.

        Returns:
          (list of tasks canceled,
           for each task that couldn't be canceled, the error message).

          Only tasks that were running are included in the return value.


raise NotImplementedError()<EOL>

Return a list of tasks based on the search criteria.

        If any of the filters are empty or {'*'}, then no filtering is performed on
        that field. Filtering by both a job id list and job name list is
        unsupported.

        Args:
          statuses: {'*'}, or a set of job status strings to return. Valid
            status strings are 'RUNNING', 'SUCCESS', 'FAILURE', or 'CANCELED'.
          user_ids: a set of ids for the user(s) who launched the job.
          job_ids: a set of job ids to return.
          job_names: a set of job names to return.
          task_ids: a set of specific tasks within the specified job(s) to return.
          task_attempts: a set of specific task attempts within the specified
            tasks(s) to return.
          labels: a list of LabelParam, each must match the job(s) returned.
          create_time_min: a timezone-aware datetime value for the earliest create
                           time of a task, inclusive.
          create_time_max: a timezone-aware datetime value for the most recent
                           create time of a task, inclusive.
          max_tasks: the maximum number of job tasks to return or 0 for no limit.
          page_size: the page size to use for each query to the backend. May be
                     ignored by provider implementations.

        Returns:
          A list of Task objects.

        Raises:
          ValueError: if both a job id list and a job name list are provided


raise NotImplementedError()<EOL>

List of the error message of each given task.


raise NotImplementedError()<EOL>

Return a provider-specific representation of task data.

        Returns:
          dictionary of task data from the provider.


@abc.abstractmethod<EOL><INDENT>def get_field(self, field, default=None):<DEDENT>

raise NotImplementedError()<EOL>

Return a metadata-field for the task.

        Not all fields need to be supported by all providers.
        Field identifiers include:

        'job-name', 'job-id', 'task-id', 'task-attempt', 'user-id', 'task-status',
        'error-message', 'create-time', 'start-time', 'end-time', 'inputs',
        'outputs', 'events'

        The following are required by dstat:
        - status: The task status ('RUNNING', 'CANCELED', 'FAILED', 'SUCCESS')
        - status-message: A short message that is displayed in the default
                          dstat output. This should be as concise and useful as
                          possible ("Pending", "Running", "Error: invalid...")
        - status-detail: A longer status message that is displayed in full dstat
                         output. Ideally, this is the last few lines of a log which
                         gives the user enough information that they do not need
                         to go to the log files.

        dstat's short output shows status-message.
        dstat's full output shows status and status-detail

        Args:
          field: one of the choices listed above.
          default: the value to return if no value if found.


print(msg, file=sys.stderr)<EOL>

Utility routine to emit messages to stderr.


if not input_list:<EOL><INDENT>return []<EOL><DEDENT>output_list = []<EOL>for zone in input_list:<EOL><INDENT>if zone.endswith('<STR_LIT:*>'):<EOL><INDENT>prefix = zone[:-<NUM_LIT:1>]<EOL>output_list.extend([z for z in _ZONES if z.startswith(prefix)])<EOL><DEDENT>else:<EOL><INDENT>output_list.append(zone)<EOL><DEDENT><DEDENT>return output_list<EOL>

Returns a list of zones based on any wildcard input.

    This function is intended to provide an easy method for producing a list
    of desired zones for a pipeline to run in.

    The Pipelines API default zone list is "any zone". The problem with
    "any zone" is that it can lead to incurring Cloud Storage egress charges
    if the GCE zone selected is in a different region than the GCS bucket.
    See https://cloud.google.com/storage/pricing#network-egress.

    A user with a multi-region US bucket would want to pipelines to run in
    a "us-*" zone.
    A user with a regional bucket in US would want to restrict pipelines to
    run in a zone in that region.

    Rarely does the specific zone matter for a pipeline.

    This function allows for a simple short-hand such as:
       [ "us-*" ]
       [ "us-central1-*" ]
    These examples will expand out to the full list of US and us-central1 zones
    respectively.

    Args:
      input_list: list of zone names/patterns

    Returns:
      A list of zones, with any wildcard zone specifications expanded.


def build_pipeline_labels(job_metadata, task_metadata, task_id_pattern=None):

labels = {<EOL>Label(name, job_metadata[name])<EOL>for name in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>}<EOL>task_id = task_metadata.get('<STR_LIT>')<EOL>if task_id is not None:  <EOL><INDENT>if task_id_pattern:<EOL><INDENT>task_id = task_id_pattern % task_id<EOL><DEDENT>labels.add(Label('<STR_LIT>', str(task_id)))<EOL><DEDENT>task_attempt = task_metadata.get('<STR_LIT>')<EOL>if task_attempt is not None:<EOL><INDENT>labels.add(Label('<STR_LIT>', str(task_attempt)))<EOL><DEDENT>return labels<EOL>

Build a set() of standard job and task labels.

    Args:
      job_metadata: Job metadata, such as job-id, job-name, and user-id.
      task_metadata: Task metadata, such as the task-id.
      task_id_pattern: A pattern for the task-id value, such as "task-%d"; the
        original google label values could not be strictly numeric, so "task-"
        was prepended.

    Returns:
      A set of standard dsub Label() objects to attach to a pipeline.


def prepare_job_metadata(script, job_name, user_id, create_time):

<EOL>if job_name:<EOL><INDENT>pipeline_name = job_name<EOL>job_name_value = job_model.convert_to_label_chars(job_name)<EOL><DEDENT>else:<EOL><INDENT>pipeline_name = os.path.basename(script)<EOL>job_name_value = job_model.convert_to_label_chars(<EOL>pipeline_name.split('<STR_LIT:.>', <NUM_LIT:1>)[<NUM_LIT:0>])<EOL><DEDENT>user_id = job_model.convert_to_label_chars(user_id)<EOL>job_id = '<STR_LIT>' % (job_name_value[:<NUM_LIT:10>], user_id,<EOL>create_time.strftime('<STR_LIT>')[:<NUM_LIT:16>])<EOL>version = job_model.convert_to_label_chars('<STR_LIT>' % DSUB_VERSION)<EOL>return {<EOL>'<STR_LIT>': pipeline_name,<EOL>'<STR_LIT>': job_name_value,<EOL>'<STR_LIT>': job_id,<EOL>'<STR_LIT>': user_id,<EOL>'<STR_LIT>': version,<EOL>}<EOL>

Returns a dictionary of metadata fields for the job.


if not labels:<EOL><INDENT>return None<EOL><DEDENT>return [job_model.convert_to_label_chars(label) for label in labels]<EOL>

Converts the label strings to contain label-appropriate characters.

    Args:
      labels: A list of strings to be converted.

    Returns:
      A list of converted strings.


<EOL>m = re.match(r'<STR_LIT>',<EOL>rfc3339_utc_string)<EOL>if not m:<EOL><INDENT>return None<EOL><DEDENT>groups = m.groups()<EOL>if len(groups[<NUM_LIT:6>]) not in (<NUM_LIT:0>, <NUM_LIT:3>, <NUM_LIT:6>, <NUM_LIT:9>):<EOL><INDENT>return None<EOL><DEDENT>g = [int(val) for val in groups[:<NUM_LIT:6>]]<EOL>fraction = groups[<NUM_LIT:6>]<EOL>if not fraction:<EOL><INDENT>micros = <NUM_LIT:0><EOL><DEDENT>elif len(fraction) == <NUM_LIT:3>:<EOL><INDENT>micros = int(fraction) * <NUM_LIT:1000><EOL><DEDENT>elif len(fraction) == <NUM_LIT:6>:<EOL><INDENT>micros = int(fraction)<EOL><DEDENT>elif len(fraction) == <NUM_LIT:9>:<EOL><INDENT>micros = int(round(int(fraction) / <NUM_LIT:1000>))<EOL><DEDENT>else:<EOL><INDENT>assert False, '<STR_LIT>'.len(fraction)<EOL><DEDENT>try:<EOL><INDENT>return datetime(g[<NUM_LIT:0>], g[<NUM_LIT:1>], g[<NUM_LIT:2>], g[<NUM_LIT:3>], g[<NUM_LIT:4>], g[<NUM_LIT:5>], micros, tzinfo=pytz.utc)<EOL><DEDENT>except ValueError as e:<EOL><INDENT>assert False, '<STR_LIT>'.format(<EOL>rfc3339_utc_string, e)<EOL><DEDENT>

Converts a datestamp from RFC3339 UTC to a datetime.

    Args:
      rfc3339_utc_string: a datetime string in RFC3339 UTC "Zulu" format

    Returns:
      A datetime.


job_id = op.get_field('<STR_LIT>')<EOL>task_id = op.get_field('<STR_LIT>')<EOL>if task_id:<EOL><INDENT>return '<STR_LIT>' % (job_id, task_id)<EOL><DEDENT>else:<EOL><INDENT>return job_id<EOL><DEDENT>

Returns the job-id or job-id.task-id for the operation.


def _cancel_batch(batch_fn, cancel_fn, ops):

<EOL>canceled = []<EOL>failed = []<EOL>def handle_cancel_response(request_id, response, exception):<EOL><INDENT>"""<STR_LIT>"""<EOL>del response  <EOL>if exception:<EOL><INDENT>msg = '<STR_LIT>' % (exception.resp.status, exception.resp.reason)<EOL>if exception.resp.status == FAILED_PRECONDITION_CODE:<EOL><INDENT>detail = json.loads(exception.content)<EOL>status = detail.get('<STR_LIT:error>', {}).get('<STR_LIT:status>')<EOL>if status == FAILED_PRECONDITION_STATUS:<EOL><INDENT>msg = '<STR_LIT>'<EOL><DEDENT><DEDENT>failed.append({'<STR_LIT:name>': request_id, '<STR_LIT>': msg})<EOL><DEDENT>else:<EOL><INDENT>canceled.append({'<STR_LIT:name>': request_id})<EOL><DEDENT>return<EOL><DEDENT>batch = batch_fn(callback=handle_cancel_response)<EOL>ops_by_name = {}<EOL>for op in ops:<EOL><INDENT>op_name = op.get_field('<STR_LIT>')<EOL>ops_by_name[op_name] = op<EOL>batch.add(cancel_fn(name=op_name, body={}), request_id=op_name)<EOL><DEDENT>batch.execute()<EOL>canceled_ops = [ops_by_name[op['<STR_LIT:name>']] for op in canceled]<EOL>error_messages = []<EOL>for fail in failed:<EOL><INDENT>op = ops_by_name[fail['<STR_LIT:name>']]<EOL>error_messages.append("<STR_LIT>" %<EOL>(get_operation_full_job_id(op), fail['<STR_LIT>']))<EOL><DEDENT>return canceled_ops, error_messages<EOL>

Cancel a batch of operations.

    Args:
      batch_fn: API-specific batch function.
      cancel_fn: API-specific cancel function.
      ops: A list of operations to cancel.

    Returns:
      A list of operations canceled and a list of error messages.


<EOL>canceled_ops = []<EOL>error_messages = []<EOL>max_batch = <NUM_LIT><EOL>total_ops = len(ops)<EOL>for first_op in range(<NUM_LIT:0>, total_ops, max_batch):<EOL><INDENT>batch_canceled, batch_messages = _cancel_batch(<EOL>batch_fn, cancel_fn, ops[first_op:first_op + max_batch])<EOL>canceled_ops.extend(batch_canceled)<EOL>error_messages.extend(batch_messages)<EOL><DEDENT>return canceled_ops, error_messages<EOL>

Cancel operations.

    Args:
      batch_fn: API-specific batch function.
      cancel_fn: API-specific cancel function.
      ops: A list of operations to cancel.

    Returns:
      A list of operations canceled and a list of error messages.


if isinstance(exception, apiclient.errors.HttpError):<EOL><INDENT>if exception.resp.status in TRANSIENT_HTTP_ERROR_CODES:<EOL><INDENT>_print_error('<STR_LIT>')<EOL>return True<EOL><DEDENT><DEDENT>if isinstance(exception, socket.error):<EOL><INDENT>if exception.errno in TRANSIENT_SOCKET_ERROR_CODES:<EOL><INDENT>_print_error('<STR_LIT>')<EOL>return True<EOL><DEDENT><DEDENT>if isinstance(exception, oauth2client.client.AccessTokenRefreshError):<EOL><INDENT>_print_error('<STR_LIT>')<EOL>return True<EOL><DEDENT>if isinstance(exception, SSLError):<EOL><INDENT>_print_error('<STR_LIT>')<EOL>return True<EOL><DEDENT>if isinstance(exception, ServerNotFoundError):<EOL><INDENT>_print_error('<STR_LIT>')<EOL>return True<EOL><DEDENT>return False<EOL>

Return True if we should retry. False otherwise.

    Args:
      exception: An exception to test for transience.

    Returns:
      True if we should retry. False otherwise.


if isinstance(exception, apiclient.errors.HttpError):<EOL><INDENT>if exception.resp.status in HTTP_AUTH_ERROR_CODES:<EOL><INDENT>_print_error('<STR_LIT>')<EOL>return True<EOL><DEDENT><DEDENT>return False<EOL>

Specific check for auth error codes.

    Return True if we should retry.

    False otherwise.
    Args:
      exception: An exception to test for transience.

    Returns:
      True if we should retry. False otherwise.


if not credentials:<EOL><INDENT>credentials = oauth2client.client.GoogleCredentials.get_application_default(<EOL>)<EOL><DEDENT>return apiclient.discovery.build(<EOL>api_name, api_version, credentials=credentials)<EOL>

Configures genomics API client.

    Args:
      api_name: Name of the Google API (for example: "genomics")
      api_version: Version of the API (for example: "v2alpha1")
      credentials: Credentials to be used for the gcloud API calls.

    Returns:
      A configured Google Genomics API client with appropriate credentials.


try:<EOL><INDENT>return api.execute()<EOL><DEDENT>except Exception as exception:<EOL><INDENT>now = datetime.now().strftime('<STR_LIT>')<EOL>_print_error('<STR_LIT>' % (now, type(exception).__name__,<EOL>str(exception)))<EOL>raise exception<EOL><DEDENT>

Executes operation.

        Args:
          api: The base API object

        Returns:
           A response body object


if not skip_if_output_present:<EOL><INDENT>raise FailsException("<STR_LIT>")<EOL><DEDENT>for task_view in job_model.task_view_generator(job_descriptor):<EOL><INDENT>job_params = task_view.job_params<EOL>task_params = task_view.task_descriptors[<NUM_LIT:0>].task_params<EOL>outputs = job_params["<STR_LIT>"] | task_params["<STR_LIT>"]<EOL>if dsub_util.outputs_are_present(outputs):<EOL><INDENT>print("<STR_LIT>")<EOL>continue<EOL><DEDENT>raise FailsException("<STR_LIT>")<EOL><DEDENT>return {"<STR_LIT>": dsub_util.NO_JOB}<EOL>

Fails if there's anything to submit (so, not skipped).


<EOL>user_image = google_v2_operations.get_action_image(self._op,<EOL>_ACTION_USER_COMMAND)<EOL>need_ok = google_v2_operations.is_success(self._op)<EOL>events = {}<EOL>for event in google_v2_operations.get_events(self._op):<EOL><INDENT>if self._filter(event):<EOL><INDENT>continue<EOL><DEDENT>mapped, match = self._map(event)<EOL>name = mapped['<STR_LIT:name>']<EOL>if name == '<STR_LIT>':<EOL><INDENT>if not need_ok or '<STR_LIT>' in events:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>if name == '<STR_LIT>':<EOL><INDENT>if match.group(<NUM_LIT:1>) != user_image:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>events[name] = mapped<EOL><DEDENT>return sorted(events.values(), key=operator.itemgetter('<STR_LIT>'))<EOL>

Filter the granular v2 events down to events of interest.

        Filter through the large number of granular events returned by the
        pipelines API, and extract only those that are interesting to a user. This
        is implemented by filtering out events which are known to be uninteresting
        (i.e. the default actions run for every job) and by explicitly matching
        specific events which are interesting and mapping those to v1 style naming.

        Events which are not whitelisted or blacklisted will still be output,
        meaning any events which are added in the future won't be masked.
        We don't want to suppress display of events that we don't recognize.
        They may be important.

        Returns:
          A list of maps containing the normalized, filtered events.


description = event.get('<STR_LIT:description>', '<STR_LIT>')<EOL>start_time = google_base.parse_rfc3339_utc_string(<EOL>event.get('<STR_LIT>', '<STR_LIT>'))<EOL>for name, regex in _EVENT_REGEX_MAP.items():<EOL><INDENT>match = regex.match(description)<EOL>if match:<EOL><INDENT>return {'<STR_LIT:name>': name, '<STR_LIT>': start_time}, match<EOL><DEDENT><DEDENT>return {'<STR_LIT:name>': description, '<STR_LIT>': start_time}, None<EOL>

Extract elements from an operation event and map to a named event.


return google_base.prepare_job_metadata(script, job_name, user_id,<EOL>create_time)<EOL>


def _get_logging_env(self, logging_uri, user_project):

if not logging_uri.endswith('<STR_LIT>'):<EOL><INDENT>raise ValueError('<STR_LIT>'.format(logging_uri))<EOL><DEDENT>logging_prefix = logging_uri[:-len('<STR_LIT>')]<EOL>return {<EOL>'<STR_LIT>': '<STR_LIT>'.format(logging_prefix),<EOL>'<STR_LIT>': '<STR_LIT>'.format(logging_prefix),<EOL>'<STR_LIT>': '<STR_LIT>'.format(logging_prefix),<EOL>'<STR_LIT>': user_project,<EOL>}<EOL>

Returns the environment for actions that copy logging files.


def _get_prepare_env(self, script, job_descriptor, inputs, outputs, mounts):

<EOL>docker_paths = sorted([<EOL>var.docker_path if var.recursive else os.path.dirname(var.docker_path)<EOL>for var in inputs | outputs | mounts<EOL>if var.value<EOL>])<EOL>env = {<EOL>_SCRIPT_VARNAME: repr(script.value),<EOL>_META_YAML_VARNAME: repr(job_descriptor.to_yaml()),<EOL>'<STR_LIT>': str(len(docker_paths))<EOL>}<EOL>for idx, path in enumerate(docker_paths):<EOL><INDENT>env['<STR_LIT>'.format(idx)] = os.path.join(providers_util.DATA_MOUNT_POINT,<EOL>path)<EOL><DEDENT>return env<EOL>

Return a dict with variables for the 'prepare' action.


def _get_localization_env(self, inputs, user_project):

<EOL>non_empty_inputs = [var for var in inputs if var.value]<EOL>env = {'<STR_LIT>': str(len(non_empty_inputs))}<EOL>for idx, var in enumerate(non_empty_inputs):<EOL><INDENT>env['<STR_LIT>'.format(idx)] = var.name<EOL>env['<STR_LIT>'.format(idx)] = str(int(var.recursive))<EOL>env['<STR_LIT>'.format(idx)] = var.value<EOL>dst = os.path.join(providers_util.DATA_MOUNT_POINT, var.docker_path)<EOL>path, filename = os.path.split(dst)<EOL>if '<STR_LIT:*>' in filename:<EOL><INDENT>dst = '<STR_LIT>'.format(path)<EOL><DEDENT>env['<STR_LIT>'.format(idx)] = dst<EOL><DEDENT>env['<STR_LIT>'] = user_project<EOL>return env<EOL>

Return a dict with variables for the 'localization' action.


def _get_delocalization_env(self, outputs, user_project):

<EOL>non_empty_outputs = [var for var in outputs if var.value]<EOL>env = {'<STR_LIT>': str(len(non_empty_outputs))}<EOL>for idx, var in enumerate(non_empty_outputs):<EOL><INDENT>env['<STR_LIT>'.format(idx)] = var.name<EOL>env['<STR_LIT>'.format(idx)] = str(int(var.recursive))<EOL>env['<STR_LIT>'.format(idx)] = os.path.join(<EOL>providers_util.DATA_MOUNT_POINT, var.docker_path)<EOL>if '<STR_LIT:*>' in var.uri.basename:<EOL><INDENT>dst = var.uri.path<EOL><DEDENT>else:<EOL><INDENT>dst = var.uri<EOL><DEDENT>env['<STR_LIT>'.format(idx)] = dst<EOL><DEDENT>env['<STR_LIT>'] = user_project<EOL>return env<EOL>

Return a dict with variables for the 'delocalization' action.


def _build_user_environment(self, envs, inputs, outputs, mounts):

envs = {env.name: env.value for env in envs}<EOL>envs.update(providers_util.get_file_environment_variables(inputs))<EOL>envs.update(providers_util.get_file_environment_variables(outputs))<EOL>envs.update(providers_util.get_file_environment_variables(mounts))<EOL>return envs<EOL>

Returns a dictionary of for the user container environment.


def _get_mount_actions(self, mounts, mnt_datadisk):

actions_to_add = []<EOL>for mount in mounts:<EOL><INDENT>bucket = mount.value[len('<STR_LIT>'):]<EOL>mount_path = mount.docker_path<EOL>actions_to_add.extend([<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>'.format(bucket),<EOL>flags=['<STR_LIT>', '<STR_LIT>'],<EOL>image_uri=_GCSFUSE_IMAGE,<EOL>mounts=[mnt_datadisk],<EOL>commands=[<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', bucket,<EOL>os.path.join(providers_util.DATA_MOUNT_POINT, mount_path)<EOL>]),<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>'.format(bucket),<EOL>flags=['<STR_LIT>'],<EOL>image_uri=_GCSFUSE_IMAGE,<EOL>mounts=[mnt_datadisk],<EOL>commands=[<EOL>'<STR_LIT>',<EOL>os.path.join(providers_util.DATA_MOUNT_POINT, mount_path)<EOL>])<EOL>])<EOL><DEDENT>return actions_to_add<EOL>

Returns a list of two actions per gcs bucket to mount.