job_metadata = task_view.job_metadata<EOL>job_params = task_view.job_params<EOL>job_resources = task_view.job_resources<EOL>task_metadata = task_view.task_descriptors[<NUM_LIT:0>].task_metadata<EOL>task_params = task_view.task_descriptors[<NUM_LIT:0>].task_params<EOL>task_resources = task_view.task_descriptors[<NUM_LIT:0>].task_resources<EOL>mnt_datadisk = google_v2_pipelines.build_mount(<EOL>disk=_DATA_DISK_NAME,<EOL>path=providers_util.DATA_MOUNT_POINT,<EOL>read_only=False)<EOL>scopes = job_resources.scopes or google_base.DEFAULT_SCOPES<EOL>labels = {<EOL>label.name: label.value if label.value else '<STR_LIT>' for label in<EOL>google_base.build_pipeline_labels(job_metadata, task_metadata)<EOL>| job_params['<STR_LIT>'] | task_params['<STR_LIT>']<EOL>}<EOL>script = task_view.job_metadata['<STR_LIT>']<EOL>user_project = task_view.job_metadata['<STR_LIT>'] or '<STR_LIT>'<EOL>envs = job_params['<STR_LIT>'] | task_params['<STR_LIT>']<EOL>inputs = job_params['<STR_LIT>'] | task_params['<STR_LIT>']<EOL>outputs = job_params['<STR_LIT>'] | task_params['<STR_LIT>']<EOL>mounts = job_params['<STR_LIT>']<EOL>gcs_mounts = param_util.get_gcs_mounts(mounts)<EOL>persistent_disk_mount_params = param_util.get_persistent_disk_mounts(mounts)<EOL>persistent_disks = [<EOL>google_v2_pipelines.build_disk(<EOL>name=disk.name.replace('<STR_LIT:_>', '<STR_LIT:->'),  <EOL>size_gb=disk.disk_size or job_model.DEFAULT_MOUNTED_DISK_SIZE,<EOL>source_image=disk.value,<EOL>disk_type=disk.disk_type or job_model.DEFAULT_DISK_TYPE)<EOL>for disk in persistent_disk_mount_params<EOL>]<EOL>persistent_disk_mounts = [<EOL>google_v2_pipelines.build_mount(<EOL>disk=persistent_disk.get('<STR_LIT:name>'),<EOL>path=os.path.join(providers_util.DATA_MOUNT_POINT,<EOL>persistent_disk_mount_param.docker_path),<EOL>read_only=True)<EOL>for persistent_disk, persistent_disk_mount_param in zip(<EOL>persistent_disks, persistent_disk_mount_params)<EOL>]<EOL>optional_actions = <NUM_LIT:0><EOL>if job_resources.ssh:<EOL><INDENT>optional_actions += <NUM_LIT:1><EOL><DEDENT>mount_actions = self._get_mount_actions(gcs_mounts, mnt_datadisk)<EOL>optional_actions += len(mount_actions)<EOL>user_action = <NUM_LIT:4> + optional_actions<EOL>final_logging_action = <NUM_LIT:6> + optional_actions<EOL>logging_cmd = _LOGGING_CMD.format(<EOL>log_cp_fn=_GSUTIL_CP_FN,<EOL>log_cp_cmd=_LOG_CP_CMD.format(<EOL>user_action=user_action, logging_action='<STR_LIT>'))<EOL>continuous_logging_cmd = _CONTINUOUS_LOGGING_CMD.format(<EOL>log_msg_fn=_LOG_MSG_FN,<EOL>log_cp_fn=_GSUTIL_CP_FN,<EOL>log_cp_cmd=_LOG_CP_CMD.format(<EOL>user_action=user_action,<EOL>logging_action='<STR_LIT>'),<EOL>final_logging_action=final_logging_action,<EOL>log_interval=job_resources.log_interval or '<STR_LIT>')<EOL>logging_env = self._get_logging_env(task_resources.logging_path.uri,<EOL>user_project)<EOL>script_path = os.path.join(providers_util.SCRIPT_DIR, script.name)<EOL>prepare_command = _PREPARE_CMD.format(<EOL>log_msg_fn=_LOG_MSG_FN,<EOL>mk_runtime_dirs=_MK_RUNTIME_DIRS_CMD,<EOL>script_var=_SCRIPT_VARNAME,<EOL>python_decode_script=_PYTHON_DECODE_SCRIPT,<EOL>script_path=script_path,<EOL>mk_io_dirs=_MK_IO_DIRS)<EOL>prepare_env = self._get_prepare_env(script, task_view, inputs, outputs,<EOL>mounts)<EOL>localization_env = self._get_localization_env(inputs, user_project)<EOL>user_environment = self._build_user_environment(envs, inputs, outputs,<EOL>mounts)<EOL>delocalization_env = self._get_delocalization_env(outputs, user_project)<EOL>actions = []<EOL>actions.append(<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>',<EOL>flags='<STR_LIT>',<EOL>image_uri=_CLOUD_SDK_IMAGE,<EOL>environment=logging_env,<EOL>entrypoint='<STR_LIT>',<EOL>commands=['<STR_LIT:-c>', continuous_logging_cmd]))<EOL>if job_resources.ssh:<EOL><INDENT>actions.append(<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>',<EOL>image_uri=_SSH_IMAGE,<EOL>mounts=[mnt_datadisk],<EOL>entrypoint='<STR_LIT>',<EOL>port_mappings={_DEFAULT_SSH_PORT: _DEFAULT_SSH_PORT},<EOL>flags='<STR_LIT>'))<EOL><DEDENT>actions.append(<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>',<EOL>image_uri=_PYTHON_IMAGE,<EOL>mounts=[mnt_datadisk],<EOL>environment=prepare_env,<EOL>entrypoint='<STR_LIT>',<EOL>commands=['<STR_LIT:-c>', prepare_command]),)<EOL>actions.extend(mount_actions)<EOL>actions.extend([<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>',<EOL>image_uri=_CLOUD_SDK_IMAGE,<EOL>mounts=[mnt_datadisk],<EOL>environment=localization_env,<EOL>entrypoint='<STR_LIT>',<EOL>commands=[<EOL>'<STR_LIT:-c>',<EOL>_LOCALIZATION_CMD.format(<EOL>log_msg_fn=_LOG_MSG_FN,<EOL>recursive_cp_fn=_GSUTIL_RSYNC_FN,<EOL>cp_fn=_GSUTIL_CP_FN,<EOL>cp_loop=_LOCALIZATION_LOOP)<EOL>]),<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>',<EOL>image_uri=job_resources.image,<EOL>mounts=[mnt_datadisk] + persistent_disk_mounts,<EOL>environment=user_environment,<EOL>entrypoint='<STR_LIT>',<EOL>commands=[<EOL>'<STR_LIT>', '<STR_LIT:-c>',<EOL>_USER_CMD.format(<EOL>tmp_dir=providers_util.TMP_DIR,<EOL>working_dir=providers_util.WORKING_DIR,<EOL>user_script=script_path)<EOL>]),<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>',<EOL>image_uri=_CLOUD_SDK_IMAGE,<EOL>mounts=[mnt_datadisk],<EOL>environment=delocalization_env,<EOL>entrypoint='<STR_LIT>',<EOL>commands=[<EOL>'<STR_LIT:-c>',<EOL>_LOCALIZATION_CMD.format(<EOL>log_msg_fn=_LOG_MSG_FN,<EOL>recursive_cp_fn=_GSUTIL_RSYNC_FN,<EOL>cp_fn=_GSUTIL_CP_FN,<EOL>cp_loop=_DELOCALIZATION_LOOP)<EOL>]),<EOL>google_v2_pipelines.build_action(<EOL>name='<STR_LIT>',<EOL>flags='<STR_LIT>',<EOL>image_uri=_CLOUD_SDK_IMAGE,<EOL>environment=logging_env,<EOL>entrypoint='<STR_LIT>',<EOL>commands=['<STR_LIT:-c>', logging_cmd]),<EOL>])<EOL>assert len(actions) - <NUM_LIT:2> == user_action<EOL>assert len(actions) == final_logging_action<EOL>disks = [<EOL>google_v2_pipelines.build_disk(<EOL>_DATA_DISK_NAME,<EOL>job_resources.disk_size,<EOL>source_image=None,<EOL>disk_type=job_resources.disk_type or job_model.DEFAULT_DISK_TYPE)<EOL>]<EOL>disks.extend(persistent_disks)<EOL>network = google_v2_pipelines.build_network(<EOL>job_resources.network, job_resources.subnetwork,<EOL>job_resources.use_private_address)<EOL>if job_resources.machine_type:<EOL><INDENT>machine_type = job_resources.machine_type<EOL><DEDENT>elif job_resources.min_cores or job_resources.min_ram:<EOL><INDENT>machine_type = GoogleV2CustomMachine.build_machine_type(<EOL>job_resources.min_cores, job_resources.min_ram)<EOL><DEDENT>else:<EOL><INDENT>machine_type = job_model.DEFAULT_MACHINE_TYPE<EOL><DEDENT>accelerators = None<EOL>if job_resources.accelerator_type:<EOL><INDENT>accelerators = [<EOL>google_v2_pipelines.build_accelerator(job_resources.accelerator_type,<EOL>job_resources.accelerator_count)<EOL>]<EOL><DEDENT>service_account = google_v2_pipelines.build_service_account(<EOL>job_resources.service_account or '<STR_LIT:default>', scopes)<EOL>resources = google_v2_pipelines.build_resources(<EOL>self._project,<EOL>job_resources.regions,<EOL>google_base.get_zones(job_resources.zones),<EOL>google_v2_pipelines.build_machine(<EOL>network=network,<EOL>machine_type=machine_type,<EOL>preemptible=job_resources.preemptible,<EOL>service_account=service_account,<EOL>boot_disk_size_gb=job_resources.boot_disk_size,<EOL>disks=disks,<EOL>accelerators=accelerators,<EOL>nvidia_driver_version=job_resources.nvidia_driver_version,<EOL>labels=labels,<EOL>cpu_platform=job_resources.cpu_platform),<EOL>)<EOL>pipeline = google_v2_pipelines.build_pipeline(actions, resources, None,<EOL>job_resources.timeout)<EOL>return {'<STR_LIT>': pipeline, '<STR_LIT>': labels}<EOL>

Returns a Pipeline objects for the task.


def submit_job(self, job_descriptor, skip_if_output_present):

<EOL>param_util.validate_submit_args_or_fail(<EOL>job_descriptor,<EOL>provider_name=_PROVIDER_NAME,<EOL>input_providers=_SUPPORTED_INPUT_PROVIDERS,<EOL>output_providers=_SUPPORTED_OUTPUT_PROVIDERS,<EOL>logging_providers=_SUPPORTED_LOGGING_PROVIDERS)<EOL>launched_tasks = []<EOL>requests = []<EOL>for task_view in job_model.task_view_generator(job_descriptor):<EOL><INDENT>job_params = task_view.job_params<EOL>task_params = task_view.task_descriptors[<NUM_LIT:0>].task_params<EOL>outputs = job_params['<STR_LIT>'] | task_params['<STR_LIT>']<EOL>if skip_if_output_present:<EOL><INDENT>if dsub_util.outputs_are_present(outputs):<EOL><INDENT>print('<STR_LIT>')<EOL>continue<EOL><DEDENT><DEDENT>request = self._build_pipeline_request(task_view)<EOL>if self._dry_run:<EOL><INDENT>requests.append(request)<EOL><DEDENT>else:<EOL><INDENT>task_id = self._submit_pipeline(request)<EOL>launched_tasks.append(task_id)<EOL><DEDENT><DEDENT>if self._dry_run:<EOL><INDENT>print(<EOL>json.dumps(<EOL>requests, indent=<NUM_LIT:2>, sort_keys=True, separators=('<STR_LIT:U+002C>', '<STR_LIT>')))<EOL><DEDENT>if not requests and not launched_tasks:<EOL><INDENT>return {'<STR_LIT>': dsub_util.NO_JOB}<EOL><DEDENT>return {<EOL>'<STR_LIT>': job_descriptor.job_metadata['<STR_LIT>'],<EOL>'<STR_LIT>': job_descriptor.job_metadata['<STR_LIT>'],<EOL>'<STR_LIT>': [task_id for task_id in launched_tasks if task_id],<EOL>}<EOL>

Submit the job (or tasks) to be executed.

        Args:
          job_descriptor: all parameters needed to launch all job tasks
          skip_if_output_present: (boolean) if true, skip tasks whose output
            is present (see --skip flag for more explanation).

        Returns:
          A dictionary containing the 'user-id', 'job-id', and 'task-id' list.
          For jobs that are not task array jobs, the task-id list should be empty.

        Raises:
          ValueError: if job resources or task data contain illegal values.


def _operations_list(self, ops_filter, max_tasks, page_size, page_token):

<EOL>max_page_size = <NUM_LIT><EOL>page_size = min(sz for sz in [page_size, max_page_size, max_tasks] if sz)<EOL>api = self._service.projects().operations().list(<EOL>name='<STR_LIT>'.format(self._project),<EOL>filter=ops_filter,<EOL>pageToken=page_token,<EOL>pageSize=page_size)<EOL>response = google_base.Api.execute(api)<EOL>return [<EOL>GoogleOperation(op)<EOL>for op in response.get('<STR_LIT>', [])<EOL>if google_v2_operations.is_dsub_operation(op)<EOL>], response.get('<STR_LIT>')<EOL>

Gets the list of operations for the specified filter.

        Args:
          ops_filter: string filter of operations to return
          max_tasks: the maximum number of job tasks to return or 0 for no limit.
          page_size: the number of operations to requested on each list operation to
            the pipelines API (if 0 or None, the API default is used)
          page_token: page token returned by a previous _operations_list call.

        Returns:
          Operations matching the filter criteria.


<EOL>ops_filter = self._build_query_filter(<EOL>statuses, user_ids, job_ids, job_names, task_ids, task_attempts, labels,<EOL>create_time_min, create_time_max)<EOL>page_token = None<EOL>tasks_yielded = <NUM_LIT:0><EOL>while True:<EOL><INDENT>max_to_fetch = None<EOL>if max_tasks:<EOL><INDENT>max_to_fetch = max_tasks - tasks_yielded<EOL><DEDENT>ops, page_token = self._operations_list(ops_filter, max_to_fetch,<EOL>page_size, page_token)<EOL>for op in ops:<EOL><INDENT>yield op<EOL>tasks_yielded += <NUM_LIT:1><EOL><DEDENT>assert (max_tasks >= tasks_yielded or not max_tasks)<EOL>if not page_token or <NUM_LIT:0> < max_tasks <= tasks_yielded:<EOL><INDENT>break<EOL><DEDENT><DEDENT>

Yields operations based on the input criteria.

        If any of the filters are empty or {'*'}, then no filtering is performed on
        that field. Filtering by both a job id list and job name list is
        unsupported.

        Args:
          statuses: {'*'}, or a list of job status strings to return. Valid
            status strings are 'RUNNING', 'SUCCESS', 'FAILURE', or 'CANCELED'.
          user_ids: a list of ids for the user(s) who launched the job.
          job_ids: a list of job ids to return.
          job_names: a list of job names to return.
          task_ids: a list of specific tasks within the specified job(s) to return.
          task_attempts: a list of specific attempts within the specified tasks(s)
            to return.
          labels: a list of LabelParam with user-added labels. All labels must
                  match the task being fetched.
          create_time_min: a timezone-aware datetime value for the earliest create
                           time of a task, inclusive.
          create_time_max: a timezone-aware datetime value for the most recent
                           create time of a task, inclusive.
          max_tasks: the maximum number of job tasks to return or 0 for no limit.
          page_size: the page size to use for each query to the pipelins API.

        Raises:
          ValueError: if both a job id list and a job name list are provided

        Yeilds:
          Genomics API Operations objects.


Kills the operations associated with the specified job or job.task.

        Args:
          user_ids: List of user ids who "own" the job(s) to cancel.
          job_ids: List of job_ids to cancel.
          task_ids: List of task-ids to cancel.
          labels: List of LabelParam, each must match the job(s) to be canceled.
          create_time_min: a timezone-aware datetime value for the earliest create
                           time of a task, inclusive.
          create_time_max: a timezone-aware datetime value for the most recent
                           create time of a task, inclusive.

        Returns:
          A list of tasks canceled and a list of error messages.


if not google_v2_operations.is_done(self._op):<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>if google_v2_operations.is_success(self._op):<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>if google_v2_operations.is_canceled(self._op):<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>if google_v2_operations.is_failed(self._op):<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>raise ValueError('<STR_LIT>'.format(<EOL>self._op['<STR_LIT:name>']))<EOL>

Returns the status of this operation.

        Raises:
          ValueError: if the operation status cannot be determined.

        Returns:
          A printable status string (RUNNING, SUCCESS, CANCELED or FAILURE).


msg = None<EOL>action = None<EOL>if not google_v2_operations.is_done(self._op):<EOL><INDENT>last_event = google_v2_operations.get_last_event(self._op)<EOL>if last_event:<EOL><INDENT>msg = last_event['<STR_LIT:description>']<EOL>action_id = last_event.get('<STR_LIT>', {}).get('<STR_LIT>')<EOL>if action_id:<EOL><INDENT>action = google_v2_operations.get_action_by_id(self._op, action_id)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>msg = '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>failed_events = google_v2_operations.get_failed_events(self._op)<EOL>if failed_events:<EOL><INDENT>failed_event = failed_events[-<NUM_LIT:1>]<EOL>msg = failed_event.get('<STR_LIT>', {}).get('<STR_LIT>')<EOL>action_id = failed_event.get('<STR_LIT>', {}).get('<STR_LIT>')<EOL>if action_id:<EOL><INDENT>action = google_v2_operations.get_action_by_id(self._op, action_id)<EOL><DEDENT><DEDENT>if not msg:<EOL><INDENT>error = google_v2_operations.get_error(self._op)<EOL>if error:<EOL><INDENT>msg = error['<STR_LIT:message>']<EOL><DEDENT>else:<EOL><INDENT>msg = '<STR_LIT>'<EOL><DEDENT><DEDENT><DEDENT>return msg, action<EOL>

Returns the most relevant status string and failed action.

        This string is meant for display only.

        Returns:
          A printable status string and name of failed action (if any).


error = google_v2_operations.get_error(self._op)<EOL>if error:<EOL><INDENT>job_id = self.get_field('<STR_LIT>')<EOL>task_id = self.get_field('<STR_LIT>')<EOL>task_str = job_id if task_id is None else '<STR_LIT>'.format(<EOL>job_id, task_id)<EOL>return '<STR_LIT>'.format(task_str, error['<STR_LIT:code>'],<EOL>error['<STR_LIT:message>'])<EOL><DEDENT>return '<STR_LIT>'<EOL>

Returns an error message if the operation failed for any reason.

        Failure as defined here means ended for any reason other than 'success'.
        This means that a successful cancelation will also return an error message.

        Returns:
          string, string will be empty if job did not error.


def get_field(self, field, default=None):

value = None<EOL>if field == '<STR_LIT>':<EOL><INDENT>value = self._op['<STR_LIT:name>']<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>if self._job_descriptor:<EOL><INDENT>value = self._job_descriptor.job_metadata.get(field)<EOL><DEDENT><DEDENT>elif field in [<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>'<EOL>]:<EOL><INDENT>value = google_v2_operations.get_label(self._op, field)<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = self._operation_status()<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>if self._job_descriptor:<EOL><INDENT>task_resources = self._job_descriptor.task_descriptors[<NUM_LIT:0>].task_resources<EOL>value = task_resources.logging_path<EOL><DEDENT><DEDENT>elif field in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>if self._job_descriptor:<EOL><INDENT>items = providers_util.get_job_and_task_param(<EOL>self._job_descriptor.job_params,<EOL>self._job_descriptor.task_descriptors[<NUM_LIT:0>].task_params, field)<EOL>value = {item.name: item.value for item in items}<EOL><DEDENT><DEDENT>elif field in [<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'<EOL>]:<EOL><INDENT>if self._job_descriptor:<EOL><INDENT>value = {}<EOL>items = providers_util.get_job_and_task_param(<EOL>self._job_descriptor.job_params,<EOL>self._job_descriptor.task_descriptors[<NUM_LIT:0>].task_params, field)<EOL>value.update({item.name: item.value for item in items})<EOL><DEDENT><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>if self._job_descriptor:<EOL><INDENT>items = providers_util.get_job_and_task_param(<EOL>self._job_descriptor.job_params,<EOL>self._job_descriptor.task_descriptors[<NUM_LIT:0>].task_params, field)<EOL>value = {item.name: item.value for item in items}<EOL><DEDENT><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>ds = google_v2_operations.get_create_time(self._op)<EOL>value = google_base.parse_rfc3339_utc_string(ds)<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>ds = google_v2_operations.get_start_time(self._op)<EOL>if ds:<EOL><INDENT>value = google_base.parse_rfc3339_utc_string(ds)<EOL><DEDENT><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>ds = google_v2_operations.get_end_time(self._op)<EOL>if ds:<EOL><INDENT>value = google_base.parse_rfc3339_utc_string(ds)<EOL><DEDENT><DEDENT>elif field == '<STR_LIT:status>':<EOL><INDENT>value = self._operation_status()<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>msg, action = self._operation_status_message()<EOL>if msg.startswith('<STR_LIT>'):<EOL><INDENT>msg = msg.split('<STR_LIT>', <NUM_LIT:2>)[-<NUM_LIT:1>]<EOL><DEDENT>value = msg<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>msg, action = self._operation_status_message()<EOL>if action:<EOL><INDENT>value = '<STR_LIT>'.format(action.get('<STR_LIT:name>'), msg)<EOL><DEDENT>else:<EOL><INDENT>value = msg<EOL><DEDENT><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>last_update = google_v2_operations.get_last_update(self._op)<EOL>if last_update:<EOL><INDENT>value = google_base.parse_rfc3339_utc_string(last_update)<EOL><DEDENT><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>return _PROVIDER_NAME<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = {}<EOL>assigned_events = google_v2_operations.get_worker_assigned_events(<EOL>self._op)<EOL>if assigned_events:<EOL><INDENT>details = assigned_events[<NUM_LIT:0>].get('<STR_LIT>', {})<EOL>value['<STR_LIT>'] = details.get('<STR_LIT>')<EOL>value['<STR_LIT>'] = details.get('<STR_LIT>')<EOL><DEDENT>resources = google_v2_operations.get_resources(self._op)<EOL>if '<STR_LIT>' in resources:<EOL><INDENT>value['<STR_LIT>'] = resources['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in resources:<EOL><INDENT>value['<STR_LIT>'] = resources['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in resources:<EOL><INDENT>vm = resources['<STR_LIT>']<EOL>value['<STR_LIT>'] = vm.get('<STR_LIT>')<EOL>value['<STR_LIT>'] = vm.get('<STR_LIT>')<EOL>value['<STR_LIT>'] = vm.get('<STR_LIT>')<EOL>value['<STR_LIT>'] = vm.get('<STR_LIT>', {}).get('<STR_LIT:name>')<EOL>value['<STR_LIT>'] = vm.get('<STR_LIT>', {}).get('<STR_LIT>')<EOL>value['<STR_LIT>'] = vm.get('<STR_LIT>',<EOL>{}).get('<STR_LIT>')<EOL>value['<STR_LIT>'] = vm.get('<STR_LIT>')<EOL>value['<STR_LIT>'] = vm.get('<STR_LIT>')<EOL>value['<STR_LIT>'] = vm.get('<STR_LIT>', {}).get('<STR_LIT:email>')<EOL>if '<STR_LIT>' in vm:<EOL><INDENT>datadisk = next(<EOL>(d for d in vm['<STR_LIT>'] if d['<STR_LIT:name>'] == _DATA_DISK_NAME))<EOL>if datadisk:<EOL><INDENT>value['<STR_LIT>'] = datadisk.get('<STR_LIT>')<EOL>value['<STR_LIT>'] = datadisk.get('<STR_LIT:type>')<EOL><DEDENT><DEDENT><DEDENT><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = GoogleV2EventMap(self._op).get_filtered_normalized_events()<EOL><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>if self._job_descriptor:<EOL><INDENT>value = self._job_descriptor.job_metadata.get(field)<EOL><DEDENT><DEDENT>elif field == '<STR_LIT>':<EOL><INDENT>value = self._try_op_to_script_body()<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' % field)<EOL><DEDENT>return value if value else default<EOL>

Returns a value from the operation for a specific set of field names.

        Args:
          field: a dsub-specific job metadata key
          default: default value to return if field does not exist or is empty.

        Returns:
          A text string for the field or a list for 'inputs'.

        Raises:
          ValueError: if the field label is not supported by the operation


if cores == <NUM_LIT:1> or cores % <NUM_LIT:2> == <NUM_LIT:0>:<EOL><INDENT>return cores<EOL><DEDENT>else:<EOL><INDENT>return cores + <NUM_LIT:1><EOL><DEDENT>


return int(GoogleV2CustomMachine._MEMORY_MULTIPLE * math.ceil(<EOL>ram_in_mb / GoogleV2CustomMachine._MEMORY_MULTIPLE))<EOL>

Rounds ram up to the nearest multiple of _MEMORY_MULTIPLE.


@classmethod<EOL><INDENT>def build_machine_type(cls, min_cores, min_ram):<DEDENT>

min_cores = min_cores or job_model.DEFAULT_MIN_CORES<EOL>min_ram = min_ram or job_model.DEFAULT_MIN_RAM<EOL>min_ram *= GoogleV2CustomMachine._MB_PER_GB<EOL>cores = cls._validate_cores(min_cores)<EOL>ram = cls._validate_ram(min_ram)<EOL>memory_to_cpu_ratio = ram / cores<EOL>if memory_to_cpu_ratio < GoogleV2CustomMachine._MIN_MEMORY_PER_CPU:<EOL><INDENT>adjusted_ram = GoogleV2CustomMachine._MIN_MEMORY_PER_CPU * cores<EOL>ram = cls._validate_ram(adjusted_ram)<EOL><DEDENT>elif memory_to_cpu_ratio > GoogleV2CustomMachine._MAX_MEMORY_PER_CPU:<EOL><INDENT>adjusted_cores = math.ceil(<EOL>ram / GoogleV2CustomMachine._MAX_MEMORY_PER_CPU)<EOL>cores = cls._validate_cores(adjusted_cores)<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT>return '<STR_LIT>'.format(int(cores), int(ram))<EOL>


def _format_task_name(job_id, task_id, task_attempt):

docker_name = '<STR_LIT>' % (job_id, '<STR_LIT>' if task_id is None else task_id)<EOL>if task_attempt is not None:<EOL><INDENT>docker_name += '<STR_LIT:.>' + str(task_attempt)<EOL><DEDENT>return '<STR_LIT>'.format(_convert_suffix_to_docker_chars(docker_name))<EOL>

Create a task name from a job-id, task-id, and task-attempt.

    Task names are used internally by dsub as well as by the docker task runner.
    The name is formatted as "<job-id>.<task-id>[.task-attempt]". Task names
    follow formatting conventions allowing them to be safely used as a docker
    name.

    Args:
      job_id: (str) the job ID.
      task_id: (str) the task ID.
      task_attempt: (int) the task attempt.

    Returns:
      a task name string.


<EOL>accepted_characters = string.ascii_letters + string.digits + '<STR_LIT>'<EOL>def label_char_transform(char):<EOL><INDENT>if char in accepted_characters:<EOL><INDENT>return char<EOL><DEDENT>return '<STR_LIT:->'<EOL><DEDENT>return '<STR_LIT>'.join(label_char_transform(c) for c in suffix)<EOL>

Rewrite string so that all characters are valid in a docker name suffix.


return (task.get_field('<STR_LIT>'), int(task.get_field('<STR_LIT>', <NUM_LIT:0>)),<EOL>int(task.get_field('<STR_LIT>', <NUM_LIT:0>)))<EOL>

Return a tuple for sorting 'most recent first'.


<EOL>tasks.sort(key=_task_sort_function, reverse=True)<EOL>


self._operations = []<EOL>self._resources = resources<EOL>

Run jobs on your local machine.

        Args:
          resources: module providing access to files packaged with dsub
                     (See dsub/libs/resources.py)


def _datetime_in_range(self, dt, dt_min=None, dt_max=None):

<EOL>dt = dt.replace(microsecond=<NUM_LIT:0>)<EOL>if dt_min:<EOL><INDENT>dt_min = dt_min.replace(microsecond=<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>dt_min = dsub_util.replace_timezone(datetime.datetime.min, pytz.utc)<EOL><DEDENT>if dt_max:<EOL><INDENT>dt_max = dt_max.replace(microsecond=<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>dt_max = dsub_util.replace_timezone(datetime.datetime.max, pytz.utc)<EOL><DEDENT>return dt_min <= dt <= dt_max<EOL>

Determine if the provided time is within the range, inclusive.


def _get_task_from_task_dir(self, job_id, user_id, task_id, task_attempt):

<EOL>task_dir = self._task_directory(job_id, task_id, task_attempt)<EOL>job_descriptor = self._read_task_metadata(task_dir)<EOL>if not job_descriptor:<EOL><INDENT>return None<EOL><DEDENT>if not job_descriptor.job_metadata.get('<STR_LIT>'):<EOL><INDENT>job_descriptor.job_metadata['<STR_LIT>'] = user_id<EOL><DEDENT>pid = -<NUM_LIT:1><EOL>try:<EOL><INDENT>with open(os.path.join(task_dir, '<STR_LIT>'), '<STR_LIT:r>') as f:<EOL><INDENT>pid = int(f.readline().strip())<EOL><DEDENT><DEDENT>except (IOError, OSError):<EOL><INDENT>pass<EOL><DEDENT>script = None<EOL>script_name = job_descriptor.job_metadata.get('<STR_LIT>')<EOL>if script_name:<EOL><INDENT>script = self._read_script(task_dir, script_name)<EOL><DEDENT>end_time = self._get_end_time_from_task_dir(task_dir)<EOL>last_update = self._get_last_update_time_from_task_dir(task_dir)<EOL>events = self._get_events_from_task_dir(task_dir)<EOL>status = self._get_status_from_task_dir(task_dir)<EOL>log_detail = self._get_log_detail_from_task_dir(task_dir)<EOL>if not status:<EOL><INDENT>status = '<STR_LIT>'<EOL>log_detail = ['<STR_LIT>']<EOL><DEDENT>return LocalTask(<EOL>task_status=status,<EOL>events=events,<EOL>log_detail=log_detail,<EOL>job_descriptor=job_descriptor,<EOL>end_time=end_time,<EOL>last_update=last_update,<EOL>pid=pid,<EOL>script=script)<EOL>

Return a Task object with this task's info.


def _delocalize_logging_command(self, logging_path, user_project):

<EOL>logging_prefix = os.path.splitext(logging_path.uri)[<NUM_LIT:0>]<EOL>if logging_path.file_provider == job_model.P_LOCAL:<EOL><INDENT>mkdir_cmd = '<STR_LIT>' % os.path.dirname(logging_prefix)<EOL>cp_cmd = '<STR_LIT>'<EOL><DEDENT>elif logging_path.file_provider == job_model.P_GCS:<EOL><INDENT>mkdir_cmd = '<STR_LIT>'<EOL>if user_project:<EOL><INDENT>cp_cmd = '<STR_LIT>'.format(user_project)<EOL><DEDENT>else:<EOL><INDENT>cp_cmd = '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>assert False<EOL><DEDENT>copy_logs_cmd = textwrap.dedent("""<STR_LIT>"""kdir_cmd}<EOL>opy_logs_cmd}<EOL>.format(<EOL><INDENT>mkdir_cmd=mkdir_cmd, copy_logs_cmd=copy_logs_cmd)<EOL>return body<EOL>

Returns a command to delocalize logs.

        Args:
          logging_path: location of log files.
          user_project: name of the project to be billed for the request.

        Returns:
          eg. 'gs://bucket/path/myfile' or 'gs://bucket/script-foobar-12'


def _make_job_id(self, job_name_value, user_id, create_time):

<EOL>return '<STR_LIT>' % (job_name_value[:<NUM_LIT:10>], user_id,<EOL>create_time.strftime('<STR_LIT>'))<EOL>


def _task_directory(self, job_id, task_id, task_attempt):

dir_name = '<STR_LIT>' if task_id is None else str(task_id)<EOL>if task_attempt:<EOL><INDENT>dir_name = '<STR_LIT>' % (dir_name, task_attempt)<EOL><DEDENT>return self._provider_root() + '<STR_LIT:/>' + job_id + '<STR_LIT:/>' + dir_name<EOL>

The local dir for staging files for that particular task.


if '<STR_LIT:.>' in task_dir:<EOL><INDENT>return task_dir.split('<STR_LIT:.>')<EOL><DEDENT>else:<EOL><INDENT>return task_dir, None<EOL><DEDENT>

Return task_id and task_attempt from dir_name.


def _make_environment(self, inputs, outputs, mounts):

Return a dictionary of environment variables for the container.


def _localize_inputs_recursive_command(self, task_dir, inputs):

data_dir = os.path.join(task_dir, _DATA_SUBDIR)<EOL>provider_commands = [<EOL>providers_util.build_recursive_localize_command(data_dir, inputs,<EOL>file_provider)<EOL>for file_provider in _SUPPORTED_INPUT_PROVIDERS<EOL>]<EOL>return '<STR_LIT:\n>'.join(provider_commands)<EOL>

Returns a command that will stage recursive inputs.


path, filename = os.path.split(local_file_path)<EOL>if '<STR_LIT:*>' in filename:<EOL><INDENT>return path + '<STR_LIT:/>'<EOL><DEDENT>else:<EOL><INDENT>return local_file_path<EOL><DEDENT>

Returns a directory or file path to be the target for "gsutil cp".

        If the filename contains a wildcard, then the target path must
        be a directory in order to ensure consistency whether the source pattern
        contains one or multiple files.

        Args:
          local_file_path: A full path terminating in a file or a file wildcard.

        Returns:
          The path to use as the "gsutil cp" target.


def _localize_inputs_command(self, task_dir, inputs, user_project):

commands = []<EOL>for i in inputs:<EOL><INDENT>if i.recursive or not i.value:<EOL><INDENT>continue<EOL><DEDENT>source_file_path = i.uri<EOL>local_file_path = task_dir + '<STR_LIT:/>' + _DATA_SUBDIR + '<STR_LIT:/>' + i.docker_path<EOL>dest_file_path = self._get_input_target_path(local_file_path)<EOL>commands.append('<STR_LIT>' % os.path.dirname(local_file_path))<EOL>if i.file_provider in [job_model.P_LOCAL, job_model.P_GCS]:<EOL><INDENT>if user_project:<EOL><INDENT>command = '<STR_LIT>' % (<EOL>user_project, source_file_path, dest_file_path)<EOL><DEDENT>else:<EOL><INDENT>command = '<STR_LIT>' % (source_file_path,<EOL>dest_file_path)<EOL><DEDENT>commands.append(command)<EOL><DEDENT><DEDENT>return '<STR_LIT:\n>'.join(commands)<EOL>

Returns a command that will stage inputs.


def _delocalize_outputs_commands(self, task_dir, outputs, user_project):

commands = []<EOL>for o in outputs:<EOL><INDENT>if o.recursive or not o.value:<EOL><INDENT>continue<EOL><DEDENT>dest_path = o.uri.path<EOL>local_path = task_dir + '<STR_LIT:/>' + _DATA_SUBDIR + '<STR_LIT:/>' + o.docker_path<EOL>if o.file_provider == job_model.P_LOCAL:<EOL><INDENT>commands.append('<STR_LIT>' % dest_path)<EOL><DEDENT>if o.file_provider in [job_model.P_LOCAL, job_model.P_GCS]:<EOL><INDENT>if user_project:<EOL><INDENT>command = '<STR_LIT>' % (user_project, local_path,<EOL>dest_path)<EOL><DEDENT>else:<EOL><INDENT>command = '<STR_LIT>' % (local_path, dest_path)<EOL><DEDENT>commands.append(command)<EOL><DEDENT><DEDENT>return '<STR_LIT:\n>'.join(commands)<EOL>


return self._raw._asdict()<EOL>

Return a provider-specific representation of task data.

        Returns:
          string of task data from the provider.


if not value:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>return '<STR_LIT:\n>'.join(value[-count:])<EOL>

Return the last line(s) as a single (newline delimited) string.


return {<EOL>'<STR_LIT>': network,<EOL>'<STR_LIT>': machine_type,<EOL>'<STR_LIT>': preemptible,<EOL>'<STR_LIT>': service_account,<EOL>'<STR_LIT>': boot_disk_size_gb,<EOL>'<STR_LIT>': disks,<EOL>'<STR_LIT>': accelerators,<EOL>'<STR_LIT>': labels,<EOL>'<STR_LIT>': cpu_platform,<EOL>'<STR_LIT>': nvidia_driver_version,<EOL>}<EOL>

Build a VirtualMachine object for a Pipeline request.

    Args:
      network (dict): Network details for the pipeline to run in.
      machine_type (str): GCE Machine Type string for the pipeline.
      preemptible (bool): Use a preemptible VM for the job.
      service_account (dict): Service account configuration for the VM.
      boot_disk_size_gb (int): Boot disk size in GB.
      disks (list[dict]): List of disks to mount.
      accelerators (list[dict]): List of accelerators to attach to the VM.
      labels (dict[string, string]): Labels for the VM.
      cpu_platform (str): The CPU platform to request.
      nvidia_driver_version (str): The NVIDIA driver version to use when attaching
        an NVIDIA GPU accelerator.

    Returns:
      An object representing a VirtualMachine.


return {<EOL>'<STR_LIT>': project,<EOL>'<STR_LIT>': regions,<EOL>'<STR_LIT>': zones,<EOL>'<STR_LIT>': virtual_machine,<EOL>}<EOL>

Build a Resources object for a Pipeline request.

    Args:
      project (str): Cloud project for the Pipeline to run in.
      regions (List[str]): List of regions for the pipeline to run in.
      zones (List[str]): List of zones for the pipeline to run in.
      virtual_machine(str): Virtual machine type string.

    Returns:
      An object representing a Resource.


return {<EOL>'<STR_LIT>': disk,<EOL>'<STR_LIT:path>': path,<EOL>'<STR_LIT>': read_only,<EOL>}<EOL>

Build a Mount object for a Pipeline request.

    Args:
      disk (str): Name of disk to mount, as specified in the resources section.
      path (str): Path to mount the disk at inside the container.
      read_only (boolean): If true,  disk is mounted read only in the container.

    Returns:
      An object representing a Mount.


return {<EOL>'<STR_LIT:name>': name,<EOL>'<STR_LIT>': image_uri,<EOL>'<STR_LIT>': commands,<EOL>'<STR_LIT>': entrypoint,<EOL>'<STR_LIT>': environment,<EOL>'<STR_LIT>': pid_namespace,<EOL>'<STR_LIT>': flags,<EOL>'<STR_LIT>': port_mappings,<EOL>'<STR_LIT>': mounts,<EOL>'<STR_LIT>': labels,<EOL>}<EOL>

Build an Action object for a Pipeline request.

    Args:
      name (str): An optional name for the container.
      image_uri (str): The URI to pull the container image from.
      commands (List[str]): commands and arguments to run inside the container.
      entrypoint (str): overrides the ENTRYPOINT specified in the container.
      environment (dict[str,str]): The environment to pass into the container.
      pid_namespace (str): The PID namespace to run the action inside.
      flags (str): Flags that control the execution of this action.
      port_mappings (dict[int, int]): A map of container to host port mappings for
        this container.
      mounts (List): A list of mounts to make available to the action.
      labels (dict[str]): Labels to associate with the action.

    Returns:
      An object representing an Action resource.


def build_pipeline(actions, resources, environment, timeout):

return {<EOL>'<STR_LIT>': actions,<EOL>'<STR_LIT>': resources,<EOL>'<STR_LIT>': environment,<EOL>'<STR_LIT>': timeout,<EOL>}<EOL>

Build an Pipeline argument for a Pipeline request.

    Args:
      actions (List): A list of actions to execute.
      resources (dict): An object indicating pipeline resources.
      environment (dict[str,str]): The environment to pass into the container.
      timeout (str): A duration in seconds with up to nine fractional digits,
        terminated by 's'.

    Returns:
      An object representing a Pipelines Resource.


provider = getattr(args, '<STR_LIT>', '<STR_LIT>')<EOL>if provider == '<STR_LIT>':<EOL><INDENT>return google.GoogleJobProvider(<EOL>getattr(args, '<STR_LIT>', False),<EOL>getattr(args, '<STR_LIT>', False), args.project)<EOL><DEDENT>elif provider == '<STR_LIT>':<EOL><INDENT>return google_v2.GoogleV2JobProvider(<EOL>getattr(args, '<STR_LIT>', False), getattr(args, '<STR_LIT>', False),<EOL>args.project)<EOL><DEDENT>elif provider == '<STR_LIT>':<EOL><INDENT>return local.LocalJobProvider(resources)<EOL><DEDENT>elif provider == '<STR_LIT>':<EOL><INDENT>return test_fails.FailsJobProvider()<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' + provider)<EOL><DEDENT>

Returns a provider for job submission requests.


return PROVIDER_NAME_MAP[provider.__class__]<EOL>


parser = argparse.ArgumentParser(prog=prog, formatter_class=DsubHelpFormatter)<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>default='<STR_LIT>',<EOL>choices=['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'],<EOL>help="""<STR_LIT>""",<EOL>metavar='<STR_LIT>')<EOL>return parser<EOL>

Create an argument parser, adding in the list of providers.


def parse_args(parser, provider_required_args, argv):

<EOL>epilog = '<STR_LIT>'<EOL>for provider in provider_required_args:<EOL><INDENT>epilog += '<STR_LIT>' % (provider, provider_required_args[provider])<EOL><DEDENT>parser.epilog = epilog<EOL>args = parser.parse_args(argv)<EOL>for arg in provider_required_args[args.provider]:<EOL><INDENT>if not args.__getattribute__(arg):<EOL><INDENT>parser.error('<STR_LIT>' % arg)<EOL><DEDENT><DEDENT>return args<EOL>

Add provider required arguments epilog message, parse, and validate.


provider_name = get_provider_name(provider)<EOL>args = []<EOL>if provider_name == '<STR_LIT>':<EOL><INDENT>args.append('<STR_LIT>' % project)<EOL><DEDENT>elif provider_name == '<STR_LIT>':<EOL><INDENT>args.append('<STR_LIT>' % project)<EOL><DEDENT>elif provider_name == '<STR_LIT>':<EOL><INDENT>pass<EOL><DEDENT>elif provider_name == '<STR_LIT>':<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>assert False, '<STR_LIT>' % provider<EOL><DEDENT>args.insert(<NUM_LIT:0>, '<STR_LIT>' % provider_name)<EOL>return '<STR_LIT:U+0020>'.join(args)<EOL>

A string with the arguments to point dstat to the same provider+project.


<EOL>return get_dstat_provider_args(provider_type, project)<EOL>

A string with the arguments to point ddel to the same provider+project.


if args.label and args.provider not in [<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'<EOL>]:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>' % args.provider)<EOL><DEDENT>

Raise an error if the provider doesn't support a provided flag.


def _format_task_uri(fmt, job_metadata, task_metadata):

values = {<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None,<EOL>'<STR_LIT>': None<EOL>}<EOL>for key in values:<EOL><INDENT>values[key] = task_metadata.get(key) or job_metadata.get(key) or values[key]<EOL><DEDENT>return fmt.format(**values)<EOL>

Returns a URI with placeholders replaced by metadata values.


def format_logging_uri(uri, job_metadata, task_metadata):

<EOL>fmt = str(uri)<EOL>if '<STR_LIT:{>' not in fmt:<EOL><INDENT>if uri.endswith('<STR_LIT>'):<EOL><INDENT>fmt = os.path.splitext(uri)[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>fmt = os.path.join(uri, '<STR_LIT>')<EOL><DEDENT>if task_metadata.get('<STR_LIT>') is not None:<EOL><INDENT>fmt += '<STR_LIT>'<EOL><DEDENT>if task_metadata.get('<STR_LIT>') is not None:<EOL><INDENT>fmt += '<STR_LIT>'<EOL><DEDENT>fmt += '<STR_LIT>'<EOL><DEDENT>return _format_task_uri(fmt, job_metadata, task_metadata)<EOL>

Inserts task metadata into the logging URI.

    The core behavior is inspired by the Google Pipelines API:
      (1) If a the uri ends in ".log", then that is the logging path.
      (2) Otherwise, the uri is treated as "directory" for logs and a filename
          needs to be automatically generated.

    For (1), if the job is a --tasks job, then the {task-id} is inserted
    before ".log".

    For (2), the file name generated is {job-id}, or for --tasks jobs, it is
    {job-id}.{task-id}.

    In both cases .{task-attempt} is inserted before .log for --retries jobs.

    In addition, full task metadata substitution is supported. The URI
    may include substitution strings such as
    "{job-id}", "{task-id}", "{job-name}", "{user-id}", and "{task-attempt}".

    Args:
      uri: User-specified logging URI which may contain substitution fields.
      job_metadata: job-global metadata.
      task_metadata: tasks-specific metadata.

    Returns:
      The logging_uri formatted as described above.


self._operations = [StubTask(o) for o in ops]<EOL>

Set the state of the fictional world.

        Args:
         ops: a list of dict, each representing an operation.

        Operations can have the following fields:
           - status: tuple (string,date)
           - user: string
           - job-id: string
           - job-name: string
           - task-id: string
           - task-attempt: integer
           - labels: list<dict>
           - status-message: string
           - error-messages : list of string


statuses = None if statuses == {'<STR_LIT:*>'} else statuses<EOL>user_ids = None if user_ids == {'<STR_LIT:*>'} else user_ids<EOL>job_ids = None if job_ids == {'<STR_LIT:*>'} else job_ids<EOL>job_names = None if job_names == {'<STR_LIT:*>'} else job_names<EOL>task_ids = None if task_ids == {'<STR_LIT:*>'} else task_ids<EOL>task_attempts = None if task_attempts == {'<STR_LIT:*>'} else task_attempts<EOL>if labels or create_time_min or create_time_max:<EOL><INDENT>raise NotImplementedError(<EOL>'<STR_LIT>')<EOL><DEDENT>operations = [<EOL>x for x in self._operations<EOL>if ((not statuses or x.get_field('<STR_LIT:status>', (None, None))[<NUM_LIT:0>] in statuses<EOL>) and (not user_ids or x.get_field('<STR_LIT:user>', None) in user_ids) and<EOL>(not job_ids or x.get_field('<STR_LIT>', None) in job_ids) and<EOL>(not job_names or x.get_field('<STR_LIT>', None) in job_names) and<EOL>(not task_ids or x.get_field('<STR_LIT>', None) in task_ids) and<EOL>(not task_attempts or<EOL>x.get_field('<STR_LIT>', None) in task_attempts))<EOL>]<EOL>if max_tasks > <NUM_LIT:0>:<EOL><INDENT>operations = operations[:max_tasks]<EOL><DEDENT>return operations<EOL>

Return a list of operations. See base.py for additional detail.


return '<STR_LIT>'.format(label_key, label_value)<EOL>

Return a valid label filter for operations.list().


return '<STR_LIT>'.format(comparator, create_time)<EOL>

Return a valid createTime filter for operations.list().


return op.get('<STR_LIT>', {}).get('<STR_LIT>')<EOL>

Return the create time string of the operation.


Return the start time string of the operation.


Return the end time string of the operation.


Return the internal metadata type of the operation.


return op.get('<STR_LIT:error>')<EOL>

Return the error structure for the operation.


return op.get('<STR_LIT>', False)<EOL>

Return whether the operation has been marked done.


return is_done(op) and ('<STR_LIT:error>' not in op)<EOL>

Return whether the operation has completed successfully.


error = get_error(op)<EOL>return error and error.get('<STR_LIT:code>', <NUM_LIT:0>) == <NUM_LIT:1><EOL>

Return whether the operation was canceled by the user.


error = get_error(op)<EOL>return error and error.get('<STR_LIT:code>', <NUM_LIT:0>) > <NUM_LIT:1><EOL>

Return whether the operation has failed.


return op.get('<STR_LIT>', {}).get('<STR_LIT>', {})<EOL>


return get_labels(op).get(name)<EOL>

Return the value for the specified label.


return op.get('<STR_LIT>', {}).get('<STR_LIT>').get('<STR_LIT>', [])<EOL>

Return the operation's array of actions.


actions = get_actions(op)<EOL>if actions and <NUM_LIT:1> <= action_id < len(actions):<EOL><INDENT>return actions[action_id - <NUM_LIT:1>]<EOL><DEDENT>


actions = get_actions(op)<EOL>for action in actions:<EOL><INDENT>if action.get('<STR_LIT:name>') == name:<EOL><INDENT>return action<EOL><DEDENT><DEDENT>

Return the value for the specified action.


action = _get_action_by_name(op, name)<EOL>if action:<EOL><INDENT>return action.get('<STR_LIT>')<EOL><DEDENT>

Return the environment for the operation.


return op.get('<STR_LIT>', {}).get('<STR_LIT>', [])<EOL>

Return the array of events for the operation.


events = get_events(op)<EOL>if events:<EOL><INDENT>return events[<NUM_LIT:0>]<EOL><DEDENT>return None<EOL>

Return the last event (if any) for the operation.


events = get_events(op)<EOL>if events:<EOL><INDENT>return [<EOL>e for e in events if int(e.get('<STR_LIT>', {}).get('<STR_LIT>', <NUM_LIT:0>)) != <NUM_LIT:0><EOL>]<EOL><DEDENT>return None<EOL>

Return the events (if any) with a non-zero exitStatus.


events = get_events(op)<EOL>if not events:<EOL><INDENT>return None<EOL><DEDENT>return [e for e in events if e.get('<STR_LIT>', {}).get('<STR_LIT>') == event_type]<EOL>


last_update = get_end_time(op)<EOL>if not last_update:<EOL><INDENT>last_event = get_last_event(op)<EOL>if last_event:<EOL><INDENT>last_update = last_event['<STR_LIT>']<EOL><DEDENT><DEDENT>if not last_update:<EOL><INDENT>last_update = get_create_time(op)<EOL><DEDENT>return last_update<EOL>

Return the most recent timestamp in the operation.


return op.get('<STR_LIT>', {}).get('<STR_LIT>').get('<STR_LIT>', {})<EOL>


return get_metadata_type(<EOL>op) == '<STR_LIT>'<EOL>

Check that an operation is a genomics pipeline run.

    An operation is a Genomics Pipeline run if the request metadata's @type
    is "type.googleapis.com/google.genomics.v2alpha1.Metadata".

    Args:
      op: a pipelines operation.

    Returns:
      Boolean, true if the operation is a RunPipelineRequest.


if not is_pipeline(op):<EOL><INDENT>return False<EOL><DEDENT>for name in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>if not get_label(op, name):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>

Determine if a pipelines operation is a dsub request.

    We don't have a rigorous way to identify an operation as being submitted
    by dsub. Our best option is to check for certain fields that have always
    been part of dsub operations.

    - labels: job-id, job-name, and user-id have always existed. The dsub-version
              label has always existed for the google-v2 provider.

    Args:
      op: a pipelines operation.

    Returns:
      Boolean, true if the pipeline run was generated by dsub.


return [t.raw_task_data() for t in tasklist]<EOL>

Convert returned operations to raw operations.


self._chronology = chronology<EOL>self._now = <NUM_LIT:0><EOL>self._next = six.advance_iterator(self._chronology)<EOL>

The chronology is a generator that updates the world.

        Yield how many seconds of fake time elapse. Think of each yield as a
        "sleep". We read the very first value immediately, so that the world can be
        initialized in the chronology.

        Args:
          chronology: a generator that updates the state of the world and then
          sleeps via yielding.


if seconds < <NUM_LIT:0>:<EOL><INDENT>raise IOError("<STR_LIT>")<EOL><DEDENT>target = self._now + seconds<EOL>if target < self._next:<EOL><INDENT>self._now = target<EOL>return<EOL><DEDENT>self._now = target<EOL>for step in self._chronology:<EOL><INDENT>self._next += step<EOL>if self._next > target:<EOL><INDENT>return<EOL><DEDENT><DEDENT>raise BaseException("<STR_LIT>")<EOL>

Sleep for this many fictitious seconds.

        The world will update accordingly.

        Args:
          seconds: how many seconds to pretend sleep for.

        Raises:
          IOError: for negative delays (matches time.sleep).
          BaseException: at the end of times, to detect infinite loop bugs
        and the like.


return self._now<EOL>


if sys.version_info[<NUM_LIT:0>] < <NUM_LIT:3>:<EOL><INDENT>return stdoutbytes<EOL><DEDENT>encoding = sys.stdout.encoding if sys.stdout.encoding else '<STR_LIT:utf-8>'<EOL>return stdoutbytes.decode(encoding)<EOL>

Convert stdout to a string in python 2 or 3.


result = list(difflib.Differ().compare(<EOL>str1.splitlines(True), str2.splitlines(True)))<EOL>same = True<EOL>for line in result:<EOL><INDENT>if line[<NUM_LIT:0>] in ('<STR_LIT:->', '<STR_LIT:+>'):<EOL><INDENT>same = False<EOL>sys.stdout.write(line)<EOL><DEDENT><DEDENT>return same<EOL>

Determine if two strings differ, emit differences to stdout.


def expand_tsv_fields(newenv, tmpl_file, tsv_file):

with open(tmpl_file, '<STR_LIT:r>') as f:<EOL><INDENT>input_lines = f.readlines()<EOL><DEDENT>with open(tsv_file, '<STR_LIT:w>') as f:<EOL><INDENT>f.write(input_lines[<NUM_LIT:0>])<EOL>for line in input_lines[<NUM_LIT:1>:]:<EOL><INDENT>curr = []<EOL>for field in line.split('<STR_LIT:\t>'):<EOL><INDENT>cell = subprocess.check_output(<EOL>'<STR_LIT>' % field, env=newenv, shell=True)<EOL>curr.append(cell)<EOL><DEDENT>f.write('<STR_LIT:\t>'.join(curr) + '<STR_LIT:\n>')<EOL><DEDENT><DEDENT>

Write a TSV file, expanding environment variables in the template.


def get_field_from_tsv(tsv_file, match_column, match_value, return_column):

with open(tsv_file, '<STR_LIT:r>') as f:<EOL><INDENT>input_lines = f.readlines()<EOL><DEDENT>fields = input_lines[<NUM_LIT:0>].strip().split('<STR_LIT:\t>')<EOL>match_column_idx = fields.index(match_column)<EOL>return_column_idx = fields.index(return_column)<EOL>for line in input_lines[<NUM_LIT:1>:]:<EOL><INDENT>fields = line.strip().split('<STR_LIT:\t>')<EOL>if re.search(match_value, fields[match_column_idx]):<EOL><INDENT>return fields[return_column_idx]<EOL><DEDENT><DEDENT>

Reads a TSV, matches on a column value and returns another column.


if value.startswith('<STR_LIT:">') and value.endswith('<STR_LIT:">'):<EOL><INDENT>return value[<NUM_LIT:1>:-<NUM_LIT:1>].replace('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>return value<EOL>

Remove surrounding single or double quotes.


e = dict(os.environ)<EOL>for var in TEST_VARS + TEST_E2E_VARS:<EOL><INDENT>e[var] = globals()[var]<EOL><DEDENT>return e<EOL>

Merge the current enviornment and test variables into a dictionary.


<EOL>google_opt_args = [<EOL>("<STR_LIT>", "<STR_LIT>"),<EOL>("<STR_LIT>", "<STR_LIT>")<EOL>]<EOL>opt_args = []<EOL>for var in google_opt_args:<EOL><INDENT>val = globals().get(var[<NUM_LIT:0>])<EOL>if val:<EOL><INDENT>opt_args.append(var[<NUM_LIT:1>], val)<EOL><DEDENT><DEDENT>return dsub_command.call([<EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", PROJECT_ID,<EOL>"<STR_LIT>", LOGGING,<EOL>"<STR_LIT>", "<STR_LIT>"<EOL>] + opt_args + dsub_args)<EOL>

Call dsub appending google-provider required arguments.


<EOL>google_v2_opt_args = [<EOL>("<STR_LIT>", "<STR_LIT>"),<EOL>("<STR_LIT>", "<STR_LIT>")<EOL>]<EOL>opt_args = []<EOL>for var in google_v2_opt_args:<EOL><INDENT>val = globals().get(var[<NUM_LIT:0>])<EOL>if val:<EOL><INDENT>opt_args.append(var[<NUM_LIT:1>], val)<EOL><DEDENT><DEDENT>return dsub_command.call([<EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", PROJECT_ID,<EOL>"<STR_LIT>", LOGGING,<EOL>"<STR_LIT>", "<STR_LIT>"<EOL>] + opt_args + dsub_args)<EOL>

Call dsub appending google-v2 required arguments.


<EOL>return dsub_command.call([<EOL>"<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", LOGGING,<EOL>] + dsub_args)<EOL>

Call dsub appending local-provider required arguments.


filename = os.path.join(os.path.dirname(__file__), '<STR_LIT>')<EOL>with open(filename, '<STR_LIT:r>') as versionfile:<EOL><INDENT>for line in versionfile:<EOL><INDENT>if line.startswith('<STR_LIT>'):<EOL><INDENT>version = line.partition('<STR_LIT:=>')[<NUM_LIT:2>]<EOL>return version.strip().strip('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>raise ValueError('<STR_LIT>')<EOL>

Get the dsub version out of the _dsub_version.py source file.

    Setup.py should not import dsub version from dsub directly since ambiguity in
    import order could lead to an old version of dsub setting the version number.
    Parsing the file directly is simpler than using import tools (whose interface
    varies between python 2.7, 3.4, and 3.5).

    Returns:
      string of dsub version.

    Raises:
      ValueError: if the version is not found.


def normalize_query(query_string, terms=TERMS, norm_space=NORM_SPACE):

return [<EOL>norm_space('<STR_LIT:U+0020>', (t[<NUM_LIT:0>] or t[<NUM_LIT:1>]).strip()) for t in terms(query_string)]<EOL>

Example:
>>> normalize_query('  some random  words "with   quotes  " and   spaces')
['some', 'random', 'words', 'with quotes', 'and', 'spaces']


loggers = list(Logger.manager.loggerDict.items())<EOL>loggers.append(("<STR_LIT:root>", getLogger()))<EOL>level_filter = LevelFilter(levels)<EOL>for logger_name, logger in loggers:<EOL><INDENT>for handler in getattr(logger, "<STR_LIT>", []):<EOL><INDENT>handler.addFilter(level_filter)<EOL><DEDENT><DEDENT>

see --quiet flag help for what this does


<EOL>sys.stdout.write(line)<EOL>sys.stdout.flush()<EOL>


def execute(self, arg_str='<STR_LIT>', **kwargs):

cmd = "<STR_LIT>".format(self.cmd_prefix, self.script, arg_str)<EOL>expected_ret_code = kwargs.pop('<STR_LIT:code>', <NUM_LIT:0>)<EOL>environ = self.environ<EOL>for k in list(kwargs.keys()):<EOL><INDENT>if k.isupper():<EOL><INDENT>environ[k] = kwargs.pop(k)<EOL><DEDENT><DEDENT>kwargs.setdefault("<STR_LIT>", subprocess.STDOUT)<EOL>kwargs["<STR_LIT>"] = True<EOL>kwargs["<STR_LIT>"] = subprocess.PIPE<EOL>kwargs["<STR_LIT>"] = self.cwd<EOL>kwargs["<STR_LIT>"] = environ<EOL>process = None<EOL>self.buf = deque(maxlen=self.bufsize)<EOL>try:<EOL><INDENT>process = subprocess.Popen(<EOL>cmd,<EOL>**kwargs<EOL>)<EOL>for line in iter(process.stdout.readline, b"<STR_LIT>"):<EOL><INDENT>line = line.decode(self.encoding)<EOL>self.buf.append(line.rstrip())<EOL>yield line<EOL><DEDENT>process.wait()<EOL>if process.returncode != expected_ret_code:<EOL><INDENT>if process.returncode > <NUM_LIT:0>:<EOL><INDENT>raise RuntimeError("<STR_LIT>".format(<EOL>cmd,<EOL>process.returncode,<EOL>self.output<EOL>))<EOL><DEDENT><DEDENT><DEDENT>except subprocess.CalledProcessError as e:<EOL><INDENT>if e.returncode != expected_ret_code:<EOL><INDENT>raise RuntimeError("<STR_LIT>".format(<EOL>cmd,<EOL>e.returncode,<EOL>self.output<EOL>))<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>if process:<EOL><INDENT>process.stdout.close()<EOL><DEDENT><DEDENT>

runs the passed in arguments and returns an iterator on the output of
        running command


quiet = False<EOL>progress_class = kwargs.pop("<STR_LIT>", Progress)<EOL>kwargs["<STR_LIT>"] = istdout.info<EOL>kwargs["<STR_LIT:width>"] = kwargs.get("<STR_LIT:width>", globals()["<STR_LIT>"])<EOL>kwargs["<STR_LIT>"] = length<EOL>pbar = progress_class(**kwargs)<EOL>pbar.update(<NUM_LIT:0>)<EOL>yield pbar<EOL>pbar.update(length)<EOL>br()<EOL>

display a progress that can update in place

    example --
        total_length = 1000
        with echo.progress(total_length) as p:
            for x in range(total_length):
                # do something crazy
                p.update(x)

    length -- int -- the total size of what you will be updating progress on


kwargs["<STR_LIT>"] = ProgressBar<EOL>return progress(length, **kwargs)<EOL>

display a progress bar

    example --
        total_length = 1000
        with echo.progress_bar(total_length) as bar:
            for x in range(total_length):
                # do something crazy
                bar.update(x)

    length -- int -- the total size of what you will be iterating on


def increment(itr, n=<NUM_LIT:1>, format_msg="<STR_LIT>"):

for i, v in enumerate(itr, n):<EOL><INDENT>with prefix(format_msg, i):<EOL><INDENT>yield v<EOL><DEDENT><DEDENT>

Similar to enumerate but will set format_msg.format(n) into the prefix on
    each iteration

    :Example:
        for v in increment(["foo", "bar"]):
            echo.out(v) # 1. foo\n2. bar

    :param itr: iterator, any iterator you want to set a numeric prefix on on every
        iteration
    :param n: integer, the starting integer for the numeric prefix
    :param format_msg: string, this will basically do: format_msg.format(n) so there
        should only be one set of curly brackets
    :returns: yield generator


def prefix(format_msg, *args, **kwargs):

return Prefix(format_msg, *args, **kwargs)<EOL>

Return a prefix instance

    :Example:
        with echo.prefix("foo "):
            echo.out("bar") # "foo bar"


stderr.exception(e)<EOL>

print an exception message to stderr (this does not honor quiet)


exc_info = kwargs.pop("<STR_LIT>", False)<EOL>stderr.warning(str(format_msg).format(*args, **kwargs), exc_info=exc_info)<EOL>


<EOL>istdout.info(c)<EOL>

print one or more characters without a newline at the end

    example --
        for x in range(1000):
            echo.ch(".")

    c -- string -- the chars that will be output


def out(format_msg="<STR_LIT>", *args, **kwargs):

logmethod = kwargs.get("<STR_LIT>", stdout.info)<EOL>if format_msg != "<STR_LIT>":<EOL><INDENT>if Prefix.has():<EOL><INDENT>if isinstance(format_msg, basestring):<EOL><INDENT>format_msg = Prefix.get() + format_msg<EOL><DEDENT>else:<EOL><INDENT>format_msg = Prefix.get() + str(format_msg)<EOL><DEDENT><DEDENT>if isinstance(format_msg, basestring):<EOL><INDENT>if args or kwargs:<EOL><INDENT>s = format_msg.format(*args, **kwargs)<EOL><DEDENT>else:<EOL><INDENT>s = format_msg<EOL><DEDENT>logmethod(s)<EOL><INDENT>width = globals()["<STR_LIT:width>"]<EOL>s = textwrap.fill(s, width=width)<EOL>stdout.info(s)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>logmethod(str(format_msg))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>logmethod("<STR_LIT>")<EOL><DEDENT>

print format_msg to stdout, taking into account --quiet setting


def verbose(format_msg="<STR_LIT>", *args, **kwargs):

kwargs["<STR_LIT>"] = stdout.debug<EOL>out(format_msg, *args, **kwargs)<EOL>

print format_msg to stdout, taking into account --verbose flag


if not width: width = globals()["<STR_LIT>"]<EOL>bar("<STR_LIT:_>", width=width)<EOL>blank()<EOL>

similar to the html horizontal rule in html


for x in range(count):<EOL><INDENT>out("<STR_LIT>")<EOL><DEDENT>


bullets(*lines, numbers=True)<EOL>


bullets(*lines, numbers=False)<EOL>


sep = kwargs.get("<STR_LIT>", "<STR_LIT:*>")<EOL>count = kwargs.get("<STR_LIT:width>", globals()["<STR_LIT>"])<EOL>out(sep * count)<EOL>if lines:<EOL><INDENT>out(sep)<EOL>for line in lines:<EOL><INDENT>out("<STR_LIT>".format(sep, line))<EOL><DEDENT>out(sep)<EOL>out(sep * count)<EOL><DEDENT>

prints a banner

    sep -- string -- the character that will be on the line on the top and bottom
        and before any of the lines, defaults to *
    count -- integer -- the line width, defaults to 80


return table([row], **kwargs)<EOL>

This is more of a specialty version of table that just prints one row

    this is handy for when you are iterating through data and don't want to print
    it all at the end but would rather print it one row at a time

    :param row: list, the columns of the row you want to print
    :param **kwargs: dict, all the options in table()


ret = []<EOL>prefix = kwargs.get('<STR_LIT>', '<STR_LIT>')<EOL>buf_count = kwargs.get('<STR_LIT>', <NUM_LIT:2>)<EOL>if len(columns) == <NUM_LIT:1>:<EOL><INDENT>columns = list(columns[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>columns = list(zip(*columns))<EOL><DEDENT>headers = kwargs.get("<STR_LIT>", [])<EOL>if headers:<EOL><INDENT>columns.insert(<NUM_LIT:0>, headers)<EOL><DEDENT>widths = kwargs.get("<STR_LIT>", [])<EOL>row_counts = Counter()<EOL>for i in range(len(widths)):<EOL><INDENT>row_counts[i] = int(widths[i])<EOL><DEDENT>width = int(kwargs.get("<STR_LIT:width>", <NUM_LIT:0>))<EOL>for row in columns:<EOL><INDENT>for i, c in enumerate(row):<EOL><INDENT>if isinstance(c, basestring):<EOL><INDENT>cl = len(c)<EOL><DEDENT>else:<EOL><INDENT>cl = len(str(c))<EOL><DEDENT>if cl > row_counts[i]:<EOL><INDENT>row_counts[i] = cl<EOL><DEDENT><DEDENT><DEDENT>width = int(kwargs.get("<STR_LIT:width>", <NUM_LIT:0>))<EOL>if width:<EOL><INDENT>for i in row_counts:<EOL><INDENT>if row_counts[i] < width:<EOL><INDENT>row_counts[i] = width<EOL><DEDENT><DEDENT><DEDENT>def colstr(c):<EOL><INDENT>if isinstance(c, basestring): return c<EOL>return str(c)<EOL><DEDENT>def rowstr(row, prefix, row_counts):<EOL><INDENT>row_format = prefix<EOL>cols = list(map(colstr, row))<EOL>for i in range(len(row_counts)):<EOL><INDENT>c = cols[i]<EOL>if re.match(r"<STR_LIT>", c):<EOL><INDENT>if i == <NUM_LIT:0>:<EOL><INDENT>row_format += "<STR_LIT>" + str(row_counts[i]) + "<STR_LIT:}>"<EOL><DEDENT>else:<EOL><INDENT>row_format += "<STR_LIT>" + str(row_counts[i] + buf_count) + "<STR_LIT:}>"<EOL><DEDENT><DEDENT>else:<EOL><INDENT>row_format += "<STR_LIT>" + str(row_counts[i] + buf_count) + "<STR_LIT:}>"<EOL><DEDENT><DEDENT>return row_format.format(*cols)<EOL><DEDENT>for row in columns:<EOL><INDENT>ret.append(rowstr(row, prefix, row_counts))<EOL><DEDENT>out(os.linesep.join(ret))<EOL>

format columned data so we can easily print it out on a console, this just takes
columns of data and it will format it into properly aligned columns, it's not
fancy, but it works for most type of strings that I need it for, like server name
lists.

other formatting options:
    http://stackoverflow.com/a/8234511/5006

other packages that probably do this way better:
    https://stackoverflow.com/a/26937531/5006

:Example:
    >>> echo.table([(1, 2), (3, 4), (5, 6), (7, 8), (9, 0)])
    1  2
    3  4
    5  6
    7  8
    9  0
    >>> echo.table([1, 3, 5, 7, 9], [2, 4, 6, 8, 0])
    1  2
    3  4
    5  6
    7  8
    9  0

:param *columns: can either be a list of rows or multiple lists representing each
    column in the table
:param **kwargs: dict
    prefix -- string -- what you want before each row (eg, a tab)
    buf_count -- integer -- how many spaces between longest col value and its neighbor
    headers -- list -- the headers you want, must match column count
    widths -- list -- the widths of each column you want to use, this doesn't have
        to match column count, so you can do something like [0, 5] to set the
        width of the second column
    width -- int -- similar to widths except it will set this value for all columns


if not re.match("<STR_LIT>", question):<EOL><INDENT>question = "<STR_LIT>".format(question)<EOL><DEDENT>while True:<EOL><INDENT>if sys.version_info[<NUM_LIT:0>] > <NUM_LIT:2>:<EOL><INDENT>answer = input(question)<EOL><DEDENT>else:<EOL><INDENT>answer = raw_input(question)<EOL><DEDENT>if not choices or answer in choices:<EOL><INDENT>break<EOL><DEDENT><DEDENT>return answer<EOL>

echo a prompt to the user and wait for an answer

    question -- string -- the prompt for the user
    choices -- list -- if given, only exit when prompt matches one of the choices
    return -- string -- the answer that was given by the user


@arg("<STR_LIT:path>", default=os.getcwd(), nargs='<STR_LIT:?>', help="<STR_LIT>")<EOL>def main(path):

basepath = os.path.abspath(os.path.expanduser(str(path)))<EOL>echo.h2("<STR_LIT>".format(basepath))<EOL>echo.br()<EOL>for root_dir, dirs, files in os.walk(basepath, topdown=True):<EOL><INDENT>for f in fnmatch.filter(files, '<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>filepath = os.path.join(root_dir, f)<EOL>with open(filepath, encoding="<STR_LIT>") as fp:<EOL><INDENT>body = fp.read()<EOL>is_console = "<STR_LIT>" in body<EOL>is_console = is_console or "<STR_LIT:code>" in body<EOL>is_console = is_console and "<STR_LIT>" in body<EOL>if is_console:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>s = captain.Script(filepath)<EOL>if s.can_run_from_cli():<EOL><INDENT>rel_filepath = s.call_path(basepath)<EOL>p = s.parser<EOL>echo.h3(rel_filepath)<EOL>desc = p.description<EOL>if desc:<EOL><INDENT>echo.indent(desc, indent=("<STR_LIT:U+0020>" * <NUM_LIT:4>))<EOL><DEDENT>subcommands = s.subcommands<EOL>if subcommands:<EOL><INDENT>echo.br()<EOL>echo.indent("<STR_LIT>", indent=("<STR_LIT:U+0020>" * <NUM_LIT:4>))<EOL>for sc in subcommands.keys():<EOL><INDENT>echo.indent(sc, indent=("<STR_LIT:U+0020>" * <NUM_LIT:6>))<EOL><DEDENT><DEDENT>echo.br()<EOL><DEDENT><DEDENT>except captain.ParseError:<EOL><INDENT>pass<EOL><DEDENT>except Exception as e:<EOL><INDENT>echo.err("<STR_LIT>", f)<EOL>echo.verbose(e.message)<EOL>echo.br()<EOL><DEDENT><DEDENT><DEDENT>

scan path directory and any subdirectories for valid captain scripts


if mod_name and mod_name == "<STR_LIT:__main__>":<EOL><INDENT>calling_mod = sys.modules.get("<STR_LIT:__main__>", None)<EOL><DEDENT>else:<EOL><INDENT>calling_mod = discover_if_calling_mod()<EOL><DEDENT>if calling_mod:<EOL><INDENT>s = Script(inspect.getfile(calling_mod), module=calling_mod)<EOL>raw_args = sys.argv[<NUM_LIT:1>:]<EOL>try:<EOL><INDENT>ret_code = s.run(raw_args)<EOL><DEDENT>except Stop as e:<EOL><INDENT>ret_code = e.code<EOL>msg = str(e)<EOL>if msg:<EOL><INDENT>if ret_code != <NUM_LIT:0>:<EOL><INDENT>echo.err(msg)<EOL><DEDENT>else:<EOL><INDENT>echo.out(msg)<EOL><DEDENT><DEDENT><DEDENT>sys.exit(ret_code)<EOL><DEDENT>

A stand-in for the normal sys.exit()

    all the magic happens here, when this is called at the end of a script it will
    figure out all the available commands and arguments that can be passed in,
    then handle exiting the script and returning the status code.

    :Example:

        from captain import exit
        exit(__name__)

    This also acts as a guard against the script being traditionally imported, so
    even if you have this at the end of the script, it won't actually exit if the
    script is traditionally imported


module = self.module<EOL>subcommands = self.subcommands<EOL>if subcommands:<EOL><INDENT>module_desc = inspect.getdoc(module)<EOL>parser = Parser(description=module_desc, module=module)<EOL>subparsers = parser.add_subparsers()<EOL>for sc_name, callback in subcommands.items():<EOL><INDENT>sc_name = sc_name.replace("<STR_LIT:_>", "<STR_LIT:->")<EOL>cb_desc = inspect.getdoc(callback)<EOL>sc_parser = subparsers.add_parser(<EOL>sc_name,<EOL>callback=callback,<EOL>help=cb_desc<EOL>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>parser = Parser(callback=self.callbacks[self.function_name], module=module)<EOL><DEDENT>return parser<EOL>


<EOL>if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>if "<STR_LIT:__main__>" in sys.modules:<EOL><INDENT>mod = sys.modules["<STR_LIT:__main__>"]<EOL>path = self.normalize_path(mod.__file__)<EOL>if os.path.splitext(path) == os.path.splitext(self.path):<EOL><INDENT>self._module = mod<EOL><DEDENT>else:<EOL><INDENT>self._module = imp.load_source('<STR_LIT>', self.path)<EOL><DEDENT><DEDENT><DEDENT>return self._module<EOL>

load the module so we can actually run the script's function


if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._body = inspect.getsource(self.module)<EOL><DEDENT>return self._body<EOL>


parser = self.parser<EOL>args, kwargs = parser.parse_callback_args(raw_args)<EOL>callback = kwargs.pop("<STR_LIT>")<EOL>if parser.has_injected_quiet():<EOL><INDENT>levels = kwargs.pop("<STR_LIT>", "<STR_LIT>")<EOL>logging.inject_quiet(levels)<EOL><DEDENT>try:<EOL><INDENT>ret_code = callback(*args, **kwargs)<EOL>ret_code = int(ret_code) if ret_code else <NUM_LIT:0><EOL><DEDENT>except ArgError as e:<EOL><INDENT>echo.err("<STR_LIT>", parser.prog, str(e))<EOL>ret_code = <NUM_LIT:2><EOL><DEDENT>return ret_code<EOL>

parse and import the script, and then run the script's main function


rel_filepath = self.path<EOL>if basepath:<EOL><INDENT>rel_filepath = os.path.relpath(self.path, basepath)<EOL><DEDENT>basename = self.name<EOL>if basename in set(['<STR_LIT>', '<STR_LIT>']):<EOL><INDENT>rel_filepath = os.path.dirname(rel_filepath)<EOL><DEDENT>return rel_filepath<EOL>

return that path to be able to call this script from the passed in
        basename

        example --
            basepath = /foo/bar
            self.path = /foo/bar/che/baz.py
            self.call_path(basepath) # che/baz.py

        basepath -- string -- the directory you would be calling this script in
        return -- string -- the minimum path that you could use to execute this script
            in basepath


if self.parsed: return<EOL>self.callbacks = {}<EOL>regex = re.compile("<STR_LIT>".format(self.function_name), flags=re.I)<EOL>mains = set()<EOL>body = self.body<EOL>ast_tree = ast.parse(self.body, self.path)<EOL>for n in ast_tree.body:<EOL><INDENT>if hasattr(n, '<STR_LIT:name>'):<EOL><INDENT>if regex.match(n.name):<EOL><INDENT>mains.add(n.name)<EOL><DEDENT><DEDENT>if hasattr(n, '<STR_LIT:value>'):<EOL><INDENT>ns = n.value<EOL>if hasattr(ns, '<STR_LIT:id>'):<EOL><INDENT>if regex.match(ns.id):<EOL><INDENT>mains.add(ns.id)<EOL><DEDENT><DEDENT><DEDENT>if hasattr(n, '<STR_LIT>'):<EOL><INDENT>ns = n.targets[<NUM_LIT:0>]<EOL>if hasattr(ns, '<STR_LIT:id>'):<EOL><INDENT>if regex.match(ns.id):<EOL><INDENT>mains.add(ns.id)<EOL><DEDENT><DEDENT><DEDENT>if hasattr(n, '<STR_LIT>'):<EOL><INDENT>for ns in n.names:<EOL><INDENT>if hasattr(ns, '<STR_LIT:name>'):<EOL><INDENT>if regex.match(ns.name):<EOL><INDENT>mains.add(ns.name)<EOL><DEDENT><DEDENT>if getattr(ns, '<STR_LIT>', None):<EOL><INDENT>if regex.match(ns.asname):<EOL><INDENT>mains.add(ns.asname)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if len(mains) > <NUM_LIT:0>:<EOL><INDENT>module = self.module<EOL>for function_name in mains:<EOL><INDENT>cb = getattr(module, function_name, None)<EOL>if cb and callable(cb):<EOL><INDENT>self.callbacks[function_name] = cb<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>raise ParseError("<STR_LIT>")<EOL><DEDENT>self.parsed = True<EOL>return len(self.callbacks) > <NUM_LIT:0><EOL>

load the script and set the parser and argument info

        I feel that this is way too brittle to be used long term, I think it just
        might be best to import the stupid module, the thing I don't like about that
        is then we import basically everything, which seems bad?


ret = False<EOL>ast_tree = ast.parse(self.body, self.path)<EOL>calls = self._find_calls(ast_tree, __name__, "<STR_LIT>")<EOL>for call in calls:<EOL><INDENT>if re.search("<STR_LIT>".format(re.escape(call)), self.body):<EOL><INDENT>ret = True<EOL>break<EOL><DEDENT><DEDENT>return ret<EOL>

return True if this script can be run from the command line


def _find_calls(self, ast_tree, called_module, called_func):

s = set()<EOL>s.add("<STR_LIT>".format(called_module, called_func))<EOL>if hasattr(ast_tree, '<STR_LIT:body>'):<EOL><INDENT>if isinstance(ast_tree.body, collections.Iterable):<EOL><INDENT>for ast_body in ast_tree.body:<EOL><INDENT>s.update(self._find_calls(ast_body, called_module, called_func))<EOL><DEDENT><DEDENT><DEDENT>elif hasattr(ast_tree, '<STR_LIT>'):<EOL><INDENT>if hasattr(ast_tree, '<STR_LIT>'):<EOL><INDENT>if ast_tree.module == called_module:<EOL><INDENT>for ast_name in ast_tree.names:<EOL><INDENT>if ast_name.name == called_func:<EOL><INDENT>s.add(unicode(ast_name.asname if ast_name.asname is not None else ast_name.name))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for ast_name in ast_tree.names:<EOL><INDENT>if hasattr(ast_name, '<STR_LIT:name>') and (ast_name.name == called_module):<EOL><INDENT>call = "<STR_LIT>".format(<EOL>ast_name.asname if ast_name.asname is not None else ast_name.name,<EOL>called_func<EOL>)<EOL>s.add(call)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return s<EOL>

scan the abstract source tree looking for possible ways to call the called_module
and called_func

borrowed from pout

ast_tree -- _ast.* instance -- the internal ast object that is being checked, returned from compile()
    with ast.PyCF_ONLY_AST flag
called_module -- string -- we are checking the ast for imports of this module
called_func -- string -- we are checking the ast for aliases of this function
return -- set -- the list of possible calls the ast_tree could make to call the called_func


def __init__(self, code, msg="<STR_LIT>", *args, **kwargs):

self.code = code<EOL>super(Stop, self).__init__(msg, *args, **kwargs)<EOL>

create the error

        :param code: integer, the exit code
        :param msg: string, the message will be printed out to stderr if code!=0 or
            stdout if code==0


ret = False<EOL>val = self.callback<EOL>if self.is_class(): return False<EOL>ret = not inspect.isfunction(val) and not inspect.ismethod(val)<EOL><INDENT>if is_py2:<EOL><INDENT>ret = isinstance(val, types.InstanceType) or hasattr(val, '<STR_LIT>')and not (hasattr(val, '<STR_LIT>') or hasattr(val, '<STR_LIT>'))<EOL><DEDENT>else:<EOL><INDENT>ret = not inspect.isfunction(val) and not inspect.ismethod(val)<EOL><DEDENT><DEDENT>return ret<EOL>

return True if callback is an instance of a class


if self.is_instance() or self.is_class(): return False<EOL>return isinstance(self.callback, (Callable, classmethod))<EOL>

return True if callback is a vanilla plain jane function


if kwargs:<EOL><INDENT>self.parser_kwargs.update(kwargs)<EOL><DEDENT>self.parser_kwargs.setdefault('<STR_LIT>', self.name)<EOL>if '<STR_LIT:default>' in kwargs:<EOL><INDENT>self.parser_kwargs["<STR_LIT:default>"] = kwargs["<STR_LIT:default>"]<EOL>self.parser_kwargs["<STR_LIT>"] = False<EOL><DEDENT>elif '<STR_LIT:action>' in kwargs:<EOL><INDENT>if kwargs['<STR_LIT:action>'] in set(['<STR_LIT>', '<STR_LIT:store_true>']):<EOL><INDENT>self.parser_kwargs['<STR_LIT>'] = False<EOL><DEDENT>elif kwargs['<STR_LIT:action>'] in set(['<STR_LIT:version>']):<EOL><INDENT>self.parser_kwargs.pop('<STR_LIT>', False)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.parser_kwargs.setdefault("<STR_LIT>", True)<EOL><DEDENT>

these kwargs come from the @arg decorator, they are then merged into any
        keyword arguments that were automatically generated from the main function
        introspection


def xs(name, parser_args, list_args):<EOL><INDENT>"""<STR_LIT>"""<EOL>for args, kwargs in list_args:<EOL><INDENT>if len(set(args) & parser_args) > <NUM_LIT:0>:<EOL><INDENT>yield args, kwargs<EOL><DEDENT>else:<EOL><INDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>if kwargs['<STR_LIT>'] == name:<EOL><INDENT>yield args, kwargs<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>for args, kwargs in xs(self.name, self.parser_args, list_args):<EOL><INDENT>self.merge_args(args)<EOL>self.merge_kwargs(kwargs)<EOL><DEDENT>

find any matching parser_args from list_args and merge them into this
        instance

        list_args -- list -- an array of (args, kwargs) tuples


kwargs = {}<EOL>if isinstance(na, (type, types.FunctionType)):<EOL><INDENT>kwargs['<STR_LIT:type>'] = na<EOL>kwargs['<STR_LIT>'] = True<EOL>kwargs["<STR_LIT:default>"] = argparse.SUPPRESS<EOL><DEDENT>elif isinstance(na, bool):<EOL><INDENT>kwargs['<STR_LIT:action>'] = '<STR_LIT>' if na else '<STR_LIT:store_true>'<EOL>kwargs['<STR_LIT>'] = False<EOL><DEDENT>elif isinstance(na, (int, float, str)):<EOL><INDENT>kwargs['<STR_LIT:type>'] = type(na)<EOL>kwargs['<STR_LIT:default>'] = na<EOL>kwargs['<STR_LIT>'] = False<EOL><DEDENT>elif isinstance(na, (list, set)):<EOL><INDENT>na = list(na)<EOL>kwargs['<STR_LIT:action>'] = '<STR_LIT>'<EOL>kwargs['<STR_LIT>'] = True<EOL>if len(na) > <NUM_LIT:0>:<EOL><INDENT>if isinstance(na[<NUM_LIT:0>], type):<EOL><INDENT>kwargs['<STR_LIT:type>'] = na[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>kwargs['<STR_LIT:action>'] = '<STR_LIT:store>'<EOL>l = set()<EOL>ltype = None<EOL>for elt in na:<EOL><INDENT>vtype = type(elt)<EOL>l.add(elt)<EOL>if ltype is None:<EOL><INDENT>ltype = vtype<EOL><DEDENT>else:<EOL><INDENT>if ltype is not vtype:<EOL><INDENT>ltype = str<EOL><DEDENT><DEDENT><DEDENT>kwargs['<STR_LIT>'] = l<EOL>kwargs['<STR_LIT:type>'] = ltype<EOL><DEDENT><DEDENT><DEDENT>self.parser_kwargs.update(kwargs)<EOL>

this is used for introspection from the main() method when there is an
        argument with a default value, this figures out how to set up the ArgParse
        arguments


def _fill_text(self, text, width, indent):

lines = []<EOL>for line in text.splitlines(False):<EOL><INDENT>if line:<EOL><INDENT>lines.extend(textwrap.wrap(<EOL>line.strip(),<EOL>width,<EOL>initial_indent=indent,<EOL>subsequent_indent=indent<EOL>))<EOL><DEDENT>else:<EOL><INDENT>lines.append(line)<EOL><DEDENT><DEDENT>text = "<STR_LIT:\n>".join(lines)<EOL>return text<EOL>

Overridden to not get rid of newlines

        https://github.com/python/cpython/blob/2.7/Lib/argparse.py#L620


args = []<EOL>arg_info = self.arg_info<EOL>kwargs = dict(arg_info['<STR_LIT>'])<EOL>parsed_args = []<EOL>unknown_args = getattr(self, "<STR_LIT>", False)<EOL>if unknown_args:<EOL><INDENT>parsed_args, parsed_unknown_args = self.parse_known_args(raw_args)<EOL>d = defaultdict(list)<EOL>for k, v in ((k.lstrip('<STR_LIT:->'), v) for k,v in (a.split('<STR_LIT:=>') for a in parsed_unknown_args)):<EOL><INDENT>d[k].append(v)<EOL><DEDENT>for k in (k for k in d if len(d[k])==<NUM_LIT:1>):<EOL><INDENT>d[k] = d[k][<NUM_LIT:0>]<EOL><DEDENT>kwargs.update(d)<EOL><DEDENT>else:<EOL><INDENT>parsed_args = self.parse_args(raw_args)<EOL><DEDENT>kwargs.update(vars(parsed_args))<EOL>for k in arg_info['<STR_LIT>']:<EOL><INDENT>args.append(kwargs.pop(k))<EOL><DEDENT>if arg_info['<STR_LIT:args>']:<EOL><INDENT>args.extend(kwargs.pop(arg_info['<STR_LIT:args>']))<EOL><DEDENT>return args, kwargs<EOL>

This is the method that is called from Script.run(), this is the insertion
        point for parsing all the arguments though on init this will find all args it
        can, so this method pulls already found args from class variables


arg_info = self.arg_info<EOL>main = self.callback<EOL>cbi = CallbackInspect(main)<EOL>all_arg_names = set()<EOL>decorator_args = cbi.args<EOL>args, args_name, kwargs_name, args_defaults = cbi.argspec<EOL>arg_info['<STR_LIT>'] = args<EOL>default_offset = len(args) - len(args_defaults)<EOL>comp_args = set()<EOL>for da in decorator_args:<EOL><INDENT>comp_args.update(da[<NUM_LIT:0>])<EOL><DEDENT>for i, arg_name in enumerate(args):<EOL><INDENT>if arg_name in comp_args: continue<EOL>a = ScriptKwarg(arg_name)<EOL>default_i = i - default_offset<EOL>if default_i >= <NUM_LIT:0>:<EOL><INDENT>na = args_defaults[default_i]<EOL>a.set_default(na)<EOL><DEDENT>a.merge_from_list(decorator_args)<EOL>if a.required:<EOL><INDENT>arg_info['<STR_LIT>'].append(a.name)<EOL><DEDENT>else:<EOL><INDENT>arg_info['<STR_LIT>'][a.name] = a.default<EOL><DEDENT>all_arg_names |= a.parser_args<EOL>if "<STR_LIT:default>" not in a.parser_kwargsand "<STR_LIT:action>" not in a.parser_kwargsand "<STR_LIT>" not in a.parser_kwargs:<EOL><INDENT>keys = self._option_string_actions.keys()<EOL>found_arg = False<EOL>for pa in a.parser_args:<EOL><INDENT>if pa in keys:<EOL><INDENT>found_arg = True<EOL>break<EOL><DEDENT><DEDENT>if not found_arg:<EOL><INDENT>self.add_argument(*a.parser_args, **a.parser_kwargs)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.add_argument(*a.parser_args, **a.parser_kwargs)<EOL><DEDENT><DEDENT>self.unknown_args = False<EOL>if self.add_help:<EOL><INDENT>if args_name:<EOL><INDENT>a = ScriptArg(args_name, nargs='<STR_LIT:*>')<EOL>a.merge_from_list(decorator_args)<EOL>all_arg_names |= a.parser_args<EOL>self.add_argument(*a.parser_args, **a.parser_kwargs)<EOL>arg_info['<STR_LIT:args>'] = args_name<EOL><DEDENT>if kwargs_name:<EOL><INDENT>self.unknown_args = True<EOL>arg_info['<STR_LIT>'] = kwargs_name<EOL><DEDENT><DEDENT>for da, dkw in decorator_args:<EOL><INDENT>if da[<NUM_LIT:0>] not in all_arg_names:<EOL><INDENT>arg_name = da[<NUM_LIT:0>]<EOL>if arg_name.startswith("<STR_LIT:->"):<EOL><INDENT>a = ScriptKwarg(*da)<EOL><DEDENT>else:<EOL><INDENT>a = ScriptArg(*da)<EOL><DEDENT>a.merge_kwargs(dkw)<EOL>self.add_argument(*a.parser_args, **a.parser_kwargs)<EOL><DEDENT><DEDENT>self.arg_info = arg_info<EOL>

when a new parser is created this is the method that is called from its
        __init__ method to find all the arguments


if not self.has_injected_quiet(): return arg_strings<EOL>action = self._option_string_actions.get(self.quiet_flags[<NUM_LIT:0>])<EOL>if action:<EOL><INDENT>count = len(arg_strings)<EOL>new_args = []<EOL>i = <NUM_LIT:0><EOL>while i < count:<EOL><INDENT>arg_string = arg_strings[i]<EOL>if arg_string in action.option_strings:<EOL><INDENT>if (i + <NUM_LIT:1>) < count:<EOL><INDENT>narg_string = arg_strings[i + <NUM_LIT:1>]<EOL>if narg_string in self._option_string_actions:<EOL><INDENT>new_args.append("<STR_LIT>".format(arg_string, action.const))<EOL><DEDENT>elif re.match(r"<STR_LIT>".format(action.const), narg_string):<EOL><INDENT>new_args.append("<STR_LIT>".format(arg_string, narg_string))<EOL>i += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>new_args.append("<STR_LIT>".format(arg_string, action.const))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>new_args.append("<STR_LIT>".format(arg_string, action.const))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>new_args.append(arg_string)<EOL><DEDENT>i += <NUM_LIT:1><EOL><DEDENT>arg_strings = new_args<EOL><DEDENT>return arg_strings<EOL>

This is a hack to allow `--quiet` and `--quiet DI` to work correctly,
        basically it goes through all arg_strings and if it finds --quiet it checks
        the next argument to see if it is some combination of DIWEC, if it is then
        it combines it to `--quiet=ARG` and returns the modified arg_strings list

        :param arg_strings: list, the raw arguments
        :returns: list, the arg_strings changed if needed


d = defaultdict(list)<EOL>i = <NUM_LIT:0><EOL>length = len(args)<EOL>while i < length:<EOL><INDENT>if args[i].startswith("<STR_LIT:->"):<EOL><INDENT>s = args[i].lstrip("<STR_LIT:->")<EOL>bits = s.split("<STR_LIT:=>", <NUM_LIT:1>)<EOL>if len(bits) > <NUM_LIT:1>:<EOL><INDENT>key = bits[<NUM_LIT:0>]<EOL>val = bits[<NUM_LIT:1>].strip("<STR_LIT>")<EOL>d[key].append(val)<EOL><DEDENT>else:<EOL><INDENT>if i + <NUM_LIT:1> < length:<EOL><INDENT>if args[i + <NUM_LIT:1>].startswith("<STR_LIT:->"):<EOL><INDENT>d[s].append(True)<EOL><DEDENT>else:<EOL><INDENT>d[s].append(args[i + <NUM_LIT:1>])<EOL>i += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>d["<STR_LIT:*>"].append(args[i])<EOL><DEDENT>i += <NUM_LIT:1><EOL><DEDENT>super(UnknownParser, self).__init__(d)<EOL>

:param args: list, the list of extra args returned from parse_known_args
:returns: dict, key is the arg name (* for non positional args) and value is
    a list of found arguments (so --foo 1 --foo 2 is supported). The value is
    always a list


def make_buffer_get_request(self, expected_status=<NUM_LIT:200>, expected_count=None, **get_params):

response = self.client.get(<EOL>reverse('<STR_LIT>'),<EOL>get_params,<EOL>format='<STR_LIT>'<EOL>)<EOL>self.assertEqual(response.status_code, expected_status)<EOL>if expected_status == <NUM_LIT:200>:<EOL><INDENT>t_id = get_params.get("<STR_LIT>")<EOL>if expected_count is None:<EOL><INDENT>expected_count = Buffer.objects.filter(transfer_session_id=t_id).count()<EOL><DEDENT>data = json.loads(response.content.decode())<EOL>if isinstance(data, dict) and "<STR_LIT>" in data:<EOL><INDENT>data = data["<STR_LIT>"]<EOL><DEDENT>model_uuids = [d["<STR_LIT>"] for d in data]<EOL>Buffer.objects.filter(transfer_session_id=t_id, model_uuid__in=model_uuids).delete()<EOL>errors = validate_and_create_buffer_data(data, TransferSession.objects.get(id=t_id))<EOL>self.assertFalse(errors)<EOL>self.assertEqual(expected_count, Buffer.objects.filter(transfer_session_id=t_id, model_uuid__in=model_uuids).count())<EOL>self.assertEqual(expected_count, len(data))<EOL>for record in data:<EOL><INDENT>self.assertEqual(<NUM_LIT:3>, len(record["<STR_LIT>"]))<EOL><DEDENT>return data<EOL><DEDENT>

Make a GET request to the buffer endpoint. Warning: Deletes the local buffer instances before validating.


if os.path.isfile(SQLITE_VARIABLE_FILE_CACHE):<EOL><INDENT>return<EOL><DEDENT>conn = sqlite3.connect('<STR_LIT>')<EOL>low = <NUM_LIT:1><EOL>high = <NUM_LIT:1000>  <EOL>conn.execute('<STR_LIT>')<EOL>while low < high - <NUM_LIT:1>:<EOL><INDENT>guess = (low + high) // <NUM_LIT:2><EOL>try:<EOL><INDENT>statement = '<STR_LIT>' % '<STR_LIT:U+002C>'.join(['<STR_LIT:?>' for _ in range(guess)])<EOL>values = [i for i in range(guess)]<EOL>conn.execute(statement, values)<EOL><DEDENT>except sqlite3.DatabaseError as ex:<EOL><INDENT>if '<STR_LIT>' in str(ex):<EOL><INDENT>high = guess<EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>else:<EOL><INDENT>low = guess<EOL><DEDENT><DEDENT>conn.close()<EOL>with open(SQLITE_VARIABLE_FILE_CACHE, '<STR_LIT:w>') as file:<EOL><INDENT>file.write(str(low))<EOL><DEDENT>

SQLite has a limit on the max number of variables allowed for parameter substitution. This limit is usually 999, but
can be compiled to a different number. This function calculates what the max is for the sqlite version running on the device.
We use the calculated value to chunk our SQL bulk insert statements when deserializing from the store to the app layer.


for f in klass_model._meta.concrete_fields:<EOL><INDENT>if f.related_model:<EOL><INDENT>if issubclass(klass_model, f.related_model):<EOL><INDENT>return f.attname<EOL><DEDENT><DEDENT><DEDENT>return None<EOL>

Return whether this model has a self ref FK, and the name for the field


_serialize_into_store(self.profile, filter=filter)<EOL>

Takes data from app layer and serializes the models into the store.


_deserialize_from_store(self.profile)<EOL>

Takes data from the store and integrates into the application.


@receiver(post_delete)<EOL>def add_to_deleted_models(sender, instance=None, *args, **kwargs):

if issubclass(sender, SyncableModel):<EOL><INDENT>instance._update_deleted_models()<EOL><DEDENT>

Whenever a model is deleted, we record its ID in a separate model for tracking purposes. During serialization, we will mark
the model as deleted in the store.


if force_new:<EOL><INDENT>with transaction.atomic():<EOL><INDENT>SharedKey.objects.filter(current=True).update(current=False)<EOL>key = Key()<EOL>return SharedKey.objects.create(public_key=key,<EOL>private_key=key,<EOL>current=True)<EOL><DEDENT><DEDENT>try:<EOL><INDENT>return SharedKey.objects.get(current=True)<EOL><DEDENT>except SharedKey.DoesNotExist:<EOL><INDENT>key = Key()<EOL>return SharedKey.objects.create(public_key=key,<EOL>private_key=key,<EOL>current=True)<EOL><DEDENT>

Create a shared public/private key pair for certificate pushing,
if the settings allow.


return {instance: fsic2.get(instance, <NUM_LIT:0>) for instance, counter in six.iteritems(fsic1) if fsic2.get(instance, <NUM_LIT:0>) < counter}<EOL>

We set the lower counter between two same instance ids.
If an instance_id exists in one fsic but not the other we want to give that counter a value of 0.

:param fsic1: dictionary containing (instance_id, counter) pairs
:param fsic2: dictionary containing (instance_id, counter) pairs
:return ``dict`` of fsics to be used in queueing the correct records to the buffer


<EOL>current_id = InstanceIDModel.get_current_instance_and_increment_counter()<EOL>with transaction.atomic():<EOL><INDENT>prefix_condition = None<EOL>if filter:<EOL><INDENT>prefix_condition = functools.reduce(lambda x, y: x | y, [Q(_morango_partition__startswith=prefix) for prefix in filter])<EOL><DEDENT>syncable_dict = _profile_models[profile]<EOL>for (_, klass_model) in six.iteritems(syncable_dict):<EOL><INDENT>new_store_records = []<EOL>new_rmc_records = []<EOL>klass_queryset = klass_model.objects.filter(_morango_dirty_bit=True)<EOL>if prefix_condition:<EOL><INDENT>klass_queryset = klass_queryset.filter(prefix_condition)<EOL><DEDENT>store_records_dict = Store.objects.in_bulk(id_list=klass_queryset.values_list('<STR_LIT:id>', flat=True))<EOL>for app_model in klass_queryset:<EOL><INDENT>try:<EOL><INDENT>store_model = store_records_dict[app_model.id]<EOL>if store_model.dirty_bit:<EOL><INDENT>store_model.conflicting_serialized_data = store_model.serialized + "<STR_LIT:\n>" + store_model.conflicting_serialized_data<EOL>store_model.dirty_bit = False<EOL><DEDENT>ser_dict = json.loads(store_model.serialized)<EOL>ser_dict.update(app_model.serialize())<EOL>store_model.serialized = DjangoJSONEncoder().encode(ser_dict)<EOL>RecordMaxCounter.objects.update_or_create(defaults={'<STR_LIT>': current_id.counter},<EOL>instance_id=current_id.id,<EOL>store_model_id=store_model.id)<EOL>store_model.last_saved_instance = current_id.id<EOL>store_model.last_saved_counter = current_id.counter<EOL>store_model.deleted = False<EOL>store_model.hard_deleted = False<EOL>store_model.save()<EOL><DEDENT>except KeyError:<EOL><INDENT>kwargs = {<EOL>'<STR_LIT:id>': app_model.id,<EOL>'<STR_LIT>': DjangoJSONEncoder().encode(app_model.serialize()),<EOL>'<STR_LIT>': current_id.id,<EOL>'<STR_LIT>': current_id.counter,<EOL>'<STR_LIT>': app_model.morango_model_name,<EOL>'<STR_LIT>': app_model.morango_profile,<EOL>'<STR_LIT>': app_model._morango_partition,<EOL>'<STR_LIT>': app_model._morango_source_id,<EOL>}<EOL>self_ref_fk = _self_referential_fk(klass_model)<EOL>if self_ref_fk:<EOL><INDENT>self_ref_fk_value = getattr(app_model, self_ref_fk)<EOL>kwargs.update({'<STR_LIT>': self_ref_fk_value or '<STR_LIT>'})<EOL><DEDENT>new_store_records.append(Store(**kwargs))<EOL>new_rmc_records.append(RecordMaxCounter(store_model_id=app_model.id, instance_id=current_id.id, counter=current_id.counter))<EOL><DEDENT><DEDENT>Store.objects.bulk_create(new_store_records)<EOL>RecordMaxCounter.objects.bulk_create(new_rmc_records)<EOL>klass_queryset.update(update_dirty_bit_to=False)<EOL><DEDENT>deleted_ids = DeletedModels.objects.filter(profile=profile).values_list('<STR_LIT:id>', flat=True)<EOL>deleted_store_records = Store.objects.filter(id__in=deleted_ids)<EOL>deleted_store_records.update(dirty_bit=False, deleted=True, last_saved_instance=current_id.id, last_saved_counter=current_id.counter)<EOL>RecordMaxCounter.objects.filter(instance_id=current_id.id, store_model_id__in=deleted_ids).update(counter=current_id.counter)<EOL>new_rmc_ids = deleted_store_records.exclude(recordmaxcounter__instance_id=current_id.id).values_list("<STR_LIT:id>", flat=True)<EOL>RecordMaxCounter.objects.bulk_create([RecordMaxCounter(store_model_id=r_id, instance_id=current_id.id, counter=current_id.counter) for r_id in new_rmc_ids])<EOL>DeletedModels.objects.filter(profile=profile).delete()<EOL>hard_deleted_ids = HardDeletedModels.objects.filter(profile=profile).values_list('<STR_LIT:id>', flat=True)<EOL>hard_deleted_store_records = Store.objects.filter(id__in=hard_deleted_ids)<EOL>hard_deleted_store_records.update(hard_deleted=True, serialized='<STR_LIT:{}>', conflicting_serialized_data='<STR_LIT>')<EOL>HardDeletedModels.objects.filter(profile=profile).delete()<EOL>if not filter:<EOL><INDENT>DatabaseMaxCounter.objects.update_or_create(instance_id=current_id.id, partition="<STR_LIT>", defaults={'<STR_LIT>': current_id.counter})<EOL><DEDENT>else:<EOL><INDENT>for f in filter:<EOL><INDENT>DatabaseMaxCounter.objects.update_or_create(instance_id=current_id.id, partition=f, defaults={'<STR_LIT>': current_id.counter})<EOL><DEDENT><DEDENT><DEDENT>


<EOL>_serialize_into_store(profile)<EOL>fk_cache = {}<EOL>with transaction.atomic():<EOL><INDENT>syncable_dict = _profile_models[profile]<EOL>excluded_list = []<EOL>for model_name, klass_model in six.iteritems(syncable_dict):<EOL><INDENT>self_ref_fk = _self_referential_fk(klass_model)<EOL>query = Q(model_name=klass_model.morango_model_name)<EOL>for klass in klass_model.morango_model_dependencies:<EOL><INDENT>query |= Q(model_name=klass.morango_model_name)<EOL><DEDENT>if self_ref_fk:<EOL><INDENT>clean_parents = Store.objects.filter(dirty_bit=False, profile=profile).filter(query).char_ids_list()<EOL>dirty_children = Store.objects.filter(dirty_bit=True, profile=profile).filter(Q(_self_ref_fk__in=clean_parents) | Q(_self_ref_fk='<STR_LIT>')).filter(query)<EOL>while len(dirty_children) > <NUM_LIT:0>:<EOL><INDENT>for store_model in dirty_children:<EOL><INDENT>try:<EOL><INDENT>app_model = store_model._deserialize_store_model(fk_cache)<EOL>if app_model:<EOL><INDENT>with mute_signals(signals.pre_save, signals.post_save):<EOL><INDENT>app_model.save(update_dirty_bit_to=False)<EOL><DEDENT><DEDENT>store_model.dirty_bit = False<EOL>store_model.save(update_fields=['<STR_LIT>'])<EOL><DEDENT>except exceptions.ValidationError:<EOL><INDENT>excluded_list.append(store_model.id)<EOL><DEDENT><DEDENT>clean_parents = Store.objects.filter(dirty_bit=False, profile=profile).filter(query).char_ids_list()<EOL>dirty_children = Store.objects.filter(dirty_bit=True, profile=profile, _self_ref_fk__in=clean_parents).filter(query)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>db_values = []<EOL>fields = klass_model._meta.fields<EOL>for store_model in Store.objects.filter(model_name=model_name, profile=profile, dirty_bit=True):<EOL><INDENT>try:<EOL><INDENT>app_model = store_model._deserialize_store_model(fk_cache)<EOL>if app_model:<EOL><INDENT>for f in fields:<EOL><INDENT>value = getattr(app_model, f.attname)<EOL>db_value = f.get_db_prep_value(value, connection)<EOL>db_values.append(db_value)<EOL><DEDENT><DEDENT><DEDENT>except exceptions.ValidationError:<EOL><INDENT>excluded_list.append(store_model.id)<EOL><DEDENT><DEDENT>if db_values:<EOL><INDENT>num_of_rows = len(db_values) // len(fields)<EOL>placeholder_tuple = tuple(['<STR_LIT:%s>' for _ in range(len(fields))])<EOL>placeholder_list = [str(placeholder_tuple) for _ in range(num_of_rows)]<EOL>with connection.cursor() as cursor:<EOL><INDENT>DBBackend._bulk_insert_into_app_models(cursor, klass_model._meta.db_table, fields, db_values, placeholder_list)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>Store.objects.exclude(id__in=excluded_list).filter(profile=profile, dirty_bit=True).update(dirty_bit=False)<EOL><DEDENT>


last_saved_by_conditions = []<EOL>filter_prefixes = Filter(transfersession.filter)<EOL>server_fsic = json.loads(transfersession.server_fsic)<EOL>client_fsic = json.loads(transfersession.client_fsic)<EOL>if transfersession.push:<EOL><INDENT>fsics = _fsic_queuing_calc(client_fsic, server_fsic)<EOL><DEDENT>else:<EOL><INDENT>fsics = _fsic_queuing_calc(server_fsic, client_fsic)<EOL><DEDENT>if not fsics:<EOL><INDENT>return<EOL><DEDENT>for instance, counter in six.iteritems(fsics):<EOL><INDENT>last_saved_by_conditions += ["<STR_LIT>".format(instance, counter)]<EOL><DEDENT>if fsics:<EOL><INDENT>last_saved_by_conditions = [_join_with_logical_operator(last_saved_by_conditions, '<STR_LIT>')]<EOL><DEDENT>partition_conditions = []<EOL>for prefix in filter_prefixes:<EOL><INDENT>partition_conditions += ["<STR_LIT>".format(prefix)]<EOL><DEDENT>if filter_prefixes:<EOL><INDENT>partition_conditions = [_join_with_logical_operator(partition_conditions, '<STR_LIT>')]<EOL><DEDENT>fsic_and_partition_conditions = _join_with_logical_operator(last_saved_by_conditions + partition_conditions, '<STR_LIT>')<EOL>where_condition = _join_with_logical_operator([fsic_and_partition_conditions, "<STR_LIT>".format(transfersession.sync_session.profile)], '<STR_LIT>')<EOL>with connection.cursor() as cursor:<EOL><INDENT>queue_buffer = """<STR_LIT>""".format(outgoing_buffer=Buffer._meta.db_table,<EOL>transfer_session_id=transfersession.id,<EOL>condition=where_condition,<EOL>store=Store._meta.db_table)<EOL>cursor.execute(queue_buffer)<EOL>queue_rmc_buffer = """<STR_LIT>""".format(outgoing_rmcb=RecordMaxCounterBuffer._meta.db_table,<EOL>transfer_session_id=transfersession.id,<EOL>record_max_counter=RecordMaxCounter._meta.db_table,<EOL>outgoing_buffer=Buffer._meta.db_table)<EOL>cursor.execute(queue_rmc_buffer)<EOL><DEDENT>

Takes a chunk of data from the store to be put into the buffer to be sent to another morango instance.


with connection.cursor() as cursor:<EOL><INDENT>DBBackend._dequeuing_delete_rmcb_records(cursor, transfersession.id)<EOL>DBBackend._dequeuing_delete_buffered_records(cursor, transfersession.id)<EOL>current_id = InstanceIDModel.get_current_instance_and_increment_counter()<EOL>DBBackend._dequeuing_merge_conflict_buffer(cursor, current_id, transfersession.id)<EOL>DBBackend._dequeuing_merge_conflict_rmcb(cursor, transfersession.id)<EOL>DBBackend._dequeuing_update_rmcs_last_saved_by(cursor, current_id, transfersession.id)<EOL>DBBackend._dequeuing_delete_mc_rmcb(cursor, transfersession.id)<EOL>DBBackend._dequeuing_delete_mc_buffer(cursor, transfersession.id)<EOL>DBBackend._dequeuing_insert_remaining_buffer(cursor, transfersession.id)<EOL>DBBackend._dequeuing_insert_remaining_rmcb(cursor, transfersession.id)<EOL>DBBackend._dequeuing_delete_remaining_rmcb(cursor, transfersession.id)<EOL>DBBackend._dequeuing_delete_remaining_buffer(cursor, transfersession.id)<EOL><DEDENT>if getattr(settings, '<STR_LIT>', True):<EOL><INDENT>_deserialize_from_store(transfersession.sync_session.profile)<EOL><DEDENT>

Takes data from the buffers and merges into the store and record max counters.


if not isinstance(data, int) and not isinstance(data, long):<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>res = []<EOL>while data > <NUM_LIT:0> or not res:<EOL><INDENT>for j in range(<NUM_LIT:5>):<EOL><INDENT>if not j % <NUM_LIT:2>:<EOL><INDENT>res += CONSONANTS[(data & <NUM_LIT>)]<EOL>data >>= <NUM_LIT:4><EOL><DEDENT>else:<EOL><INDENT>res += VOWELS[(data & <NUM_LIT>)]<EOL>data >>= <NUM_LIT:2><EOL><DEDENT><DEDENT>if data > <NUM_LIT:0>:<EOL><INDENT>res += '<STR_LIT:->'<EOL><DEDENT><DEDENT>res.reverse()<EOL>return '<STR_LIT>'.join(res)<EOL>

:params data: integer
:returns: proquint made from input data
:type data: int
:rtype: string


if not isinstance(data, basestring):<EOL><INDENT>raise TypeError('<STR_LIT>')<EOL><DEDENT>res = <NUM_LIT:0><EOL>for part in data.split('<STR_LIT:->'):<EOL><INDENT>if len(part) != <NUM_LIT:5>:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>for j in range(<NUM_LIT:5>):<EOL><INDENT>try:<EOL><INDENT>if not j % <NUM_LIT:2>:<EOL><INDENT>res <<= <NUM_LIT:4><EOL>res |= CONSONANTS.index(part[j])<EOL><DEDENT>else:<EOL><INDENT>res <<= <NUM_LIT:2><EOL>res |= VOWELS.index(part[j])<EOL><DEDENT><DEDENT>except ValueError:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(part[j]))<EOL><DEDENT><DEDENT><DEDENT>return res<EOL>

:params data: proquint
:returns: proquint decoded into an integer
:type data: string
:rtype: int


return from_int(int(uuid.uuid4().hex[:<NUM_LIT:8>], <NUM_LIT:16>)).replace('<STR_LIT:->', '<STR_LIT>')<EOL>


self_fk = []<EOL>for f in class_model._meta.concrete_fields:<EOL><INDENT>if f.related_model in self_fk:<EOL><INDENT>return True<EOL><DEDENT>if f.related_model == class_model:<EOL><INDENT>self_fk.append(class_model)<EOL><DEDENT><DEDENT>return False<EOL>

We check whether a class has more than 1 FK reference to itself.


import django.apps<EOL>from morango.models import SyncableModel<EOL>from morango.manager import SyncableModelManager<EOL>from morango.query import SyncableModelQuerySet<EOL>model_list = []<EOL>for model_class in django.apps.apps.get_models():<EOL><INDENT>if issubclass(model_class, SyncableModel):<EOL><INDENT>name = model_class.__name__<EOL>if _multiple_self_ref_fk_check(model_class):<EOL><INDENT>raise InvalidMorangoModelConfiguration("<STR_LIT>")<EOL><DEDENT>try:<EOL><INDENT>from mptt import models<EOL>from morango.utils.morango_mptt import MorangoMPTTModel, MorangoMPTTTreeManager, MorangoTreeQuerySet<EOL>if issubclass(model_class, models.MPTTModel):<EOL><INDENT>if not issubclass(model_class, MorangoMPTTModel):<EOL><INDENT>raise InvalidMorangoModelConfiguration("<STR_LIT>".format(name))<EOL><DEDENT>if not isinstance(model_class.objects, MorangoMPTTTreeManager):<EOL><INDENT>raise InvalidMPTTManager("<STR_LIT>".format(name))<EOL><DEDENT>if not isinstance(model_class.objects.none(), MorangoTreeQuerySet):<EOL><INDENT>raise InvalidMPTTQuerySet("<STR_LIT>".format(name))<EOL><DEDENT><DEDENT><DEDENT>except ImportError:<EOL><INDENT>pass<EOL><DEDENT>if not isinstance(model_class.objects, SyncableModelManager):<EOL><INDENT>raise InvalidSyncableManager("<STR_LIT>".format(name))<EOL><DEDENT>if not isinstance(model_class.objects.none(), SyncableModelQuerySet):<EOL><INDENT>raise InvalidSyncableQueryset("<STR_LIT>".format(name))<EOL><DEDENT>if model_class._meta.many_to_many:<EOL><INDENT>raise UnsupportedFieldType("<STR_LIT>")<EOL><DEDENT>if not hasattr(model_class, '<STR_LIT>'):<EOL><INDENT>raise InvalidMorangoModelConfiguration("<STR_LIT>".format(name))<EOL><DEDENT>if not hasattr(model_class, '<STR_LIT>'):<EOL><INDENT>raise InvalidMorangoModelConfiguration("<STR_LIT>".format(name))<EOL><DEDENT>profile = model_class.morango_profile<EOL>_profile_models[profile] = _profile_models.get(profile, [])<EOL>if model_class.morango_model_name is not None:<EOL><INDENT>_insert_model_into_profile_dict(model_class, profile)<EOL><DEDENT><DEDENT><DEDENT>for profile, model_list in iteritems(_profile_models):<EOL><INDENT>syncable_models_dict = OrderedDict()<EOL>for model_class in model_list:<EOL><INDENT>syncable_models_dict[model_class.morango_model_name] = model_class<EOL><DEDENT>_profile_models[profile] = syncable_models_dict<EOL><DEDENT>

Per profile, adds each model to a dictionary mapping the morango model name to its model class.
We sort by ForeignKey dependencies to safely sync data.