def configure_ci_jobs(<EOL>config_url, rosdistro_name, ci_build_name,<EOL>groovy_script=None, dry_run=False):

config = get_config_index(config_url)<EOL>build_files = get_ci_build_files(config, rosdistro_name)<EOL>build_file = build_files[ci_build_name]<EOL>index = get_index(config.rosdistro_index_url)<EOL>targets = []<EOL>for os_name in build_file.targets.keys():<EOL><INDENT>for os_code_name in build_file.targets[os_name].keys():<EOL><INDENT>for arch in build_file.targets[os_name][os_code_name]:<EOL><INDENT>targets.append((os_name, os_code_name, arch))<EOL><DEDENT><DEDENT><DEDENT>print('<STR_LIT>')<EOL>for os_name, os_code_name, arch in targets:<EOL><INDENT>print('<STR_LIT>', os_name, os_code_name, arch)<EOL><DEDENT>dist_file = get_distribution_file(index, rosdistro_name, build_file)<EOL>if not dist_file:<EOL><INDENT>print('<STR_LIT>')<EOL>return<EOL><DEDENT>ci_view_name = get_ci_view_name(rosdistro_name)<EOL>from ros_buildfarm.jenkins import connect<EOL>jenkins = connect(config.jenkins_url) if groovy_script is None else False<EOL>view_configs = {}<EOL>views = {<EOL>ci_view_name: configure_ci_view(<EOL>jenkins, ci_view_name, dry_run=dry_run)<EOL>}<EOL>if not jenkins:<EOL><INDENT>view_configs.update(views)<EOL><DEDENT>groovy_data = {<EOL>'<STR_LIT>': dry_run,<EOL>'<STR_LIT>': len(view_configs),<EOL>}<EOL>ci_job_names = []<EOL>job_configs = OrderedDict()<EOL>is_disabled = False<EOL>for os_name, os_code_name, arch in targets:<EOL><INDENT>try:<EOL><INDENT>job_name, job_config = configure_ci_job(<EOL>config_url, rosdistro_name, ci_build_name,<EOL>os_name, os_code_name, arch,<EOL>config=config, build_file=build_file,<EOL>index=index, dist_file=dist_file,<EOL>jenkins=jenkins, views=views,<EOL>is_disabled=is_disabled,<EOL>groovy_script=groovy_script,<EOL>dry_run=dry_run,<EOL>trigger_timer=build_file.jenkins_job_schedule)<EOL>ci_job_names.append(job_name)<EOL>if groovy_script is not None:<EOL><INDENT>print("<STR_LIT>" % job_name)<EOL>job_configs[job_name] = job_config<EOL><DEDENT><DEDENT>except JobValidationError as e:<EOL><INDENT>print(e.message, file=sys.stderr)<EOL><DEDENT><DEDENT>groovy_data['<STR_LIT>'] = len(job_configs)<EOL>groovy_data['<STR_LIT>'] = {}<EOL>if groovy_script is not None:<EOL><INDENT>print(<EOL>"<STR_LIT>" %<EOL>(groovy_script, len(job_configs)))<EOL>content = expand_template(<EOL>'<STR_LIT>', groovy_data)<EOL>write_groovy_script_and_configs(<EOL>groovy_script, content, job_configs, view_configs)<EOL><DEDENT>


def configure_doc_job(<EOL>config_url, rosdistro_name, doc_build_name,<EOL>repo_name, os_name, os_code_name, arch,<EOL>config=None, build_file=None,<EOL>index=None, dist_file=None, dist_cache=None,<EOL>jenkins=None, views=None,<EOL>is_disabled=False,<EOL>groovy_script=None,<EOL>doc_repository=None,<EOL>dry_run=False):

if config is None:<EOL><INDENT>config = get_config_index(config_url)<EOL><DEDENT>if build_file is None:<EOL><INDENT>build_files = get_doc_build_files(config, rosdistro_name)<EOL>build_file = build_files[doc_build_name]<EOL><DEDENT>if index is None:<EOL><INDENT>index = get_index(config.rosdistro_index_url)<EOL><DEDENT>if dist_file is None:<EOL><INDENT>dist_file = get_distribution_file(index, rosdistro_name, build_file)<EOL>if not dist_file:<EOL><INDENT>raise JobValidationError(<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT>repo_names = dist_file.repositories.keys()<EOL>if repo_name is not None:<EOL><INDENT>if repo_name not in repo_names:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % repo_name +<EOL>'<STR_LIT>' %<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(repo_names)))<EOL><DEDENT>repo = dist_file.repositories[repo_name]<EOL>if not repo.doc_repository:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % repo_name)<EOL><DEDENT>if not repo.doc_repository.version:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % repo_name)<EOL><DEDENT>doc_repository = repo.doc_repository<EOL><DEDENT>if os_name not in build_file.targets.keys():<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % os_name +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(build_file.targets.keys())))<EOL><DEDENT>if os_code_name not in build_file.targets[os_name].keys():<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % os_code_name +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(build_file.targets[os_name].keys())))<EOL><DEDENT>if arch not in build_file.targets[os_name][os_code_name]:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % arch +<EOL>'<STR_LIT>' % '<STR_LIT:U+002CU+0020>'.join(sorted(<EOL>build_file.targets[os_name][os_code_name])))<EOL><DEDENT>if dist_cache is None and build_file.notify_maintainers:<EOL><INDENT>dist_cache = get_distribution_cache(index, rosdistro_name)<EOL><DEDENT>if jenkins is None:<EOL><INDENT>from ros_buildfarm.jenkins import connect<EOL>jenkins = connect(config.jenkins_url)<EOL><DEDENT>if views is None:<EOL><INDENT>view_name = get_doc_view_name(<EOL>rosdistro_name, doc_build_name)<EOL>configure_doc_view(jenkins, view_name, dry_run=dry_run)<EOL><DEDENT>job_name = get_doc_job_name(<EOL>rosdistro_name, doc_build_name,<EOL>repo_name, os_name, os_code_name, arch)<EOL>job_config = _get_doc_job_config(<EOL>config, config_url, rosdistro_name, doc_build_name,<EOL>build_file, os_name, os_code_name, arch, doc_repository,<EOL>repo_name, dist_cache=dist_cache, is_disabled=is_disabled)<EOL>if isinstance(jenkins, object) and jenkins is not False:<EOL><INDENT>from ros_buildfarm.jenkins import configure_job<EOL>configure_job(jenkins, job_name, job_config, dry_run=dry_run)<EOL><DEDENT>return job_name, job_config<EOL>

Configure a single Jenkins doc job.

This includes the following steps:
- clone the doc repository to use
- clone the ros_buildfarm repository
- write the distribution repository keys into files
- invoke the run_doc_job.py script


def configure_doc_jobs(<EOL>config_url, rosdistro_name, doc_build_name, groovy_script=None,<EOL>dry_run=False, whitelist_repository_names=None):

config = get_config_index(config_url)<EOL>build_files = get_doc_build_files(config, rosdistro_name)<EOL>build_file = build_files[doc_build_name]<EOL>index = get_index(config.rosdistro_index_url)<EOL>dist_cache = None<EOL>if build_file.notify_maintainers:<EOL><INDENT>dist_cache = get_distribution_cache(index, rosdistro_name)<EOL><DEDENT>targets = []<EOL>for os_name in build_file.targets.keys():<EOL><INDENT>for os_code_name in build_file.targets[os_name].keys():<EOL><INDENT>for arch in build_file.targets[os_name][os_code_name]:<EOL><INDENT>targets.append((os_name, os_code_name, arch))<EOL><DEDENT><DEDENT><DEDENT>print('<STR_LIT>')<EOL>for os_name, os_code_name, arch in targets:<EOL><INDENT>print('<STR_LIT>', os_name, os_code_name, arch)<EOL><DEDENT>dist_file = get_distribution_file(index, rosdistro_name, build_file)<EOL>if not dist_file:<EOL><INDENT>print('<STR_LIT>')<EOL>return<EOL><DEDENT>doc_view_name = get_doc_view_name(rosdistro_name, doc_build_name)<EOL>from ros_buildfarm.jenkins import connect<EOL>jenkins = connect(config.jenkins_url) if groovy_script is None else False<EOL>view_configs = {}<EOL>views = {}<EOL>views[doc_view_name] = configure_doc_view(<EOL>jenkins, doc_view_name, dry_run=dry_run)<EOL>if not jenkins:<EOL><INDENT>view_configs.update(views)<EOL><DEDENT>groovy_data = {<EOL>'<STR_LIT>': dry_run,<EOL>'<STR_LIT>': len(view_configs),<EOL>}<EOL>repo_names = dist_file.repositories.keys()<EOL>filtered_repo_names = build_file.filter_repositories(repo_names)<EOL>job_names = []<EOL>job_configs = OrderedDict()<EOL>for repo_name in sorted(repo_names):<EOL><INDENT>if whitelist_repository_names:<EOL><INDENT>if repo_name not in whitelist_repository_names:<EOL><INDENT>print(<EOL>"<STR_LIT>" %<EOL>repo_name, file=sys.stderr)<EOL>continue<EOL><DEDENT><DEDENT>is_disabled = repo_name not in filtered_repo_names<EOL>if is_disabled and build_file.skip_ignored_repositories:<EOL><INDENT>print("<STR_LIT>" % repo_name,<EOL>file=sys.stderr)<EOL>continue<EOL><DEDENT>repo = dist_file.repositories[repo_name]<EOL>if not repo.doc_repository:<EOL><INDENT>print("<STR_LIT>" % repo_name)<EOL>continue<EOL><DEDENT>if not repo.doc_repository.version:<EOL><INDENT>print("<STR_LIT>" % repo_name)<EOL>continue<EOL><DEDENT>for os_name, os_code_name, arch in targets:<EOL><INDENT>try:<EOL><INDENT>job_name, job_config = configure_doc_job(<EOL>config_url, rosdistro_name, doc_build_name,<EOL>repo_name, os_name, os_code_name, arch,<EOL>config=config, build_file=build_file,<EOL>index=index, dist_file=dist_file,<EOL>dist_cache=dist_cache, jenkins=jenkins, views=views,<EOL>is_disabled=is_disabled,<EOL>groovy_script=groovy_script,<EOL>dry_run=dry_run)<EOL>job_names.append(job_name)<EOL>if groovy_script is not None:<EOL><INDENT>print("<STR_LIT>" % job_name)<EOL>job_configs[job_name] = job_config<EOL><DEDENT><DEDENT>except JobValidationError as e:<EOL><INDENT>print(e.message, file=sys.stderr)<EOL><DEDENT><DEDENT><DEDENT>groovy_data['<STR_LIT>'] = len(job_configs)<EOL>groovy_data['<STR_LIT>'] = {}<EOL>job_prefix = '<STR_LIT>' % doc_view_name<EOL>if not whitelist_repository_names:<EOL><INDENT>groovy_data['<STR_LIT>']['<STR_LIT>'] = (job_prefix, job_names)<EOL>if groovy_script is None:<EOL><INDENT>from ros_buildfarm.jenkins import remove_jobs<EOL>print('<STR_LIT>')<EOL>remove_jobs(jenkins, job_prefix, job_names, dry_run=dry_run)<EOL><DEDENT><DEDENT>if groovy_script is not None:<EOL><INDENT>print(<EOL>"<STR_LIT>" %<EOL>(groovy_script, len(view_configs), len(job_configs)))<EOL>content = expand_template(<EOL>'<STR_LIT>', groovy_data)<EOL>write_groovy_script_and_configs(<EOL>groovy_script, content, job_configs, view_configs=view_configs)<EOL><DEDENT>

Configure all Jenkins doc jobs.

L{configure_doc_job} will be invoked for doc repository and target
which matches the build file criteria.


def get_version_status(<EOL>package_descriptors, targets, repos_data,<EOL>strip_version=False, strip_os_code_name=False):

status = {}<EOL>for package_descriptor in package_descriptors.values():<EOL><INDENT>pkg_name = package_descriptor.pkg_name<EOL>debian_pkg_name = package_descriptor.debian_pkg_name<EOL>ref_version = package_descriptor.version<EOL>if strip_version:<EOL><INDENT>ref_version = _strip_version_suffix(ref_version)<EOL><DEDENT>status[pkg_name] = {}<EOL>for target in targets:<EOL><INDENT>statuses = []<EOL>for repo_data in repos_data:<EOL><INDENT>version = repo_data.get(target, {}).get(debian_pkg_name, None)<EOL>if strip_version:<EOL><INDENT>version = _strip_version_suffix(version)<EOL><DEDENT>if strip_os_code_name:<EOL><INDENT>version = _strip_os_code_name_suffix(<EOL>version, target.os_code_name)<EOL><DEDENT>if ref_version:<EOL><INDENT>if not version:<EOL><INDENT>statuses.append('<STR_LIT>')<EOL><DEDENT>elif version.startswith(ref_version):  <EOL><INDENT>statuses.append('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>if _version_is_gt_other(version, ref_version):<EOL><INDENT>statuses.append('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>statuses.append('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if not version:<EOL><INDENT>statuses.append('<STR_LIT:ignore>')<EOL><DEDENT>else:<EOL><INDENT>statuses.append('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>status[pkg_name][target] = statuses<EOL><DEDENT><DEDENT>return status<EOL>

For each package and target check if it is affected by a sync.

This is the case when the package version in the testing repo is different
from the version in the main repo.

:return: a dict indexed by package names containing
  dicts indexed by targets containing
  a list of status strings (one for each repo)


def get_package_counts(package_descriptors, targets, repos_data):

counts = {}<EOL>for target in targets:<EOL><INDENT>counts[target] = [<NUM_LIT:0>] * len(repos_data)<EOL><DEDENT>for package_descriptor in package_descriptors.values():<EOL><INDENT>debian_pkg_name = package_descriptor.debian_pkg_name<EOL>for target in targets:<EOL><INDENT>for i, repo_data in enumerate(repos_data):<EOL><INDENT>version = repo_data.get(target, {}).get(debian_pkg_name, None)<EOL>if version:<EOL><INDENT>counts[target][i] += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>return counts<EOL>

Get the number of packages per target and repository.

:return: a dict indexed by targets containing
  a list of integer values (one for each repo)


def configure_devel_job(<EOL>config_url, rosdistro_name, source_build_name,<EOL>repo_name, os_name, os_code_name, arch,<EOL>pull_request=False,<EOL>config=None, build_file=None,<EOL>index=None, dist_file=None, dist_cache=None,<EOL>jenkins=None, views=None,<EOL>is_disabled=False,<EOL>groovy_script=None,<EOL>source_repository=None,<EOL>build_targets=None,<EOL>dry_run=False):

if config is None:<EOL><INDENT>config = get_config_index(config_url)<EOL><DEDENT>if build_file is None:<EOL><INDENT>build_files = get_source_build_files(config, rosdistro_name)<EOL>build_file = build_files[source_build_name]<EOL><DEDENT>if build_targets is not None:<EOL><INDENT>build_file.targets = build_targets<EOL><DEDENT>if index is None:<EOL><INDENT>index = get_index(config.rosdistro_index_url)<EOL><DEDENT>if dist_file is None:<EOL><INDENT>dist_file = get_distribution_file(index, rosdistro_name, build_file)<EOL>if not dist_file:<EOL><INDENT>raise JobValidationError(<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT>repo_names = dist_file.repositories.keys()<EOL>if repo_name is not None:<EOL><INDENT>if repo_name not in repo_names:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % repo_name +<EOL>'<STR_LIT>' %<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(repo_names)))<EOL><DEDENT>repo = dist_file.repositories[repo_name]<EOL>if not repo.source_repository:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % repo_name)<EOL><DEDENT>if not repo.source_repository.version:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % repo_name)<EOL><DEDENT>source_repository = repo.source_repository<EOL><DEDENT>if os_name not in build_file.targets.keys():<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % os_name +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(build_file.targets.keys())))<EOL><DEDENT>if os_code_name not in build_file.targets[os_name].keys():<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % os_code_name +<EOL>'<STR_LIT>' +<EOL>'<STR_LIT:U+002CU+0020>'.join(sorted(build_file.targets[os_name].keys())))<EOL><DEDENT>if arch not in build_file.targets[os_name][os_code_name]:<EOL><INDENT>raise JobValidationError(<EOL>"<STR_LIT>" % arch +<EOL>'<STR_LIT>' % '<STR_LIT:U+002CU+0020>'.join(sorted(<EOL>build_file.targets[os_name][os_code_name])))<EOL><DEDENT>if dist_cache is None and build_file.notify_maintainers:<EOL><INDENT>dist_cache = get_distribution_cache(index, rosdistro_name)<EOL><DEDENT>if jenkins is None:<EOL><INDENT>from ros_buildfarm.jenkins import connect<EOL>jenkins = connect(config.jenkins_url)<EOL><DEDENT>if views is None:<EOL><INDENT>view_name = get_devel_view_name(<EOL>rosdistro_name, source_build_name, pull_request=pull_request)<EOL>configure_devel_view(jenkins, view_name, dry_run=dry_run)<EOL><DEDENT>job_name = get_devel_job_name(<EOL>rosdistro_name, source_build_name,<EOL>repo_name, os_name, os_code_name, arch, pull_request)<EOL>job_config = _get_devel_job_config(<EOL>index, config, rosdistro_name, source_build_name,<EOL>build_file, os_name, os_code_name, arch, source_repository,<EOL>repo_name, pull_request, job_name, dist_cache=dist_cache,<EOL>is_disabled=is_disabled)<EOL>if isinstance(jenkins, object) and jenkins is not False:<EOL><INDENT>from ros_buildfarm.jenkins import configure_job<EOL>configure_job(jenkins, job_name, job_config, dry_run=dry_run)<EOL><DEDENT>return job_name, job_config<EOL>

Configure a single Jenkins devel job.

This includes the following steps:
- clone the source repository to use
- clone the ros_buildfarm repository
- write the distribution repository keys into files
- invoke the release/run_devel_job.py script


if not isinstance(bundle, BlueprintBundle):<EOL><INDENT>raise IncompatibleBundle('<STR_LIT>'<EOL>.format(BlueprintBundle))<EOL><DEDENT>elif len(bundle.blueprints) == <NUM_LIT:0>:<EOL><INDENT>raise MissingBlueprints("<STR_LIT>")<EOL><DEDENT>elif self._bundle_exists(bundle.path):<EOL><INDENT>raise ConflictingPath("<STR_LIT>".format(bundle.path))<EOL><DEDENT>elif self._journey_path == bundle.path == '<STR_LIT:/>':<EOL><INDENT>raise ConflictingPath("<STR_LIT>".format(bundle.path))<EOL><DEDENT>self._attached_bundles.append(bundle)<EOL>

Attaches a bundle object

        :param bundle: :class:`flask_journey.BlueprintBundle` object
        :raises:
            - IncompatibleBundle if the bundle is not of type `BlueprintBundle`
            - ConflictingPath if a bundle already exists at bundle.path
            - MissingBlueprints if the bundle doesn't contain any blueprints


if len(self._attached_bundles) == <NUM_LIT:0>:<EOL><INDENT>raise NoBundlesAttached("<STR_LIT>")<EOL><DEDENT>for bundle in self._attached_bundles:<EOL><INDENT>processed_bundle = {<EOL>'<STR_LIT:path>': bundle.path,<EOL>'<STR_LIT:description>': bundle.description,<EOL>'<STR_LIT>': []<EOL>}<EOL>for (bp, description) in bundle.blueprints:<EOL><INDENT>blueprint = self._register_blueprint(app, bp, bundle.path,<EOL>self.get_bp_path(bp), description)<EOL>processed_bundle['<STR_LIT>'].append(blueprint)<EOL><DEDENT>self._registered_bundles.append(processed_bundle)<EOL><DEDENT>

Initializes Journey extension

        :param app: App passed from constructor or directly to init_app
        :raises:
            - NoBundlesAttached if no bundles has been attached attached


routes = []<EOL>for bundle in self._registered_bundles:<EOL><INDENT>bundle_path = bundle['<STR_LIT:path>']<EOL>for blueprint in bundle['<STR_LIT>']:<EOL><INDENT>bp_path = blueprint['<STR_LIT:path>']<EOL>for child in blueprint['<STR_LIT>']:<EOL><INDENT>routes.append(<EOL>(<EOL>child['<STR_LIT>'],<EOL>bundle_path + bp_path + child['<STR_LIT:path>'],<EOL>child['<STR_LIT>']<EOL>)<EOL>)<EOL><DEDENT><DEDENT><DEDENT>return routes<EOL>

Returns simple info about registered blueprints

        :return: Tuple containing endpoint, path and allowed methods for each route


routes = []<EOL>for child in app.url_map.iter_rules():<EOL><INDENT>if child.rule.startswith(base_path):<EOL><INDENT>relative_path = child.rule[len(base_path):]<EOL>routes.append({<EOL>'<STR_LIT:path>': relative_path,<EOL>'<STR_LIT>': child.endpoint,<EOL>'<STR_LIT>': list(child.methods)<EOL>})<EOL><DEDENT><DEDENT>return routes<EOL>

Returns detailed information about registered blueprint routes matching the `BlueprintBundle` path

        :param app: App instance to obtain rules from
        :param base_path: Base path to return detailed route info for
        :return: List of route detail dicts


if obj is not None and not isinstance(obj, Schema):<EOL><INDENT>raise IncompatibleSchema('<STR_LIT>'.format(Schema))<EOL><DEDENT>return obj<EOL>

Ensures the passed schema instance is compatible

    :param obj: object to validate
    :return: obj
    :raises:
        - IncompatibleSchema if the passed schema is of an incompatible type


compressed_bits = cStringIO.StringIO()<EOL>f = gzip.GzipFile(fileobj=compressed_bits, mode='<STR_LIT:wb>')<EOL>f.write(input_str)<EOL>f.close()<EOL>return compressed_bits.getvalue()<EOL>

Compress the log message in order to send less bytes to the wire.


try:<EOL><INDENT>doc = yaml.load(file(filename).read())<EOL>project_id = doc["<STR_LIT>"]["<STR_LIT>"]<EOL>access_token = doc["<STR_LIT>"]["<STR_LIT>"]<EOL>api_domain = doc["<STR_LIT>"]["<STR_LIT>"]<EOL>return project_id, access_token, api_domain<EOL><DEDENT>except:<EOL><INDENT>return None, None, None<EOL><DEDENT>

Format for the file is:

     credentials:
         project_id: ...
         access_token: ...
         api_domain: ...

:param filename: The filename to parse
:return: A tuple with:
            - project_id
            - access_token
            - api_domain


args = []<EOL>for k, v in kwargs.iteritems():<EOL><INDENT>if isinstance(v, basestring):<EOL><INDENT>qv = v.encode('<STR_LIT:utf-8>') if isinstance(v, unicode) else v<EOL>args.append('<STR_LIT>' % (k, urllib.quote(qv, SAFE_URL_CARS)))<EOL><DEDENT>elif isinstance(v, collections.Iterable):<EOL><INDENT>for i in v:<EOL><INDENT>qv = i.encode('<STR_LIT:utf-8>') if isinstance(i, unicode) else str(i)<EOL>args.append('<STR_LIT>' % (k, urllib.quote(qv, SAFE_URL_CARS)))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>qv = str(v)<EOL>args.append('<STR_LIT>' % (k, urllib.quote(qv, SAFE_URL_CARS)))<EOL><DEDENT><DEDENT>return '<STR_LIT:&>'.join(args)<EOL>


interface = Auth(**kwargs)<EOL>self.base_url = '<STR_LIT>'<EOL>self.gbdx_connection = interface.gbdx_connection<EOL>self.logger = interface.logger<EOL>

Construct an instance of an AnswerFactory Project

Args:
    **kwargs

Returns:
    An instance of a Project.


self.logger.debug('<STR_LIT>')<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url<EOL>}<EOL>r = self.gbdx_connection.get(url)<EOL>r.raise_for_status()<EOL>return r.json()<EOL>

Retrieves a list of AnswerFactory Recipes

Args:
    None

Returns:
    A list of JSON representations of recipes


self.logger.debug('<STR_LIT>' + recipe_id)<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url, '<STR_LIT>': recipe_id<EOL>}<EOL>r = self.gbdx_connection.delete(url)<EOL>r.raise_for_status()<EOL>

Deletes an AnswerFactory Recipe by id

Args:
     recipe_id: The id of the recipe to delete

Returns:
     Nothing


Construct an instance of an AnswerFactory Recipe

Args:
    **kwargs

Returns:
    An instance of a Recipe.


self.logger.debug('<STR_LIT>' + project_id)<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url, '<STR_LIT>': project_id<EOL>}<EOL>r = self.gbdx_connection.delete(url)<EOL>r.raise_for_status()<EOL>

Deletes a project by id

Args:
     project_id: The project id to delete

Returns:
     Nothing


<EOL>if '<STR_LIT:id>' in recipe and recipe['<STR_LIT:id>'] is not None:<EOL><INDENT>self.logger.debug("<STR_LIT>" + json.dumps(recipe))<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url, '<STR_LIT>': recipe['<STR_LIT:id>']<EOL>}<EOL>r = self.gbdx_connection.put(url, json=recipe)<EOL>try:<EOL><INDENT>r.raise_for_status()<EOL><DEDENT>except:<EOL><INDENT>print(r.text)<EOL>raise<EOL><DEDENT>return recipe['<STR_LIT:id>']<EOL><DEDENT>else:<EOL><INDENT>self.logger.debug("<STR_LIT>" + json.dumps(recipe))<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url<EOL>}<EOL>r = self.gbdx_connection.post(url, json=recipe)<EOL>try:<EOL><INDENT>r.raise_for_status()<EOL><DEDENT>except:<EOL><INDENT>print(r.text)<EOL>raise<EOL><DEDENT>recipe_json = r.json()<EOL>return recipe_json['<STR_LIT:id>']<EOL><DEDENT>

Saves an AnswerFactory Recipe

Args:
    recipe (dict): Dictionary specifying a recipe

Returns:
    AnswerFactory Recipe id


self.logger.debug('<STR_LIT>' + project_id)<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url, '<STR_LIT>': project_id<EOL>}<EOL>r = self.gbdx_connection.get(url)<EOL>r.raise_for_status()<EOL>return r.json()<EOL>

Retrieves an AnswerFactory Project by id

Args:
    project_id

Returns:
    A JSON representation of the project


self.logger.debug('<STR_LIT>' + workflow_id)<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.workflows_url, '<STR_LIT>': workflow_id<EOL>}<EOL>r = self.gbdx_connection.get(url)<EOL>r.raise_for_status()<EOL>return r.json()['<STR_LIT:state>']<EOL>

Checks workflow status.

         Args:
             workflow_id (str): Workflow id.

         Returns:
             Workflow status (str).


def search(self, lookback_h=<NUM_LIT:12>, owner=None, state="<STR_LIT:all>"):

postdata = {<EOL>"<STR_LIT>": lookback_h,<EOL>"<STR_LIT:state>": state<EOL>}<EOL>if owner is not None:<EOL><INDENT>postdata['<STR_LIT>'] = owner<EOL><DEDENT>url = "<STR_LIT>".format(self.base_url)<EOL>headers = {'<STR_LIT:Content-Type>':'<STR_LIT:application/json>'}<EOL>r = self.gbdx_connection.post(url, headers=headers, data=json.dumps(postdata))<EOL>return r.json()<EOL>

Cancels GBDX batch workflow.

         Params:
            lookback_h (int): Look back time in hours.
            owner (str): Workflow owner to search by
            state (str): State to filter by, eg:
                "submitted",
                "scheduled",
                "started",
                "canceled",
                "cancelling",
                "failed",
                "succeeded",
                "timedout",
                "pending",
                "running",
                "complete",
                "waiting",
                "all"

         Returns:
             Batch Workflow status (str).


def get_stdout(self, workflow_id, task_id):

url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.workflows_url, '<STR_LIT>': workflow_id, '<STR_LIT>': task_id<EOL>}<EOL>r = self.gbdx_connection.get(url)<EOL>r.raise_for_status()<EOL>return r.text<EOL>

Get stdout for a particular task.

         Args:
             workflow_id (str): Workflow id.
             task_id (str): Task id.

         Returns:
             Stdout of the task (string).


url = '<STR_LIT>' % self.base_url<EOL>r = requests.get(url) <EOL>try:<EOL><INDENT>return r.json() == "<STR_LIT>"<EOL><DEDENT>except:<EOL><INDENT>return False<EOL><DEDENT>

Check the heartbeat of the ordering API

Args: None

Returns:  True or False


self.logger.debug('<STR_LIT>' + order_id)<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url, '<STR_LIT>': order_id<EOL>}<EOL>r = self.gbdx_connection.get(url)<EOL>r.raise_for_status()<EOL>return r.json().get("<STR_LIT>", {})<EOL>

Checks imagery order status. There can be more than one image per
           order and this function returns the status of all images
           within the order.

           Args:
               order_id (str): The id of the order placed.

           Returns:
               List of dictionaries, one per image. Each dictionary consists
               of the keys 'acquisition_id', 'location' and 'state'.


interface = Auth(**kwargs)<EOL>self.base_url = '<STR_LIT>' % interface.root_url<EOL>self.gbdx_connection = interface.gbdx_connection<EOL>self.logger = interface.logger<EOL>

Instantiate the GBDX Ordering Interface

           Returns:
               An instance of the Ordering interface.


<EOL>sat_index = meta['<STR_LIT>'].upper() + "<STR_LIT:_>" + meta['<STR_LIT>'].upper()<EOL>acf = np.asarray(meta['<STR_LIT>'])  <EOL>ebw = np.asarray(meta['<STR_LIT>'])  <EOL>gain = np.asarray(constants.DG_ABSCAL_GAIN[sat_index])<EOL>scale = (acf / ebw) * gain<EOL>offset = np.asarray(constants.DG_ABSCAL_OFFSET[sat_index])<EOL>e_sun_index = meta['<STR_LIT>'].upper() + "<STR_LIT:_>" + meta['<STR_LIT>'].upper()<EOL>e_sun = np.asarray(constants.DG_ESUN[e_sun_index])<EOL>sun = ephem.Sun()<EOL>img_obs = ephem.Observer()<EOL>img_obs.lon = meta['<STR_LIT>'][<NUM_LIT:1>]<EOL>img_obs.lat = meta['<STR_LIT>'][<NUM_LIT:0>]<EOL>img_obs.elevation = meta['<STR_LIT>'][<NUM_LIT:2>]<EOL>img_obs.date = datetime.datetime.fromtimestamp(meta['<STR_LIT>']['<STR_LIT>'] / <NUM_LIT>).strftime(<EOL>'<STR_LIT>')<EOL>sun.compute(img_obs)<EOL>d_es = sun.earth_distance<EOL>theta_s = <NUM_LIT> - float(meta['<STR_LIT>'])<EOL>scale2 = (d_es ** <NUM_LIT:2> * np.pi) / (e_sun * np.cos(np.deg2rad(theta_s)))<EOL>return zip(scale, scale2, offset)<EOL>

Compute (gain, offset) tuples for each band of the specified image metadata


if prj_code in CUSTOM_PRJ:<EOL><INDENT>proj = pyproj.Proj(CUSTOM_PRJ[prj_code])<EOL><DEDENT>else:<EOL><INDENT>proj = pyproj.Proj(init=prj_code)<EOL><DEDENT>return proj<EOL>

Helper method for handling projection codes that are unknown to pyproj

Args:
    prj_code (str): an epsg proj code

Returns:
    projection: a pyproj projection


try:<EOL><INDENT>from IPython.display import Javascript, HTML, display<EOL>from gbdxtools.rda.interface import RDA<EOL>from gbdxtools import Interface<EOL>gbdx = Interface()<EOL><DEDENT>except:<EOL><INDENT>print("<STR_LIT>")<EOL>return<EOL><DEDENT>zoom = kwargs.get("<STR_LIT>", <NUM_LIT:16>)<EOL>bands = kwargs.get("<STR_LIT>")<EOL>if bands is None:<EOL><INDENT>bands = image._rgb_bands<EOL><DEDENT>wgs84_bounds = kwargs.get("<STR_LIT>", list(loads(image.metadata["<STR_LIT:image>"]["<STR_LIT>"]).bounds))<EOL>center = kwargs.get("<STR_LIT>", list(shape(image).centroid.bounds[<NUM_LIT:0>:<NUM_LIT:2>]))<EOL>if image.proj != '<STR_LIT>':<EOL><INDENT>code = image.proj.split('<STR_LIT::>')[<NUM_LIT:1>]<EOL>conn = gbdx.gbdx_connection<EOL>proj_info = conn.get('<STR_LIT>'.format(code)).json()<EOL>tfm = partial(pyproj.transform, pyproj.Proj(init='<STR_LIT>'), pyproj.Proj(init=image.proj))<EOL>bounds = list(ops.transform(tfm, box(*wgs84_bounds)).bounds)<EOL><DEDENT>else:<EOL><INDENT>proj_info = {}<EOL>bounds = wgs84_bounds<EOL><DEDENT>if not image.options.get('<STR_LIT>'):<EOL><INDENT>rda = RDA()<EOL>dra = rda.HistogramDRA(image)<EOL>image = dra.aoi(bbox=image.bounds)<EOL><DEDENT>graph_id = image.rda_id<EOL>node_id = image.rda.graph()['<STR_LIT>'][<NUM_LIT:0>]['<STR_LIT:id>']<EOL>map_id = "<STR_LIT>".format(str(int(time.time())))<EOL>scales = '<STR_LIT:U+002C>'.join(['<STR_LIT:1>'] * len(bands))<EOL>offsets = '<STR_LIT:U+002C>'.join(['<STR_LIT:0>'] * len(bands))<EOL>display(HTML(Template('''<STR_LIT>''').substitute({"<STR_LIT>": map_id})))<EOL>js = Template("""<STR_LIT>""").substitute({<EOL>"<STR_LIT>": map_id,<EOL>"<STR_LIT>": image.proj,<EOL>"<STR_LIT>": json.dumps(proj_info),<EOL>"<STR_LIT>": graph_id,<EOL>"<STR_LIT>": bounds,<EOL>"<STR_LIT>": "<STR_LIT:U+002C>".join(map(str, bands)),<EOL>"<STR_LIT>": node_id,<EOL>"<STR_LIT>": json.dumps(image.metadata["<STR_LIT:image>"]),<EOL>"<STR_LIT>": json.dumps(image.metadata["<STR_LIT>"]),<EOL>"<STR_LIT>": center,<EOL>"<STR_LIT>": zoom,<EOL>"<STR_LIT>": gbdx.gbdx_connection.access_token,<EOL>"<STR_LIT>": scales,<EOL>"<STR_LIT>": offsets,<EOL>"<STR_LIT:url>": VIRTUAL_RDA_URL<EOL>})<EOL>display(Javascript(js))<EOL>

Show a slippy map preview of the image. Requires iPython.

    Args:
        image (image): image object to display
        zoom (int): zoom level to intialize the map, default is 16
        center (list): center coordinates to initialize the map, defaults to center of image
        bands (list): bands of image to display, defaults to the image's default RGB bands


@lru_cache(maxsize=<NUM_LIT>)<EOL>def load_url(url, token, shape=(<NUM_LIT:8>, <NUM_LIT>, <NUM_LIT>)):

_, ext = os.path.splitext(urlparse(url).path)<EOL>success = False<EOL>for i in xrange(MAX_RETRIES):<EOL><INDENT>thread_id = threading.current_thread().ident<EOL>_curl = _curl_pool[thread_id]<EOL>_curl.setopt(_curl.URL, url)<EOL>_curl.setopt(pycurl.NOSIGNAL, <NUM_LIT:1>)<EOL>_curl.setopt(pycurl.HTTPHEADER, ['<STR_LIT>'.format(token)])<EOL>with NamedTemporaryFile(prefix="<STR_LIT>", suffix=ext, delete=False) as temp: <EOL><INDENT>_curl.setopt(_curl.WRITEDATA, temp.file)<EOL>_curl.perform()<EOL>code = _curl.getinfo(pycurl.HTTP_CODE)<EOL>try:<EOL><INDENT>if(code != <NUM_LIT:200>):<EOL><INDENT>raise TypeError("<STR_LIT>".format(url, code))<EOL><DEDENT>temp.file.flush()<EOL>temp.close()<EOL>arr = imread(temp.name)<EOL>if len(arr.shape) == <NUM_LIT:3>:<EOL><INDENT>arr = np.rollaxis(arr, <NUM_LIT:2>, <NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>arr = np.expand_dims(arr, axis=<NUM_LIT:0>)<EOL><DEDENT>success = True<EOL>return arr<EOL><DEDENT>except Exception as e:<EOL><INDENT>_curl.close()<EOL>del _curl_pool[thread_id]<EOL><DEDENT>finally:<EOL><INDENT>temp.close()<EOL>os.remove(temp.name)<EOL><DEDENT><DEDENT><DEDENT>if success is False:<EOL><INDENT>raise TypeError("<STR_LIT>".format(url, code))<EOL><DEDENT>return arr<EOL>

Loads a geotiff url inside a thread and returns as an ndarray


def to_geotiff(arr, path='<STR_LIT>', proj=None, spec=None, bands=None, **kwargs):

assert has_rasterio, "<STR_LIT>" <EOL>try:<EOL><INDENT>img_md = arr.rda.metadata["<STR_LIT:image>"]<EOL>x_size = img_md["<STR_LIT>"]<EOL>y_size = img_md["<STR_LIT>"]<EOL><DEDENT>except (AttributeError, KeyError):<EOL><INDENT>x_size = kwargs.get("<STR_LIT>", <NUM_LIT>)<EOL>y_size = kwargs.get("<STR_LIT>", <NUM_LIT>)<EOL><DEDENT>try:<EOL><INDENT>tfm = kwargs['<STR_LIT>'] if '<STR_LIT>' in kwargs else arr.affine<EOL><DEDENT>except:<EOL><INDENT>tfm = None<EOL><DEDENT>dtype = arr.dtype.name if arr.dtype.name != '<STR_LIT>' else '<STR_LIT>' <EOL>if spec is not None and spec.lower() == '<STR_LIT>':<EOL><INDENT>if bands is None:<EOL><INDENT>bands = arr._rgb_bands<EOL><DEDENT>if not arr.options.get('<STR_LIT>'):<EOL><INDENT>from gbdxtools.rda.interface import RDA<EOL>rda = RDA()<EOL>dra = rda.HistogramDRA(arr)<EOL>arr = dra.aoi(bbox=arr.bounds)<EOL><DEDENT>arr = arr[bands,...].astype(np.uint8)<EOL>dtype = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>if bands is not None:<EOL><INDENT>arr = arr[bands,...]<EOL><DEDENT><DEDENT>meta = {<EOL>'<STR_LIT:width>': arr.shape[<NUM_LIT:2>],<EOL>'<STR_LIT>': arr.shape[<NUM_LIT:1>],<EOL>'<STR_LIT:count>': arr.shape[<NUM_LIT:0>],<EOL>'<STR_LIT>': dtype,<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': tfm<EOL>}<EOL>if proj is not None:<EOL><INDENT>meta["<STR_LIT>"] = {'<STR_LIT>': proj}<EOL><DEDENT>if "<STR_LIT>" in kwargs and kwargs["<STR_LIT>"]:<EOL><INDENT>meta.update(blockxsize=x_size, blockysize=y_size, tiled="<STR_LIT:yes>")<EOL><DEDENT>with rasterio.open(path, "<STR_LIT:w>", **meta) as dst:<EOL><INDENT>writer = rio_writer(dst)<EOL>result = store(arr, writer, compute=False)<EOL>result.compute(scheduler=threaded_get)<EOL><DEDENT>return path<EOL>

Write out a geotiff file of the image

    Args:
        path (str): path to write the geotiff file to, default is ./output.tif
        proj (str): EPSG string of projection to reproject to
        spec (str): if set to 'rgb', write out color-balanced 8-bit RGB tif
        bands (list): list of bands to export. If spec='rgb' will default to RGB bands

    Returns:
        str: path the geotiff was written to


def _histogram_stretch(self, data, **kwargs):

limits = {}<EOL>for x in range(<NUM_LIT:3>):<EOL><INDENT>band = data[:,:,x]<EOL>try:<EOL><INDENT>limits[x] = np.percentile(band, kwargs.get("<STR_LIT>", [<NUM_LIT:0>,<NUM_LIT:100>]))<EOL><DEDENT>except IndexError:<EOL><INDENT>return data<EOL><DEDENT><DEDENT>for x in range(<NUM_LIT:3>):<EOL><INDENT>band = data[:,:,x]<EOL>if <NUM_LIT:0> in band:<EOL><INDENT>band = np.ma.masked_values(band, <NUM_LIT:0>).compressed()<EOL><DEDENT>top = limits[x][<NUM_LIT:1>]<EOL>bottom = limits[x][<NUM_LIT:0>]<EOL>if top != bottom: <EOL><INDENT>data[:,:,x] = (data[:,:,x] - bottom) / float(top - bottom) * <NUM_LIT><EOL><DEDENT><DEDENT>data = np.clip(data, <NUM_LIT:0>, <NUM_LIT:255>).astype("<STR_LIT>")<EOL>if "<STR_LIT>" in kwargs:<EOL><INDENT>invGamma = <NUM_LIT:1.0> / kwargs['<STR_LIT>']<EOL>lut = np.array([((i / <NUM_LIT>) ** invGamma) * <NUM_LIT:255><EOL>for i in np.arange(<NUM_LIT:0>, <NUM_LIT>)]).astype("<STR_LIT>")<EOL>data = np.take(lut, data)<EOL><DEDENT>return data<EOL>

perform a contrast stretch and/or gamma adjustment


if "<STR_LIT>" in kwargs:<EOL><INDENT>use_bands = kwargs["<STR_LIT>"]<EOL>assert len(use_bands) == <NUM_LIT:3>, '<STR_LIT>'<EOL>del kwargs["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>use_bands = self._rgb_bands<EOL><DEDENT>if kwargs.get('<STR_LIT>') == True:<EOL><INDENT>return self.histogram_match(use_bands, **kwargs)<EOL><DEDENT>if "<STR_LIT>" not in kwargs:<EOL><INDENT>if "<STR_LIT>" not in kwargs:<EOL><INDENT>if not self.options.get('<STR_LIT>'):<EOL><INDENT>kwargs['<STR_LIT>'] = [<NUM_LIT:2>,<NUM_LIT>]<EOL><DEDENT><DEDENT>return self.histogram_stretch(use_bands, **kwargs)<EOL><DEDENT>elif kwargs["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>return self.histogram_equalize(use_bands, **kwargs)<EOL><DEDENT>elif kwargs["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>return self.histogram_match(use_bands, **kwargs)<EOL><DEDENT>elif kwargs["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>return self.histogram_stretch(use_bands, stretch=[<NUM_LIT:0>, <NUM_LIT:100>], **kwargs)<EOL><DEDENT>elif kwargs["<STR_LIT>"] == "<STR_LIT:ignore>" or self.options.get('<STR_LIT>'):<EOL><INDENT>data = self._read(self[use_bands,...], **kwargs)<EOL>return np.rollaxis(data, <NUM_LIT:0>, <NUM_LIT:3>)<EOL><DEDENT>else:<EOL><INDENT>raise KeyError('<STR_LIT>')<EOL><DEDENT>

Convert the image to a 3 band RGB for plotting

        This method shares the same arguments as plot(). It will perform visual adjustment on the
        image and prepare the data for plotting in MatplotLib. Values are converted to an
        appropriate precision and the axis order is changed to put the band axis last.


data = self._read(self[self._ndwi_bands,...]).astype(np.float32)<EOL>return (data[<NUM_LIT:1>,:,:] - data[<NUM_LIT:0>,:,:]) / (data[<NUM_LIT:0>,:,:] + data[<NUM_LIT:1>,:,:])<EOL>

Calculates Normalized Difference Water Index using Coastal and NIR2 bands for WV02, WV03.
For Landsat8 and sentinel2 calculated by using Green and NIR bands.

Returns: numpy array of ndwi values


def histogram_equalize(self, use_bands, **kwargs):

data = self._read(self[use_bands,...], **kwargs)<EOL>data = np.rollaxis(data.astype(np.float32), <NUM_LIT:0>, <NUM_LIT:3>)<EOL>flattened = data.flatten()<EOL>if <NUM_LIT:0> in data:<EOL><INDENT>masked = np.ma.masked_values(data, <NUM_LIT:0>).compressed()<EOL>image_histogram, bin_edges = np.histogram(masked, <NUM_LIT>)<EOL><DEDENT>else:<EOL><INDENT>image_histogram, bin_edges = np.histogram(flattened, <NUM_LIT>)<EOL><DEDENT>bins = (bin_edges[:-<NUM_LIT:1>] + bin_edges[<NUM_LIT:1>:]) / <NUM_LIT><EOL>cdf = image_histogram.cumsum() <EOL>cdf = cdf / float(cdf[-<NUM_LIT:1>])<EOL>image_equalized = np.interp(flattened, bins, cdf).reshape(data.shape)<EOL>if '<STR_LIT>' in kwargs or '<STR_LIT>' in kwargs:<EOL><INDENT>return self._histogram_stretch(image_equalized, **kwargs)<EOL><DEDENT>else:<EOL><INDENT>return image_equalized<EOL><DEDENT>

Equalize and the histogram and normalize value range
            Equalization is on all three bands, not per-band


def histogram_stretch(self, use_bands, **kwargs):

data = self._read(self[use_bands,...], **kwargs)<EOL>data = np.rollaxis(data.astype(np.float32), <NUM_LIT:0>, <NUM_LIT:3>)<EOL>return self._histogram_stretch(data, **kwargs)<EOL>


def histogram_match(self, use_bands, blm_source=None, **kwargs):

assert has_rio, "<STR_LIT>"<EOL>data = self._read(self[use_bands,...], **kwargs)<EOL>data = np.rollaxis(data.astype(np.float32), <NUM_LIT:0>, <NUM_LIT:3>)<EOL>if <NUM_LIT:0> in data:<EOL><INDENT>data = np.ma.masked_values(data, <NUM_LIT:0>)<EOL><DEDENT>bounds = self._reproject(box(*self.bounds), from_proj=self.proj, to_proj="<STR_LIT>").bounds<EOL>if blm_source == '<STR_LIT>':<EOL><INDENT>from gbdxtools.images.browse_image import BrowseImage<EOL>ref = BrowseImage(self.cat_id, bbox=bounds).read()<EOL><DEDENT>else:<EOL><INDENT>from gbdxtools.images.tms_image import TmsImage<EOL>tms = TmsImage(zoom=self._calc_tms_zoom(self.affine[<NUM_LIT:0>]), bbox=bounds, **kwargs)<EOL>ref = np.rollaxis(tms.read(), <NUM_LIT:0>, <NUM_LIT:3>)<EOL><DEDENT>out = np.dstack([rio_match(data[:,:,idx], ref[:,:,idx].astype(np.double)/<NUM_LIT>)<EOL>for idx in range(data.shape[-<NUM_LIT:1>])])<EOL>if '<STR_LIT>' in kwargs or '<STR_LIT>' in kwargs:<EOL><INDENT>return self._histogram_stretch(out, **kwargs)<EOL><DEDENT>else:<EOL><INDENT>return out<EOL><DEDENT>


def materialize(self, node=None, bounds=None, callback=None, out_format='<STR_LIT>', **kwargs):

kwargs.update({<EOL>"<STR_LIT>": node,<EOL>"<STR_LIT>": bounds,<EOL>"<STR_LIT>": callback,<EOL>"<STR_LIT>": out_format<EOL>})<EOL>return self.rda._materialize(**kwargs)<EOL>

Materializes images into gbdx user buckets in s3.
Note: This method is only available to RDA based image classes.

Args:
  node (str): the node in the graph to materialize
  bounds (list): optional bbox for cropping what gets materialized in s3
  out_format (str): VECTOR_TILE, VECTOR, TIF, TILE_STREAM
  callback (str): a callback url like an `sns://`
Returns:
  job_id (str): the job_id of the materialization


def warp(self, dem=None, proj="<STR_LIT>", **kwargs):

try:<EOL><INDENT>img_md = self.rda.metadata["<STR_LIT:image>"]<EOL>x_size = img_md["<STR_LIT>"]<EOL>y_size = img_md["<STR_LIT>"]<EOL><DEDENT>except (AttributeError, KeyError):<EOL><INDENT>x_size = kwargs.get("<STR_LIT>", <NUM_LIT>)<EOL>y_size = kwargs.get("<STR_LIT>", <NUM_LIT>)<EOL><DEDENT>if self.proj is None:<EOL><INDENT>from_proj = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>from_proj = self.proj<EOL><DEDENT>try:<EOL><INDENT>center = wkt.loads(self.rda.metadata["<STR_LIT:image>"]["<STR_LIT>"]).centroid<EOL>g = box(*(center.buffer(self.rda.metadata["<STR_LIT>"]["<STR_LIT>"] / <NUM_LIT:2>).bounds))<EOL>tfm = partial(pyproj.transform, pyproj.Proj(init="<STR_LIT>"), pyproj.Proj(init=proj))<EOL>gsd = kwargs.get("<STR_LIT>", ops.transform(tfm, g).area ** <NUM_LIT:0.5>)<EOL>current_bounds = wkt.loads(self.rda.metadata["<STR_LIT:image>"]["<STR_LIT>"]).bounds<EOL><DEDENT>except (AttributeError, KeyError, TypeError):<EOL><INDENT>tfm = partial(pyproj.transform, pyproj.Proj(init=self.proj), pyproj.Proj(init=proj))<EOL>gsd = kwargs.get("<STR_LIT>", (ops.transform(tfm, shape(self)).area / (self.shape[<NUM_LIT:1>] * self.shape[<NUM_LIT:2>])) ** <NUM_LIT:0.5> )<EOL>current_bounds = self.bounds<EOL><DEDENT>tfm = partial(pyproj.transform, pyproj.Proj(init=from_proj), pyproj.Proj(init=proj))<EOL>itfm = partial(pyproj.transform, pyproj.Proj(init=proj), pyproj.Proj(init=from_proj))<EOL>output_bounds = ops.transform(tfm, box(*current_bounds)).bounds<EOL>gtf = Affine.from_gdal(output_bounds[<NUM_LIT:0>], gsd, <NUM_LIT:0.0>, output_bounds[<NUM_LIT:3>], <NUM_LIT:0.0>, -<NUM_LIT:1> * gsd)<EOL>ll = ~gtf * (output_bounds[:<NUM_LIT:2>])<EOL>ur = ~gtf * (output_bounds[<NUM_LIT:2>:])<EOL>x_chunks = int((ur[<NUM_LIT:0>] - ll[<NUM_LIT:0>]) / x_size) + <NUM_LIT:1><EOL>y_chunks = int((ll[<NUM_LIT:1>] - ur[<NUM_LIT:1>]) / y_size) + <NUM_LIT:1><EOL>num_bands = self.shape[<NUM_LIT:0>]<EOL>try:<EOL><INDENT>dtype = RDA_TO_DTYPE[img_md["<STR_LIT>"]]<EOL><DEDENT>except:<EOL><INDENT>dtype = '<STR_LIT>'<EOL><DEDENT>daskmeta = {<EOL>"<STR_LIT>": {},<EOL>"<STR_LIT>": (num_bands, y_size, x_size),<EOL>"<STR_LIT>": dtype,<EOL>"<STR_LIT:name>": "<STR_LIT>".format(self.name),<EOL>"<STR_LIT>": (num_bands, y_chunks * y_size, x_chunks * x_size)<EOL>}<EOL>def px_to_geom(xmin, ymin):<EOL><INDENT>xmax = int(xmin + x_size)<EOL>ymax = int(ymin + y_size)<EOL>bounds = list((gtf * (xmin, ymax)) + (gtf * (xmax, ymin)))<EOL>return box(*bounds)<EOL><DEDENT>full_bounds = box(*output_bounds)<EOL>dasks = []<EOL>if isinstance(dem, GeoDaskImage):<EOL><INDENT>if dem.proj != proj:<EOL><INDENT>dem = dem.warp(proj=proj, dem=dem)<EOL><DEDENT>dasks.append(dem.dask)<EOL><DEDENT>for y in xrange(y_chunks):<EOL><INDENT>for x in xrange(x_chunks):<EOL><INDENT>xmin = x * x_size<EOL>ymin = y * y_size<EOL>geometry = px_to_geom(xmin, ymin)<EOL>daskmeta["<STR_LIT>"][(daskmeta["<STR_LIT:name>"], <NUM_LIT:0>, y, x)] = (self._warp, geometry, gsd, dem, proj, dtype, <NUM_LIT:5>)<EOL><DEDENT><DEDENT>daskmeta["<STR_LIT>"], _ = optimization.cull(HighLevelGraph.merge(daskmeta["<STR_LIT>"], *dasks), list(daskmeta["<STR_LIT>"].keys()))<EOL>gi = mapping(full_bounds)<EOL>gt = AffineTransform(gtf, proj)<EOL>image = GeoDaskImage(daskmeta, __geo_interface__ = gi, __geo_transform__ = gt)<EOL>return image[box(*output_bounds)]<EOL>

Delayed warp across an entire AOI or Image

        Creates a new dask image by deferring calls to the warp_geometry on chunks

        Args:
            dem (ndarray): optional. A DEM for warping to specific elevation planes
            proj (str): optional. An EPSG proj string to project the image data into ("EPSG:32612")

        Returns:
            daskarray: a warped image as deferred image array


arr = self<EOL>if bands is not None:<EOL><INDENT>arr = self[bands, ...]<EOL><DEDENT>return arr.compute(scheduler=threaded_get)<EOL>

Reads data from a dask array and returns the computed ndarray matching the given bands

        Args:
            bands (list): band indices to read from the image. Returns bands in the order specified in the list of bands.

        Returns:
            ndarray: a numpy array of image data


darr = super(GeoDaskImage, self).map_blocks(*args, **kwargs)<EOL>return GeoDaskImage(darr, __geo_interface__ = self.__geo_interface__,<EOL>__geo_transform__ = self.__geo_transform__)<EOL>

Queue a deferred function to run on each block of image

        This is identical to Dask's map_block functinos, but returns a GeoDaskImage to preserve
        the geospatial information.

        Args: see dask.Array.map_blocks

        Returns:
            GeoDaskImage: a dask array with the function queued up to run when the image is read


def iterwindows(self, count=<NUM_LIT:64>, window_shape=(<NUM_LIT>, <NUM_LIT>)):

if count is None:<EOL><INDENT>while True:<EOL><INDENT>yield self.randwindow(window_shape)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for i in xrange(count):<EOL><INDENT>yield self.randwindow(window_shape)<EOL><DEDENT><DEDENT>

Iterate over random windows of an image

        Args:
            count (int): the number of the windows to generate. Defaults to 64, if `None` will continue to iterate over random windows until stopped.
            window_shape (tuple): The desired shape of each image as (height, width) in pixels.

        Yields:
            image: an image of the given shape and same type.


try:<EOL><INDENT>if isinstance(geom, dict):<EOL><INDENT>if '<STR_LIT>' in geom:<EOL><INDENT>geom = shape(geom['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>geom = shape(geom)<EOL><DEDENT><DEDENT>elif isinstance(geom, BaseGeometry):<EOL><INDENT>geom = shape(geom)<EOL><DEDENT>else:<EOL><INDENT>geom = wkt.loads(geom)<EOL><DEDENT><DEDENT>except:<EOL><INDENT>raise TypeError ("<STR_LIT>")<EOL><DEDENT>if geom.disjoint(shape(self)):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>(xmin, ymin, xmax, ymax) = ops.transform(self.__geo_transform__.rev, geom).bounds<EOL>_nbands, ysize, xsize = self.shape<EOL>if clip:<EOL><INDENT>xmin = max(xmin, <NUM_LIT:0>)<EOL>ymin = max(ymin, <NUM_LIT:0>)<EOL>xmax = min(xmax, xsize)<EOL>ymax = min(ymax, ysize)<EOL><DEDENT>return (xmin, ymin, xmax, ymax)<EOL>

Returns the bounds of a geometry object in pixel coordinates

        Args:
            geom: Shapely geometry object or GeoJSON as Python dictionary or WKT string
            clip (bool): Clip the bounds to the min/max extent of the image

        Returns:
            list: bounds in pixels [min x, min y, max x, max y] clipped to image bounds


bbox = kwargs.get('<STR_LIT>', None)<EOL>wkt_geom = kwargs.get('<STR_LIT>', None)<EOL>geojson = kwargs.get('<STR_LIT>', None)<EOL>if bbox is not None:<EOL><INDENT>g = box(*bbox)<EOL><DEDENT>elif wkt_geom is not None:<EOL><INDENT>g = wkt.loads(wkt_geom)<EOL><DEDENT>elif geojson is not None:<EOL><INDENT>g = shape(geojson)<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>if self.proj is None:<EOL><INDENT>return g<EOL><DEDENT>else:<EOL><INDENT>return self._reproject(g, from_proj=kwargs.get('<STR_LIT>', '<STR_LIT>'))<EOL><DEDENT>

Finds supported geometry types, parses them and returns the bbox


if '<STR_LIT>' not in kwargs:<EOL><INDENT>kwargs['<STR_LIT>'] = self.proj<EOL><DEDENT>return to_geotiff(self, **kwargs)<EOL>

Creates a geotiff on the filesystem

        Args:
            path (str): optional, path to write the geotiff file to, default is ./output.tif
            proj (str): optional, EPSG string of projection to reproject to
            spec (str): optional, if set to 'rgb', write out color-balanced 8-bit RGB tif
            bands (list): optional, list of bands to export. If spec='rgb' will default to RGB bands,
                otherwise will export all bands

        Returns:
            str: path the geotiff was written to


g = self._parse_geoms(**kwargs)<EOL>if g is None:<EOL><INDENT>return self<EOL><DEDENT>else:<EOL><INDENT>return self[g]<EOL><DEDENT>

Subsets the Image by the given bounds

        Args:
            bbox (list): optional. A bounding box array [minx, miny, maxx, maxy]
            wkt (str): optional. A WKT geometry string
            geojson (str): optional. A GeoJSON geometry dictionary

        Returns:
            image: an image instance of the same type


url = '<STR_LIT>'.format(cat_id)<EOL>auth = Auth()<EOL>r = _req_with_retries(auth.gbdx_connection, url)<EOL>try: <EOL><INDENT>data = r.json()<EOL>return data['<STR_LIT>'] is not None<EOL><DEDENT>except:<EOL><INDENT>return False<EOL><DEDENT>

Checks to see if a CatalogID can be atmos. compensated or not.

Args:
  catalogID (str): The catalog ID from the platform catalog.
Returns:
  available (bool): Whether or not the image can be acomp'd


url = '<STR_LIT>'.format(cat_id)<EOL>auth = Auth()<EOL>r = _req_with_retries(auth.gbdx_connection, url)<EOL>if r is not None:<EOL><INDENT>return r.status_code == <NUM_LIT:200><EOL><DEDENT>return False<EOL>

Checks to see if a CatalogID has been ordered or not.

Args:
  catalogID (str): The catalog ID from the platform catalog.
Returns:
  ordered (bool): Whether or not the image has been ordered


tfm = partial(pyproj.transform,<EOL>pyproj.Proj(init="<STR_LIT>"),<EOL>pyproj.Proj(init="<STR_LIT>"))<EOL>bounds = ops.transform(tfm, box(*bounds)).bounds<EOL>west, south, east, north = bounds<EOL>epsilon = <NUM_LIT><EOL>if east != west and north != south:<EOL><INDENT>west += epsilon<EOL>south += epsilon<EOL>east -= epsilon<EOL>north -= epsilon<EOL><DEDENT>params = [west, south, east, north, [self.zoom_level]]<EOL>tile_coords = [(tile.x, tile.y) for tile in mercantile.tiles(*params)]<EOL>xtiles, ytiles = zip(*tile_coords)<EOL>minx = min(xtiles)<EOL>miny = min(ytiles)<EOL>maxx = max(xtiles) <EOL>maxy = max(ytiles)<EOL>return minx, miny, maxx, maxy<EOL>

convert mercator bbox to tile index limits


result = {}<EOL>for dictionary in dict_args:<EOL><INDENT>result.update(dictionary)<EOL><DEDENT>return result<EOL>

Given any number of dicts, shallow copy and merge into a new dict,
precedence goes to key value pairs in latter dicts.


return can_acomp(cat_id)<EOL>


url = self.get_url % index<EOL>r = self.gbdx_connection.get(url + ID)<EOL>r.raise_for_status()<EOL>return r.json()<EOL>

Retrieves a vector.  Not usually necessary because searching is the best way to find & get stuff.

        Args:
            ID (str): ID of the vector object
            index (str): Optional.  Index the object lives in.  defaults to 'vector-web-s'

        Returns:
            record (dict): A dict object identical to the json representation of the catalog record


if self.value:<EOL><INDENT>base = '<STR_LIT>' % (self.agg_type, self.value)<EOL><DEDENT>else:<EOL><INDENT>base = '<STR_LIT:%s>' % self.agg_type<EOL><DEDENT>if self.children:<EOL><INDENT>if isinstance(self.children, six.string_types):<EOL><INDENT>return '<STR_LIT>' % (base, self.children)<EOL><DEDENT>elif isinstance(self.children, AggregationDef):<EOL><INDENT>return '<STR_LIT>' % (base, self.children.__repr__())<EOL><DEDENT>else: <EOL><INDENT>kids = []<EOL>for child in self.children:<EOL><INDENT>kids.append(child.__repr__())<EOL><DEDENT>kids_str = '<STR_LIT>' % '<STR_LIT:U+002C>'.join(kids)<EOL>return '<STR_LIT>' % (base, kids_str)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return base<EOL><DEDENT>

Creates a string representation of an aggregation definition suitable for use in VectorServices calls

        Returns:
            A string representation of an aggregation definition suitable for use in VectorServices calls


def query_iteratively(self, searchAreaWkt, query, count=<NUM_LIT:100>, ttl='<STR_LIT>', index=default_index):

search_area_polygon = from_wkt(searchAreaWkt)<EOL>left, lower, right, upper = search_area_polygon.bounds<EOL>params = {<EOL>"<STR_LIT:q>": query,<EOL>"<STR_LIT:count>": min(count,<NUM_LIT:1000>),<EOL>"<STR_LIT>": ttl,<EOL>"<STR_LIT:left>": left,<EOL>"<STR_LIT:right>": right,<EOL>"<STR_LIT>": lower,<EOL>"<STR_LIT>": upper<EOL>}<EOL>url = self.query_index_page_url % index if index else self.query_page_url<EOL>r = self.gbdx_connection.get(url, params=params)<EOL>r.raise_for_status()<EOL>page = r.json()<EOL>paging_id = page['<STR_LIT>']<EOL>item_count = int(page['<STR_LIT>'])<EOL>data = page['<STR_LIT:data>']<EOL>num_results = <NUM_LIT:0><EOL>for vector in data:<EOL><INDENT>num_results += <NUM_LIT:1><EOL>if num_results > count: break<EOL>yield vector<EOL><DEDENT>if num_results == count:<EOL><INDENT>return<EOL><DEDENT>while paging_id and item_count > <NUM_LIT:0> and num_results < count:<EOL><INDENT>headers = {'<STR_LIT:Content-Type>':'<STR_LIT>'}<EOL>data = {<EOL>"<STR_LIT>": paging_id,<EOL>"<STR_LIT>": ttl<EOL>}<EOL>r = self.gbdx_connection.post(self.page_url, headers=headers, data=data)<EOL>r.raise_for_status()<EOL>page = r.json()<EOL>paging_id = page['<STR_LIT>']<EOL>item_count = int(page['<STR_LIT>'])<EOL>data = page['<STR_LIT:data>']<EOL>for vector in data:<EOL><INDENT>num_results += <NUM_LIT:1><EOL>if num_results > count: break<EOL>yield vector<EOL><DEDENT><DEDENT>

Perform a vector services query using the QUERY API
(https://gbdxdocs.digitalglobe.com/docs/vs-query-list-vector-items-returns-default-fields)

Args:
    searchAreaWkt: WKT Polygon of area to search
    query: Elastic Search query
    count: Maximum number of results to return
    ttl: Amount of time for each temporary vector page to exist

Returns:
    generator of vector results


def create_from_wkt(self, wkt, item_type, ingest_source, **attributes):

<EOL>geojson = load_wkt(wkt).__geo_interface__<EOL>vector = {<EOL>'<STR_LIT:type>': "<STR_LIT>",<EOL>'<STR_LIT>': geojson,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': item_type,<EOL>'<STR_LIT>': ingest_source,<EOL>'<STR_LIT>': attributes<EOL>}<EOL>}<EOL>return self.create(vector)[<NUM_LIT:0>]<EOL>

Create a single vector in the vector service

Args:
    wkt (str): wkt representation of the geometry
    item_type (str): item_type of the vector
    ingest_source (str): source of the vector
    attributes: a set of key-value pairs of attributes

Returns:
    id (str): string identifier of the vector created


interface = Auth(**kwargs)<EOL>self.gbdx_connection = interface.gbdx_connection<EOL>self.logger = interface.logger<EOL>self.query_url = '<STR_LIT>'<EOL>self.query_index_url = '<STR_LIT>'<EOL>self.query_page_url = '<STR_LIT>'<EOL>self.query_index_page_url = '<STR_LIT>'<EOL>self.page_url = '<STR_LIT>'<EOL>self.get_url = '<STR_LIT>'<EOL>self.create_url = '<STR_LIT>'<EOL>self.aggregations_url = '<STR_LIT>'<EOL>self.aggregations_by_index_url = '<STR_LIT>'<EOL>

Construct the Vectors interface class

        Returns:
            An instance of the Vectors interface class.


def map(self, features=None, query=None, styles=None,<EOL>bbox=[-<NUM_LIT>,-<NUM_LIT>,<NUM_LIT>,<NUM_LIT>], zoom=<NUM_LIT:10>, center=None, <EOL>image=None, image_bounds=None, cmap='<STR_LIT>',<EOL>api_key=os.environ.get('<STR_LIT>', None), **kwargs):

try:<EOL><INDENT>from IPython.display import display<EOL><DEDENT>except:<EOL><INDENT>print("<STR_LIT>")<EOL>return<EOL><DEDENT>assert api_key is not None, "<STR_LIT>"<EOL>if features is None and query is not None:<EOL><INDENT>wkt = box(*bbox).wkt<EOL>features = self.query(wkt, query, index=None)<EOL><DEDENT>elif features is None and query is None and image is None:<EOL><INDENT>print('<STR_LIT>')<EOL>return<EOL><DEDENT>if styles is not None and not isinstance(styles, list):<EOL><INDENT>styles = [styles]<EOL><DEDENT>geojson = {"<STR_LIT:type>":"<STR_LIT>", "<STR_LIT>": features}<EOL>if center is None and features is not None:<EOL><INDENT>union = cascaded_union([shape(f['<STR_LIT>']) for f in features])<EOL>lon, lat = union.centroid.coords[<NUM_LIT:0>]<EOL><DEDENT>elif center is None and image is not None:<EOL><INDENT>try:<EOL><INDENT>lon, lat = shape(image).centroid.coords[<NUM_LIT:0>]<EOL><DEDENT>except:<EOL><INDENT>lon, lat = box(*image_bounds).centroid.coords[<NUM_LIT:0>]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>lat, lon = center<EOL><DEDENT>map_id = "<STR_LIT>".format(str(int(time.time())))<EOL>map_data = VectorGeojsonLayer(geojson, styles=styles, **kwargs)<EOL>image_layer = self._build_image_layer(image, image_bounds, cmap)<EOL>template = BaseTemplate(map_id, **{<EOL>"<STR_LIT>": lat, <EOL>"<STR_LIT>": lon, <EOL>"<STR_LIT>": zoom,<EOL>"<STR_LIT>": json.dumps(map_data.datasource),<EOL>"<STR_LIT>": json.dumps(map_data.layers),<EOL>"<STR_LIT>": image_layer,<EOL>"<STR_LIT>": api_key,<EOL>"<STR_LIT>": '<STR_LIT>'<EOL>})<EOL>template.inject()<EOL>

Renders a mapbox gl map from a vector service query or a list of geojson features

Args:
  features (list): a list of geojson features
  query (str): a VectorServices query
  styles (list): a list of VectorStyles to apply to the features
  bbox (list): a bounding box to query for features ([minx, miny, maxx, maxy])
  zoom (int): the initial zoom level of the map
  center (list): a list of [lat, lon] used to center the map
  api_key (str): a valid Mapbox API key
  image (dict): a CatalogImage or a ndarray
  image_bounds (list): a list of bounds for image positioning
  Use outside of GBDX Notebooks requires a MapBox API key, sign up for free at https://www.mapbox.com/pricing/
  Pass the key using the `api_key` keyword or set an environmental variable called `MAPBOX API KEY`
  cmap (str): MatPlotLib colormap to use for rendering single band images (default: viridis)


def get_chip(self, coordinates, catid, chip_type='<STR_LIT>', chip_format='<STR_LIT>', filename='<STR_LIT>'):

def t2s1(t):<EOL><INDENT>return str(t).strip('<STR_LIT>').replace('<STR_LIT:U+002C>', '<STR_LIT>')<EOL><DEDENT>def t2s2(t):<EOL><INDENT>return str(t).strip('<STR_LIT>').replace('<STR_LIT:U+0020>', '<STR_LIT>')<EOL><DEDENT>if len(coordinates) != <NUM_LIT:4>:<EOL><INDENT>print('<STR_LIT>')<EOL>return False<EOL><DEDENT>W, S, E, N = coordinates<EOL>box = ((W, S), (W, N), (E, N), (E, S), (W, S))<EOL>box_wkt = '<STR_LIT>' + '<STR_LIT:U+002C>'.join([t2s1(corner) for corner in box]) + '<STR_LIT>'<EOL>results = self.get_images_by_catid_and_aoi(catid=catid, aoi_wkt=box_wkt)<EOL>description = self.describe_images(results)<EOL>pan_id, ms_id, num_bands = None, None, <NUM_LIT:0><EOL>for catid, images in description.items():<EOL><INDENT>for partnum, part in images['<STR_LIT>'].items():<EOL><INDENT>if '<STR_LIT>' in part.keys():<EOL><INDENT>pan_id = part['<STR_LIT>']['<STR_LIT:id>']<EOL>bucket = part['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in part.keys():<EOL><INDENT>ms_id = part['<STR_LIT>']['<STR_LIT:id>']<EOL>num_bands = <NUM_LIT:8><EOL>bucket = part['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT>elif '<STR_LIT>' in part.keys():<EOL><INDENT>ms_id = part['<STR_LIT>']['<STR_LIT:id>']<EOL>num_bands = <NUM_LIT:4><EOL>bucket = part['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT><DEDENT><DEDENT>band_str = '<STR_LIT>'<EOL>if chip_type == '<STR_LIT>':<EOL><INDENT>band_str = pan_id + '<STR_LIT>'<EOL><DEDENT>elif chip_type == '<STR_LIT>':<EOL><INDENT>band_str = ms_id + '<STR_LIT:?>'<EOL><DEDENT>elif chip_type == '<STR_LIT>':<EOL><INDENT>if num_bands == <NUM_LIT:8>:<EOL><INDENT>band_str = ms_id + '<STR_LIT>' + pan_id<EOL><DEDENT>elif num_bands == <NUM_LIT:4>:<EOL><INDENT>band_str = ms_id + '<STR_LIT>' + pan_id<EOL><DEDENT><DEDENT>location_str = '<STR_LIT>'.format(t2s2((W, N)), t2s2((E, S)))<EOL>service_url = '<STR_LIT>' + bucket + '<STR_LIT:/>'<EOL>url = service_url + band_str + location_str<EOL>url += '<STR_LIT>' + chip_format + '<STR_LIT>' + self.gbdx_connection.access_token<EOL>r = requests.get(url)<EOL>if r.status_code == <NUM_LIT:200>:<EOL><INDENT>with open(filename, '<STR_LIT:wb>') as f:<EOL><INDENT>f.write(r.content)<EOL>return True<EOL><DEDENT><DEDENT>else:<EOL><INDENT>print('<STR_LIT>')<EOL>return False<EOL><DEDENT>

Downloads a native resolution, orthorectified chip in tif format
        from a user-specified catalog id.

        Args:
            coordinates (list): Rectangle coordinates in order West, South, East, North.
                                West and East are longitudes, North and South are latitudes.
                                The maximum chip size is (2048 pix)x(2048 pix)
            catid (str): The image catalog id.
            chip_type (str): 'PAN' (panchromatic), 'MS' (multispectral), 'PS' (pansharpened).
                             'MS' is 4 or 8 bands depending on sensor.
            chip_format (str): 'TIF' or 'PNG'
            filename (str): Where to save chip.

        Returns:
            True if chip is successfully downloaded; else False.


def get_images_by_catid_and_aoi(self, catid, aoi_wkt):

self.logger.debug('<STR_LIT>')<EOL>url = '<STR_LIT>' % self.base_url<EOL>body = {"<STR_LIT>": ["<STR_LIT>" % catid],<EOL>"<STR_LIT>": ["<STR_LIT>"],<EOL>"<STR_LIT>": aoi_wkt}<EOL>r = self.gbdx_connection.post(url, data=json.dumps(body))<EOL>r.raise_for_status()<EOL>if r.status_code == <NUM_LIT:200>:<EOL><INDENT>results = r.json()<EOL>numresults = len(results['<STR_LIT>'])<EOL>self.logger.debug('<STR_LIT>'<EOL>% (numresults, catid))<EOL>return results<EOL><DEDENT>

Retrieves the IDAHO image records associated with a given catid.
        Args:
            catid (str): The source catalog ID from the platform catalog.
            aoi_wkt (str): The well known text of the area of interest.
        Returns:
            results (json): The full catalog-search response for IDAHO images
                            within the catID.


def create_leaflet_viewer(self, idaho_image_results, filename):

description = self.describe_images(idaho_image_results)<EOL>if len(description) > <NUM_LIT:0>:<EOL><INDENT>functionstring = '<STR_LIT>'<EOL>for catid, images in description.items():<EOL><INDENT>for partnum, part in images['<STR_LIT>'].items():<EOL><INDENT>num_images = len(list(part.keys()))<EOL>partname = None<EOL>if num_images == <NUM_LIT:1>:<EOL><INDENT>partname = [p for p in list(part.keys())][<NUM_LIT:0>]<EOL>pan_image_id = '<STR_LIT>'<EOL><DEDENT>elif num_images == <NUM_LIT:2>:<EOL><INDENT>partname = [p for p in list(part.keys()) if p is not '<STR_LIT>'][<NUM_LIT:0>]<EOL>pan_image_id = part['<STR_LIT>']['<STR_LIT:id>']<EOL><DEDENT>if not partname:<EOL><INDENT>self.logger.debug("<STR_LIT>")<EOL>continue<EOL><DEDENT>bandstr = {<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT:0>'<EOL>}.get(partname, '<STR_LIT>')<EOL>part_boundstr_wkt = part[partname]['<STR_LIT>']<EOL>part_polygon = from_wkt(part_boundstr_wkt)<EOL>bucketname = part[partname]['<STR_LIT>']<EOL>image_id = part[partname]['<STR_LIT:id>']<EOL>W, S, E, N = part_polygon.bounds<EOL>functionstring += "<STR_LIT>" % (<EOL>bucketname, image_id, W, S, E, N, pan_image_id)<EOL><DEDENT><DEDENT>__location__ = os.path.realpath(<EOL>os.path.join(os.getcwd(), os.path.dirname(__file__)))<EOL>try:<EOL><INDENT>with open(os.path.join(__location__, '<STR_LIT>'), '<STR_LIT:r>') as htmlfile:<EOL><INDENT>data = htmlfile.read().decode("<STR_LIT:utf8>")<EOL><DEDENT><DEDENT>except AttributeError:<EOL><INDENT>with open(os.path.join(__location__, '<STR_LIT>'), '<STR_LIT:r>') as htmlfile:<EOL><INDENT>data = htmlfile.read()<EOL><DEDENT><DEDENT>data = data.replace('<STR_LIT>', functionstring)<EOL>data = data.replace('<STR_LIT>', str(S))<EOL>data = data.replace('<STR_LIT>', str(W))<EOL>data = data.replace('<STR_LIT>', bandstr)<EOL>data = data.replace('<STR_LIT>', self.gbdx_connection.access_token)<EOL>with codecs.open(filename, '<STR_LIT:w>', '<STR_LIT:utf8>') as outputfile:<EOL><INDENT>self.logger.debug("<STR_LIT>" % filename)<EOL>outputfile.write(data)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>

Create a leaflet viewer html file for viewing idaho images.

        Args:
            idaho_image_results (dict): IDAHO image result set as returned from
                                        the catalog.
            filename (str): Where to save output html file.


<EOL>snippet = {<EOL>'<STR_LIT>': VectorStyle.get_style_value(self.opacity),<EOL>'<STR_LIT>': VectorStyle.get_style_value(self.color),<EOL>'<STR_LIT>': VectorStyle.get_style_value(self.width),<EOL>}<EOL>if self.translate:<EOL><INDENT>snippet['<STR_LIT>'] = self.translate<EOL><DEDENT>if self.dasharray:<EOL><INDENT>snippet['<STR_LIT>'] = VectorStyle.get_style_value(self.dasharray)<EOL><DEDENT>return snippet<EOL>

Renders a javascript snippet suitable for use as a mapbox-gl line paint entry

Returns:
    A dict that can be converted to a mapbox-gl javascript paint snippet


url = '<STR_LIT>' % self.base_url<EOL>r = self.gbdx_connection.get(url)<EOL>r.raise_for_status()<EOL>return r.json()<EOL>

Get user info for GBDX S3, put into instance vars for convenience.

        Args:
            None.

        Returns:
            Dictionary with S3 access key, S3 secret key, S3 session token,
            user bucket and user prefix (dict).


bucket = self.info['<STR_LIT>']<EOL>prefix = self.info['<STR_LIT>']<EOL>self.logger.debug('<STR_LIT>')<EOL>s3conn = self.client <EOL>if location[<NUM_LIT:0>] == '<STR_LIT:/>':<EOL><INDENT>location = location[<NUM_LIT:1>:]<EOL><DEDENT>if location[-<NUM_LIT:1>] == '<STR_LIT:/>':<EOL><INDENT>location = location[:-<NUM_LIT:2>]<EOL><DEDENT>self.logger.debug('<STR_LIT>')<EOL>for s3key in s3conn.list_objects(Bucket=bucket, Prefix=(prefix+'<STR_LIT:/>'+location))['<STR_LIT>']:<EOL><INDENT>s3conn.delete_object(Bucket=bucket, Key=s3key['<STR_LIT>'])<EOL><DEDENT>self.logger.debug('<STR_LIT>')<EOL>

Delete content in bucket/prefix/location.
           Location can be a directory or a file (e.g., my_dir or my_dir/my_image.tif)
           If location is a directory, all files in the directory are deleted.
           If it is a file, then that file is deleted.

           Args:
               location (str): S3 location within prefix. Can be a directory or
                               a file (e.g., my_dir or my_dir/my_image.tif).


def upload(self, local_file, s3_path=None):

if not os.path.exists(local_file):<EOL><INDENT>raise Exception(local_file + "<STR_LIT>")<EOL><DEDENT>if s3_path is None:<EOL><INDENT>s3_path = os.path.basename(local_file)<EOL><DEDENT>bucket = self.info['<STR_LIT>']<EOL>prefix = self.info['<STR_LIT>']<EOL>self.logger.debug('<STR_LIT>')<EOL>s3conn = self.client <EOL>self.logger.debug('<STR_LIT>'.format(local_file))<EOL>s3conn.upload_file(local_file, bucket, prefix+'<STR_LIT:/>'+s3_path)<EOL>self.logger.debug('<STR_LIT>')<EOL>return '<STR_LIT>'.format(bucket, prefix, s3_path)<EOL>

Upload files to your DG S3 bucket/prefix.

Args:
    local_file (str): a path to a local file to upload, directory structures are not mirrored
    s3_path: a key (location) on s3 to upload the file to

Returns:
    str: s3 path file was saved to

Examples:
    >>> upload('path/to/image.tif')
    'mybucket/myprefix/image.tif'

    >>> upload('./images/image.tif')
    'mybucket/myprefix/image.tif'

    >>> upload('./images/image.tif', s3_path='images/image.tif')
    'mybucket/myprefix/images/image.tif'


<EOL>self.generate_workflow_description()<EOL>if self.batch_values:<EOL><INDENT>self.id = self.workflow.launch_batch_workflow(self.definition)<EOL><DEDENT>else:<EOL><INDENT>self.id = self.workflow.launch(self.definition)<EOL><DEDENT>return self.id<EOL>

Execute the workflow.

Args:
    None

Returns:
    Workflow_id


if not self.id:<EOL><INDENT>raise WorkflowError('<STR_LIT>')<EOL><DEDENT>if self.batch_values:<EOL><INDENT>self.workflow.batch_workflow_cancel(self.id)<EOL><DEDENT>else:<EOL><INDENT>self.workflow.cancel(self.id)<EOL><DEDENT>

Cancel a running workflow.

Args:
    None

Returns:
    None


def __init__(self, __task_type, **kwargs):

self.name = __task_type + '<STR_LIT:_>' + str(uuid.uuid4())[:<NUM_LIT:8>]<EOL>self.name = self.name.replace('<STR_LIT::>','<STR_LIT:_>')  <EOL>self.type = __task_type<EOL>task_registry = TaskRegistry()<EOL>self.definition = task_registry.get_definition(__task_type)<EOL>self.domain = self.definition.get('<STR_LIT>', [{'<STR_LIT>': {}}])[<NUM_LIT:0>]['<STR_LIT>'].get('<STR_LIT>', '<STR_LIT:default>')<EOL>self._timeout = self.definition['<STR_LIT>'].get('<STR_LIT>')<EOL>self.inputs = Inputs(self.input_ports, task=self)<EOL>self.outputs = Outputs(self.output_ports, self.name)<EOL>self.batch_values = None<EOL>self._impersonation_allowed = None<EOL>self.set(**kwargs)<EOL>

Construct an instance of GBDX Task

Args:
    __task_type: name of the task
    **kwargs: key=value pairs for inputs to set on the task

Returns:
    An instance of Task.


<EOL>matching_multiplex_ports = [self.__getattribute__(p) for p in self._portnames<EOL>if name.startswith(p)<EOL>and name != p<EOL>and hasattr(self, p)<EOL>and self.__getattribute__(p).is_multiplex<EOL>]<EOL>for port in matching_multiplex_ports:<EOL><INDENT>return port<EOL><DEDENT>return None<EOL>

Given a name, figure out if a multiplex port prefixes this name and return it.  Otherwise return none.


def get_most_recent_images(self, results, types=[], sensors=[], N=<NUM_LIT:1>):

if not len(results):<EOL><INDENT>return None<EOL><DEDENT>if types:<EOL><INDENT>results = [r for r in results if r['<STR_LIT:type>'] in types]<EOL><DEDENT>if sensors:<EOL><INDENT>results = [r for r in results if r['<STR_LIT>'].get('<STR_LIT>') in sensors]<EOL><DEDENT>newlist = sorted(results, key=lambda k: k['<STR_LIT>'].get('<STR_LIT>'), reverse=True)<EOL>return newlist[:N]<EOL>

Return the most recent image

        Args:
            results: a catalog resultset, as returned from a search
            types: array of types you want. optional.
            sensors: array of sensornames. optional.
            N: number of recent images to return.  defaults to 1.

        Returns:
            single catalog item, or none if not found


def search(self, searchAreaWkt=None, filters=None, startDate=None, endDate=None, types=None):

<EOL>if not types:<EOL><INDENT>types = ['<STR_LIT>']<EOL><DEDENT>if startDate:<EOL><INDENT>startDateTime = datetime.datetime.strptime(startDate, '<STR_LIT>')<EOL><DEDENT>if endDate:<EOL><INDENT>endDateTime = datetime.datetime.strptime(endDate, '<STR_LIT>')<EOL><DEDENT>if startDate and endDate:<EOL><INDENT>diff = endDateTime - startDateTime<EOL>if diff.days < <NUM_LIT:0>:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT><DEDENT>postdata = {<EOL>"<STR_LIT>": searchAreaWkt,<EOL>"<STR_LIT>": types,<EOL>"<STR_LIT>": startDate,<EOL>"<STR_LIT>": endDate,<EOL>}<EOL>if filters:<EOL><INDENT>postdata['<STR_LIT>'] = filters<EOL><DEDENT>if searchAreaWkt:<EOL><INDENT>postdata['<STR_LIT>'] = searchAreaWkt<EOL><DEDENT>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url<EOL>}<EOL>headers = {'<STR_LIT:Content-Type>':'<STR_LIT:application/json>'}<EOL>r = self.gbdx_connection.post(url, headers=headers, data=json.dumps(postdata))<EOL>r.raise_for_status()<EOL>results = r.json()['<STR_LIT>']<EOL>return results<EOL>

Perform a catalog search

        Args:
            searchAreaWkt: WKT Polygon of area to search.  Optional.
            filters: Array of filters.  Optional.  Example:
            [
                "(sensorPlatformName = 'WORLDVIEW01' OR sensorPlatformName ='QUICKBIRD02')",
                "cloudCover < 10",
                "offNadirAngle < 10"
            ]
            startDate: string.  Optional.  Example: "2004-01-01T00:00:00.000Z"
            endDate: string.  Optional.  Example: "2004-01-01T00:00:00.000Z"
            types: Array of types to search for.  Optional.  Example (and default):  ["Acquisition"]

        Returns:
            catalog search resultset


interface = Auth(**kwargs)<EOL>self.base_url = '<STR_LIT>' % interface.root_url<EOL>self.gbdx_connection = interface.gbdx_connection<EOL>self.logger = interface.logger<EOL>

Construct the Catalog interface class

        Returns:
            An instance of the Catalog interface class.


self.logger.debug('<STR_LIT>')<EOL>url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url, '<STR_LIT>': catID<EOL>}<EOL>r = self.gbdx_connection.get(url)<EOL>if r.status_code == <NUM_LIT:200>:<EOL><INDENT>return r.json()['<STR_LIT>']<EOL><DEDENT>elif r.status_code == <NUM_LIT>:<EOL><INDENT>self.logger.debug('<STR_LIT>' % catID)<EOL>r.raise_for_status()<EOL><DEDENT>else:<EOL><INDENT>self.logger.debug('<STR_LIT>' % catID)<EOL>r.raise_for_status()<EOL><DEDENT>

Retrieves the strip catalog metadata given a cat ID.

        Args:
            catID (str): The source catalog ID from the platform catalog.

        Returns:
            metadata (dict): A metadata dictionary .

            TODO: have this return a class object with interesting information exposed.


def search_address(self, address, filters=None, startDate=None, endDate=None, types=None):

lat, lng = self.get_address_coords(address)<EOL>return self.search_point(lat,lng, filters=filters, startDate=startDate, endDate=endDate, types=types)<EOL>

Perform a catalog search over an address string

        Args:
            address: any address string
            filters: Array of filters.  Optional.  Example:
            [
                "(sensorPlatformName = 'WORLDVIEW01' OR sensorPlatformName ='QUICKBIRD02')",
                "cloudCover < 10",
                "offNadirAngle < 10"
            ]
            startDate: string.  Optional.  Example: "2004-01-01T00:00:00.000Z"
            endDate: string.  Optional.  Example: "2004-01-01T00:00:00.000Z"
            types: Array of types to search for.  Optional.  Example (and default):  ["Acquisition"]

        Returns:
            catalog search resultset


def get(self, catID, includeRelationships=False):

url = '<STR_LIT>' % {<EOL>'<STR_LIT>': self.base_url, '<STR_LIT>': catID<EOL>}<EOL>r = self.gbdx_connection.get(url)<EOL>r.raise_for_status()<EOL>return r.json()<EOL>

Retrieves the strip footprint WKT string given a cat ID.

        Args:
            catID (str): The source catalog ID from the platform catalog.
            includeRelationships (bool): whether to include graph links to related objects.  Default False.

        Returns:
            record (dict): A dict object identical to the json representation of the catalog record


deprecated = set(deprecated)<EOL>class Wrapper(object):<EOL><INDENT>def __getattr__(self, attr):<EOL><INDENT>if attr in deprecated:<EOL><INDENT>warnings.warn("<STR_LIT>".format(attr), GBDXDeprecation)<EOL><DEDENT>return getattr(mod, attr)<EOL><DEDENT>def __setattr__(self, attr, value):<EOL><INDENT>if attr in deprecated:<EOL><INDENT>warnings.warn("<STR_LIT>".format(attr), GBDXDeprecation)<EOL><DEDENT>return setattr(mod, attr, value)<EOL><DEDENT><DEDENT>return Wrapper()<EOL>

Return a wrapped object that warns about deprecated accesses


r = self.gbdx_connection.get(self._base_url + '<STR_LIT:/>' + task_name)<EOL>raise_for_status(r)<EOL>return r.json()<EOL>

Gets definition of a registered GBDX task.

        Args:
            task_name (str): Task name.

        Returns:
            Dictionary representing the task definition.


def register(self, task_json=None, json_filename=None):

if not task_json and not json_filename:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if task_json and json_filename:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if json_filename:<EOL><INDENT>task_json = json.load(open(json_filename, '<STR_LIT:r>'))<EOL><DEDENT>r = self.gbdx_connection.post(self._base_url, json=task_json)<EOL>raise_for_status(r)<EOL>return r.text<EOL>

Registers a new GBDX task.

        Args:
            task_json (dict): Dictionary representing task definition.
            json_filename (str): A full path of a file with json representing the task definition.
            Only one out of task_json and json_filename should be provided.
        Returns:
            Response (str).


r = self.gbdx_connection.put(self._base_url + '<STR_LIT:/>' + task_name, json=task_json)<EOL>raise_for_status(r)<EOL>return r.json()<EOL>

Updates a GBDX task.

        Args:
            task_name (str): Task name.
            task_json (dict): Dictionary representing updated task definition.

        Returns:
            Dictionary representing the updated task definition.


raise NotImplementedError()<EOL>

Constructs a layer def with the proper fields
            - implemented in subclasses

            Returns:
                layer (dict): a layer json dict used for adding to maps


raise NotImplementedError()<EOL>

Constructs a datasource def appropriate for the layer type
            - implemented in subclasses

            Returns
                datasource (dict): a datasource json dict used for adding data to maps


layers = [self._layer_def(style) for style in self.styles]<EOL>return layers<EOL>

Renders the list of layers to add to the map.

            Returns:
                layers (list): list of layer entries suitable for use in mapbox-gl 'map.addLayer()' call


@contextlib.contextmanager<EOL><INDENT>def assertWarns(self, warning, *args, **kwargs):<DEDENT>

original_filters = warnings.filters[:]<EOL>warnings.simplefilter('<STR_LIT:error>')<EOL>if len(args) == <NUM_LIT:0> and len(kwargs) == <NUM_LIT:0>:<EOL><INDENT>with self.assertRaises(warning):<EOL><INDENT>yield<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.assertRaises(warning, *args, **kwargs)<EOL><DEDENT>warnings.filters = original_filters<EOL>

A test that checks if a specified wanring was raised


return line.rstrip()<EOL>

Returns string with trailing whitespace characters removed

        Argument:
            line - Input line to cut


return line<EOL>

Returns line untouched, expected to be byte array.

        Argument:
            line - Input line to cut


return comma_list.split("<STR_LIT:U+002C>")<EOL>

Takes a string l and returns list split by comma

    Used to take comma delimited list from command line argument, and store
    python list when parsing via argparser.

    Example:
    >>>lst("1,2,3,4")
    [1,2,3,4]


<EOL>parser = argparse.ArgumentParser(description="<STR_LIT>"<EOL>+ "<STR_LIT>",<EOL>usage=_usage()[len('<STR_LIT>'):])<EOL>parser.add_argument('<STR_LIT>', "<STR_LIT>", action='<STR_LIT:store>', type=lst, default=[],<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT:-c>', "<STR_LIT>", action='<STR_LIT:store>', type=lst, default=[],<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', "<STR_LIT>", action='<STR_LIT:store>', type=lst, default=[],<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', "<STR_LIT>", action='<STR_LIT:store>', default="<STR_LIT:\t>",<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', "<STR_LIT>", action='<STR_LIT:store_true>',<EOL>help='<STR_LIT>'+<EOL>'<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', '<STR_LIT>', action='<STR_LIT:store_true>',<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT>', "<STR_LIT>", action='<STR_LIT:store>', default="<STR_LIT:\t>",<EOL>help="<STR_LIT>")<EOL>parser.add_argument('<STR_LIT:file>', nargs='<STR_LIT:*>', default="<STR_LIT:->",<EOL>help="<STR_LIT>")<EOL>return parser.parse_args(args)<EOL>

Setup argparser to process arguments and generate help


if group:<EOL><INDENT>return int(group)<EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>

Returns value of regular expression group, if valid. 0 if not

    Argument:
        group - group to get value of


result = []<EOL>line = self.line(line)<EOL>for i, field in enumerate(self.positions):<EOL><INDENT>try:<EOL><INDENT>index = _setup_index(field)<EOL>try:<EOL><INDENT>result += line[index]<EOL><DEDENT>except IndexError:<EOL><INDENT>result.append(self.invalid_pos)<EOL><DEDENT><DEDENT>except ValueError:<EOL><INDENT>result.append(str(field))<EOL><DEDENT>except TypeError:<EOL><INDENT>result.extend(self._cut_range(line, int(field[<NUM_LIT:0>]), i))<EOL><DEDENT><DEDENT>return '<STR_LIT>'.join(result)<EOL>

Returns selected positions from cut input source in desired
        arrangement.

        Argument:
            line -      input to cut


raise NotImplemented()<EOL>

Update the (default) remote by pushing the tags and removing (deleted) tags. TODO: is this last thing possible?
:return:


for root, dirs, files in os.walk(repository.working_dir):<EOL><INDENT>for f in files:<EOL><INDENT>relative_path = os.path.join(root, f)[len(repository.working_dir) + <NUM_LIT:1>:]<EOL>try:<EOL><INDENT>repository.head.commit.tree[relative_path] <EOL>yield relative_path<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>

:return: the files that have been modified and can be added


def record_results_and_push(self, results, remote_name='<STR_LIT>'):

<EOL>raise NotImplemented()<EOL>

Record the results of this experiment, by updating the tag and push to remote
:param results:  A dictionary containing the results of the experiment.
:type results: dict
:param remote_name: the name of the remote to push the tags (or None for the default)
:type remote_name: str


def __init__(self, name, parameters, directory="<STR_LIT:.>", tag_prefix="<STR_LIT>", description=None):

<EOL>self.__experiment_name = name + "<STR_LIT:@>" + time.strftime("<STR_LIT>") + '<STR_LIT:.>' + str(random.randint(<NUM_LIT:0>, <NUM_LIT:100>))<EOL>self.__results_recorded = False<EOL>self.__repository_directory = directory<EOL>if tag_prefix[-<NUM_LIT:1>] != '<STR_LIT:/>':<EOL><INDENT>tag_prefix += '<STR_LIT:/>'<EOL><DEDENT>self.__tag_name = tag_prefix + self.__experiment_name<EOL>self.__parameters = parameters<EOL>self.__description = description<EOL>

Start logging a new experiment.
:param name: the name of the experiment
:type name: str
:param parameters: a dictionary with all the parameters of the experiment.
:type parameters: dict
:param directory: a string of the directory of the git repository, where the experiment will be logged.
:type directory: str
:param tag_prefix: the prefix of the "folder" where the experiment-related tags will be placed
:type tag_prefix: str


raise NotImplementedError("<STR_LIT>")<EOL>

Called when a task for this operator is initialized within a worker on the cluster

        It provides the operator with the environment in which the bolt executes.
        Note that ``__init__()`` should not be overriden for initialization of a bolt, as it is used
        internally by BaseBolt; instead, ``initialize()`` should be used to initialize any custom
        variables or connection to databases.

        :type context: :class:`Context`
        :param context: This object can be used to get information about this task's place within the
                        topology, including the task id and component id of this task, input and output
                        information, etc.
        **Must be implemented by a subclass, otherwise NotImplementedError is raised.**


Process a single tuple of input

        The Tuple object contains metadata on it about which component/stream/task it came from.
        To emit a tuple, call ``context.emit(tuple)``.

        **Must be implemented by a subclass, otherwise NotImplementedError is raised.**

        :type tup: :class:`Tuple`
        :param tup: Tuple to process


stage_names = sets.Set()<EOL>for source in self._sources:<EOL><INDENT>source._build(bldr, stage_names)<EOL><DEDENT>for source in self._sources:<EOL><INDENT>if not source._all_built():<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT><DEDENT>

Builds the topology and returns the builder


source_streamlet = None<EOL>if callable(source):<EOL><INDENT>source_streamlet = SupplierStreamlet(source)<EOL><DEDENT>elif isinstance(source, Generator):<EOL><INDENT>source_streamlet = GeneratorStreamlet(source)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>self._sources.append(source_streamlet)<EOL>return source_streamlet<EOL>

Adds a new source to the computation DAG


pass<EOL>

Fetches the stream name that we are operating on


pass<EOL>

Fetches the task id of the current instance of the operator


pass<EOL>

The state where components can store any of their local state


self._name = name<EOL>return self<EOL>


def outer_right_join(self, join_streamlet, window_config, join_function):

from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt<EOL>join_streamlet_result = JoinStreamlet(JoinBolt.OUTER_RIGHT, window_config,<EOL>join_function, self, join_streamlet)<EOL>self._add_child(join_streamlet_result)<EOL>join_streamlet._add_child(join_streamlet_result)<EOL>return join_streamlet_result<EOL>

Return a new Streamlet by outer right join_streamlet with this streamlet


@abstractmethod<EOL><INDENT>def _build_this(self, builder, stage_names):<DEDENT>

raise RuntimeError("<STR_LIT>")<EOL>

This is the method that's implemented by the operators.
        :type builder: TopologyBuilder
        :param builder: The operator adds in the current streamlet as a spout/bolt


def reduce_by_window(self, window_config, reduce_function):

from heronpy.streamlet.impl.reducebywindowbolt import ReduceByWindowStreamlet<EOL>reduce_streamlet = ReduceByWindowStreamlet(window_config, reduce_function, self)<EOL>self._add_child(reduce_streamlet)<EOL>return reduce_streamlet<EOL>

Return a new Streamlet in which each element of this Streamlet are collected
          over a window defined by window_config and then reduced using the reduce_function
          reduce_function takes two element at one time and reduces them to one element that
          is used in the subsequent operations.


self._num_partitions = num_partitions<EOL>return self<EOL>


from heronpy.streamlet.impl.consumebolt import ConsumeStreamlet<EOL>consume_streamlet = ConsumeStreamlet(consume_function, self)<EOL>self._add_child(consume_streamlet)<EOL>return<EOL>

Calls consume_function for each element of this streamlet. This function returns nothing


from heronpy.streamlet.impl.flatmapbolt import FlatMapStreamlet<EOL>fm_streamlet = FlatMapStreamlet(flatmap_function, self)<EOL>self._add_child(fm_streamlet)<EOL>return fm_streamlet<EOL>

Return a new Streamlet by applying map_function to each element of this Streamlet
           and flattening the result


def outer_join(self, join_streamlet, window_config, join_function):

from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt<EOL>join_streamlet_result = JoinStreamlet(JoinBolt.OUTER, window_config,<EOL>join_function, self, join_streamlet)<EOL>self._add_child(join_streamlet_result)<EOL>join_streamlet._add_child(join_streamlet_result)<EOL>return join_streamlet_result<EOL>

Return a new Streamlet by outer join_streamlet with this streamlet


retval = []<EOL>for i in range(num_clones):<EOL><INDENT>retval.append(self.repartition(self.get_num_partitions()))<EOL><DEDENT>return retval<EOL>

Return num_clones number of streamlets each containing all elements
        of the current streamlet


from heronpy.streamlet.impl.filterbolt import FilterStreamlet<EOL>filter_streamlet = FilterStreamlet(filter_function, self)<EOL>self._add_child(filter_streamlet)<EOL>return filter_streamlet<EOL>

Return a new Streamlet containing only the elements that satisfy filter_function


Generate the next element
        If there is nothing at the moment to generate, return None

        **Must be implemented by a subclass, otherwise NotImplementedError is raised.**


Called when a task for this operator is initialized within a worker on the cluster

        It provides the operator with the environment in which the operator executes.
        This should be used to initialize any custom variables or connection to databases.

        :type context: :class:`Context`
        :param context: This object can be used to get information about this task's place within the
                        topology, including the task id and component id of this task, input and output
                        information, etc.
        **Must be implemented by a subclass, otherwise NotImplementedError is raised.**


if not isinstance(name, str):<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>if not isinstance(config, Config):<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>if not isinstance(builder, Builder):<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>bldr = TopologyBuilder(name=name)<EOL>builder.build(bldr)<EOL>bldr.set_config(config._api_config)<EOL>bldr.build_and_submit()<EOL>


pass<EOL>


pass<EOL>

Implements a custom stream grouping

        :param values: the values to group on
        :rtype: list of int
        :return: list of task ids to which these values are emitted


heron_options_raw = os.environ.get("<STR_LIT>")<EOL>if heron_options_raw is None:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>options = {}<EOL>for option_line in heron_options_raw.replace("<STR_LIT>", "<STR_LIT:U+0020>").split('<STR_LIT:U+002C>'):<EOL><INDENT>key, sep, value = option_line.partition("<STR_LIT:=>")<EOL>if sep:<EOL><INDENT>options[key] = value<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>" % option_line)<EOL><DEDENT><DEDENT>return options<EOL>

Retrieves heron options from the `HERON_OPTIONS` environment variable.

        Heron options have the following format:

            cmdline.topologydefn.tmpdirectory=/var/folders/tmpdir
            cmdline.topology.initial.state=PAUSED

        In this case, the returned map will contain:

            #!json
            {
              "cmdline.topologydefn.tmpdirectory": "/var/folders/tmpdir",
              "cmdline.topology.initial.state": "PAUSED"
            }

        Currently supports the following options natively:

        - `cmdline.topologydefn.tmpdirectory`: (required) the directory to which this
        topology's defn file is written
        - `cmdline.topology.initial.state`: (default: "RUNNING") the initial state of the topology
        - `cmdline.topology.name`: (default: class name) topology name on deployment

        Returns: map mapping from key to value


def add_spout(self, name, spout_cls, par, config=None, optional_outputs=None):

spout_spec = spout_cls.spec(name=name, par=par, config=config,<EOL>optional_outputs=optional_outputs)<EOL>self.add_spec(spout_spec)<EOL>return spout_spec<EOL>


for spec in specs:<EOL><INDENT>if not isinstance(spec, HeronComponentSpec):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>% str(spec))<EOL><DEDENT>if spec.name is None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if spec.name == "<STR_LIT>":<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if spec.name in self._specs:<EOL><INDENT>raise ValueError("<STR_LIT>" % (spec.name, spec))<EOL><DEDENT>self._specs[spec.name] = spec<EOL><DEDENT>

Add specs to the topology

        :type specs: HeronComponentSpec
        :param specs: specs to add to the topology


if not isinstance(config, dict):<EOL><INDENT>raise TypeError("<STR_LIT>" % str(config))<EOL><DEDENT>self._topology_config = config<EOL>

Set topology-wide configuration to the topology

        :type config: dict
        :param config: topology-wide config


@classmethod<EOL><INDENT>def init_topology(mcs, classname, class_dict):<DEDENT>

if classname == '<STR_LIT>':<EOL><INDENT>return<EOL><DEDENT>heron_options = TopologyType.get_heron_options_from_env()<EOL>initial_state = heron_options.get("<STR_LIT>", "<STR_LIT>")<EOL>tmp_directory = heron_options.get("<STR_LIT>")<EOL>if tmp_directory is None:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>topology_name = heron_options.get("<STR_LIT>", classname)<EOL>topology_id = topology_name + str(uuid.uuid4())<EOL>topology = topology_pb2.Topology()<EOL>topology.id = topology_id<EOL>topology.name = topology_name<EOL>topology.state = topology_pb2.TopologyState.Value(initial_state)<EOL>topology.topology_config.CopyFrom(TopologyType.get_topology_config_protobuf(class_dict))<EOL>TopologyType.add_bolts_and_spouts(topology, class_dict)<EOL>class_dict['<STR_LIT>'] = topology_name<EOL>class_dict['<STR_LIT>'] = topology_id<EOL>class_dict['<STR_LIT>'] = topology<EOL>class_dict['<STR_LIT>'] = tmp_directory<EOL>class_dict['<STR_LIT>'] = heron_options<EOL>


if cls.__name__ == '<STR_LIT>':<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>filename = "<STR_LIT>" % cls.topology_name<EOL>path = os.path.join(cls.topologydefn_tmpdir, filename)<EOL>with open(path, '<STR_LIT:wb>') as f:<EOL><INDENT>f.write(cls.protobuf_topology.SerializeToString())<EOL><DEDENT>

Writes the Topology .defn file to ``dest``

        This classmethod is meant be used by heron-cli when submitting a topology.


class_dict = self._construct_topo_class_dict()<EOL>topo_cls = TopologyType(self.topology_name, (Topology,), class_dict)<EOL>topo_cls.write()<EOL>

Builds the topology and submits to the destination


self.reducer.reduce(value)<EOL>


pass<EOL>

Called when this reducer is initialized/reinitialized


self.reducer = reducer_cls()<EOL>self.reducer.init()<EOL>

Initializes ReducedMetric object

        :param reducer_cls: IReducer class to use


self.value = {}<EOL>self.reducer = reducer<EOL>

Initializes MultiReducedMetric object

        :param reducer: IReducer class to use


def incr(self, key, to_add=<NUM_LIT:1>):

if key not in self.value:<EOL><INDENT>self.value[key] = CountMetric()<EOL><DEDENT>self.value[key].incr(to_add)<EOL>

Increments the value of a given key by ``to_add``


pass<EOL>

Gets the value corresponding to a key
        :param key: The key whose value we want back
        :return: The value associated with the key


@abstractmethod<EOL><INDENT>def put(self, key, value):<DEDENT>

pass<EOL>

Puts {key, value} pair into the state
        :param key: The key to get back the value
        :param value: The value associated with the key


return self.get_sources(self.get_component_id())<EOL>

Returns the declared inputs to this component
        :return: map <streamId namedtuple (same structure as protobuf msg) -> gtype>, or
                 None if not found


pass<EOL>

Returns the name of the topology
        :return: the name of the topology


raise NotImplementedError("<STR_LIT>")<EOL>

Process a single tuple of input

        The Tuple object contains metadata on it about which component/stream/task it came from.
        To emit a tuple, call ``self.emit(tuple)``.
        Note that tick tuples are not passed to this method, as the ``process_tick()`` method is
        responsible for processing them.

        **Must be implemented by a subclass, otherwise NotImplementedError is raised.**

        :type tup: :class:`Tuple`
        :param tup: Tuple to process


curtime = int(time.time())<EOL>self.current_tuples.append((tup, curtime))<EOL>self._expire(curtime)<EOL>

Process a single tuple of input

        We add the (time, tuple) pair into our current_tuples. And then look for expiring
        elemnents


curtime = int(time.time())<EOL>window_info = WindowContext(curtime - self.window_duration, curtime)<EOL>self.processWindow(window_info, list(self.current_tuples))<EOL>for tup in self.current_tuples:<EOL><INDENT>self.ack(tup)<EOL><DEDENT>self.current_tuples.clear()<EOL>


@abstractmethod<EOL><INDENT>def processWindow(self, window_info, tuples):<DEDENT>

pass<EOL>

The main interface that needs to be implemented.

        This function is called every WINDOW_DURATION_SECS seconds
        and contains the data in the last WINDOW_DURATION_SECS seconds
        in a list tuples

        :type window_info: :class:`WindowContext`
        :param window_info: The information about the window

        :type tuples: :class:`list of Tuples`
        :param tuples: The list of tuples in this window


self.delegate.fail(tup)<EOL>

Indicate that processing of a Tuple has failed

        It is compatible with StreamParse API.


@classmethod<EOL><INDENT>def spec(cls, name=None, inputs=None, par=<NUM_LIT:1>, config=None, optional_outputs=None):<DEDENT>

python_class_path = "<STR_LIT>" % (cls.__module__, cls.__name__)<EOL>if hasattr(cls, '<STR_LIT>'):<EOL><INDENT>_outputs = copy.copy(cls.outputs)<EOL><DEDENT>else:<EOL><INDENT>_outputs = []<EOL><DEDENT>if optional_outputs is not None:<EOL><INDENT>assert isinstance(optional_outputs, (list, tuple))<EOL>for out in optional_outputs:<EOL><INDENT>assert isinstance(out, (str, Stream))<EOL>_outputs.append(out)<EOL><DEDENT><DEDENT>return HeronComponentSpec(name, python_class_path, is_spout=False, par=par,<EOL>inputs=inputs, outputs=_outputs, config=config)<EOL>

Register this bolt to the topology and create ``HeronComponentSpec``

        This method takes an optional ``outputs`` argument for supporting dynamic output fields
        declaration. However, it is recommended that ``outputs`` should be declared as
        an attribute of your ``Bolt`` subclass. Also, some ways of declaring inputs is not supported
        in this implementation; please read the documentation below.

        :type name: str
        :param name: Name of this bolt.
        :type inputs: dict or list
        :param inputs: Streams that feed into this Bolt.

                       Two forms of this are acceptable:

                       1. A `dict` mapping from ``HeronComponentSpec`` to ``Grouping``.
                          In this case, default stream is used.
                       2. A `dict` mapping from ``GlobalStreamId`` to ``Grouping``.
                          This ``GlobalStreamId`` object itself is different from StreamParse, because
                          Heron does not use thrift, although its constructor method is compatible.
                       3. A `list` of ``HeronComponentSpec``. In this case, default stream with
                          SHUFFLE grouping is used.
                       4. A `list` of ``GlobalStreamId``. In this case, SHUFFLE grouping is used.
        :type par: int
        :param par: Parallelism hint for this spout.
        :type config: dict
        :param config: Component-specific config settings.
        :type optional_outputs: list of (str or Stream) or tuple of (str or Stream)
        :param optional_outputs: Additional output fields for this bolt. These fields are added to
                                 existing ``outputs`` class attributes of your bolt. This is an optional
                                 argument, and exists only for supporting dynamic output field
                                 declaration.


return tup.stream == TupleHelper.TICK_TUPLE_ID<EOL>

Returns whether or not a given HeronTuple is a tick Tuple

        It is compatible with StreamParse API.


if not isinstance(custom_config, dict):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>% str(type(custom_config)))<EOL><DEDENT>sanitized = {}<EOL>for key, value in custom_config.items():<EOL><INDENT>if not isinstance(key, str):<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>% (str(type(key)), str(key)))<EOL><DEDENT>if isinstance(value, bool):<EOL><INDENT>sanitized[key] = "<STR_LIT:true>" if value else "<STR_LIT:false>"<EOL><DEDENT>elif isinstance(value, (str, int, float)):<EOL><INDENT>sanitized[key] = str(value)<EOL><DEDENT>else:<EOL><INDENT>sanitized[key] = value<EOL><DEDENT><DEDENT>return sanitized<EOL>

Checks whether ``custom_config`` is sane and returns a sanitized dict <str -> (str|object)>

        It checks if keys are all strings and sanitizes values of a given dictionary as follows:

        - If string, number or boolean is given as a value, it is converted to string.
          For string and number (int, float), it is converted to string by a built-in ``str()`` method.
          For a boolean value, ``True`` is converted to "true" instead of "True", and ``False`` is
          converted to "false" instead of "False", in order to keep the consistency with
          Java configuration.

        - If neither of the above is given as a value, it is inserted into the sanitized dict as it is.
          These values will need to be serialized before adding to a protobuf message.


if stream_id not in self.get_out_streamids():<EOL><INDENT>raise ValueError("<STR_LIT>" % stream_id)<EOL><DEDENT>component_id = self.name or self<EOL>return GlobalStreamId(componentId=component_id, streamId=stream_id)<EOL>

Get GlobalStreamId for a given stream_id